s3stream 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +21 -11
- data/Rakefile +1 -18
- data/lib/s3stream/buffer_growth.rb +42 -0
- data/lib/s3stream/constants.rb +16 -0
- data/lib/s3stream/credentials.rb +34 -0
- data/lib/s3stream/main.rb +51 -0
- data/lib/s3stream/version.rb +1 -1
- data/lib/s3stream.rb +9 -30
- data/s3stream.gemspec +6 -8
- metadata +34 -56
- data/Guardfile +0 -14
- data/spec/lib/s3stream_spec.rb +0 -3
- data/spec/spec_helper.rb +0 -7
data/README
CHANGED
@@ -6,23 +6,33 @@ Installation
|
|
6
6
|
Configuration
|
7
7
|
=============
|
8
8
|
|
9
|
-
|
10
|
-
|
9
|
+
Log into Amazon Web Services, and lookup your access key ID and secret.
|
10
|
+
https://aws-portal.amazon.com/gp/aws/developer/account/index.html?action=access-key
|
11
|
+
|
12
|
+
Here, I'll pretend that your access key is "ABCDEFGHIJKLMNOPQ" and your secret
|
13
|
+
is "12345678901234567890abcde". Alright then...
|
14
|
+
|
15
|
+
% echo 'AWSAccessKeyId=ABCDEFGHIJKLMNOPQ' > ~/.aws-credentials
|
16
|
+
% echo 'AWSSecretKey=12345678901234567890abcde' >> ~/.aws-credentials
|
17
|
+
% export AWS_CREDENTIAL_FILE=~/.aws-credentials
|
18
|
+
|
19
|
+
- OR -
|
20
|
+
|
21
|
+
% export AMAZON_ACCESS_KEY_ID='ABCDEFGHIJKLMNOPQ'
|
22
|
+
% export AMAZON_SECRET_ACCESS_KEY='12345678901234567890abcde'
|
23
|
+
|
24
|
+
- OR -
|
25
|
+
|
26
|
+
% export AWS_ACCESS_KEY='ABCDEFGHIJKLMNOPQ'
|
27
|
+
% export AWS_SECRET_KEY='12345678901234567890abcde'
|
28
|
+
|
11
29
|
|
12
30
|
Usage
|
13
31
|
=====
|
14
|
-
From the command line:
|
15
32
|
|
16
33
|
% s3stream fetch mybucket myfile.dat > /tmp/myfile.dat
|
17
34
|
% s3stream store mybucket myfile.dat < /tmp/myfile.dat
|
18
35
|
|
19
36
|
And if you want to get fancy:
|
20
37
|
|
21
|
-
% cat <(s3stream fetch mybucket log0.log.gz | gunzip) <(s3stream fetch mybucket log1.log.gz | gunzip) |
|
22
|
-
|
23
|
-
|
24
|
-
Development
|
25
|
-
===========
|
26
|
-
gem install bundler
|
27
|
-
bundle install
|
28
|
-
guard
|
38
|
+
% cat <(s3stream fetch mybucket log0.log.gz | gunzip) <(s3stream fetch mybucket log1.log.gz | gunzip) | lzop -9 | s3stream store mybucket combined.log.lzo
|
data/Rakefile
CHANGED
@@ -1,24 +1,7 @@
|
|
1
1
|
require "bundler/gem_tasks"
|
2
|
-
require 'rspec/core/rake_task'
|
3
|
-
|
4
|
-
desc "Run all specs"
|
5
|
-
RSpec::Core::RakeTask.new(:spec) do |t|
|
6
|
-
t.rspec_opts = [
|
7
|
-
'-c',
|
8
|
-
'--format documentation',
|
9
|
-
'-r ./spec/spec_helper.rb'
|
10
|
-
]
|
11
|
-
t.pattern = 'spec/**/*_spec.rb'
|
12
|
-
end
|
13
|
-
|
14
|
-
desc "open coverage report"
|
15
|
-
task :coverage do
|
16
|
-
system 'rake spec'
|
17
|
-
system 'open coverage/index.html'
|
18
|
-
end
|
19
2
|
|
20
3
|
desc "Open development console"
|
21
4
|
task :console do
|
22
5
|
puts "Loading development console..."
|
23
|
-
system "
|
6
|
+
system "pry -I #{File.join('.', 'lib')} -r #{File.join('.', 'lib', 's3stream')}"
|
24
7
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module S3Stream
|
2
|
+
class BufferGrowth
|
3
|
+
|
4
|
+
EPSILON = 1.0e-16
|
5
|
+
|
6
|
+
def initialize(initial_buffer_size, max_file_size, max_chunks)
|
7
|
+
@max_chunks = max_chunks
|
8
|
+
@max_file_size = max_file_size
|
9
|
+
@initial_buffer_size = initial_buffer_size
|
10
|
+
end
|
11
|
+
|
12
|
+
def solve
|
13
|
+
lower_bound = 1.0
|
14
|
+
upper_bound = 1.0
|
15
|
+
upper_bound *= 2.0 until too_big?(upper_bound)
|
16
|
+
begin
|
17
|
+
puts "bounds: #{[lower_bound, upper_bound]}"
|
18
|
+
return lower_bound if (upper_bound - lower_bound) < EPSILON
|
19
|
+
guess = (lower_bound + upper_bound) / 2.0
|
20
|
+
return lower_bound if [lower_bound,upper_bound].include?(guess)
|
21
|
+
if too_big?(guess)
|
22
|
+
upper_bound = guess
|
23
|
+
else
|
24
|
+
lower_bound = guess
|
25
|
+
end
|
26
|
+
end while true
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def too_big?(growth_factor)
|
32
|
+
total = 0
|
33
|
+
buffer_size = @initial_buffer_size
|
34
|
+
(1..@max_chunks).each do
|
35
|
+
total += buffer_size
|
36
|
+
buffer_size = (buffer_size * growth_factor).to_i
|
37
|
+
return true if total > @max_file_size
|
38
|
+
end
|
39
|
+
false
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module S3Stream
|
2
|
+
KB = 1024
|
3
|
+
MB = 1024 * KB
|
4
|
+
GB = 1024 * MB
|
5
|
+
TB = 1024 * GB
|
6
|
+
|
7
|
+
MAX_CHUNKS = 10_000
|
8
|
+
MAX_FILE_SIZE = 5 * TB
|
9
|
+
INITIAL_BUFFER_SIZE = 5 * MB
|
10
|
+
|
11
|
+
BUFFER_GROWTH_FACTOR = 1.0006533241143831
|
12
|
+
#BUFFER_GROWTH_FACTOR = begin
|
13
|
+
# require 's3stream/buffer_growth'
|
14
|
+
# BufferGrowth.new(INITIAL_BUFFER_SIZE, MAX_FILE_SIZE, MAX_CHUNKS).solve
|
15
|
+
#end
|
16
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module S3Stream
|
2
|
+
|
3
|
+
private
|
4
|
+
|
5
|
+
def self.credentials
|
6
|
+
access_key_id, secret_access_key = nil
|
7
|
+
|
8
|
+
# Try AWS tools' standard way
|
9
|
+
if File.file?(ENV['AWS_CREDENTIAL_FILE'])
|
10
|
+
File.open(ENV['AWS_CREDENTIAL_FILE']) do |f|
|
11
|
+
f.lines.each do |line|
|
12
|
+
access_key_id = $1 if line =~ /^AWSAccessKeyId=(.*)$/
|
13
|
+
secret_access_key = $1 if line =~ /^AWSSecretKey=(.*)$/
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# Try aws-s3 gem's way
|
19
|
+
access_key_id ||= ENV['AMAZON_ACCESS_KEY_ID']
|
20
|
+
secret_access_key ||= ENV['AMAZON_SECRET_ACCESS_KEY']
|
21
|
+
|
22
|
+
# Try s3cmd gem's way
|
23
|
+
access_key_id ||= ENV['AWS_ACCESS_KEY']
|
24
|
+
secret_access_key ||= ENV['AWS_SECRET_KEY']
|
25
|
+
|
26
|
+
if access_key_id.nil? || secret_access_key.nil?
|
27
|
+
nil
|
28
|
+
else
|
29
|
+
{:access_key_id => access_key_id, :secret_access_key => secret_access_key}
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
CREDENTIALS = S3Stream.credentials
|
34
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module S3Stream
|
2
|
+
class Main < Thor
|
3
|
+
desc "fetch bucket filename", "download/stream the file from S3 to stdout"
|
4
|
+
def fetch(bucket_name, filename)
|
5
|
+
require "aws/s3"
|
6
|
+
AWS::S3::Base.establish_connection!(S3Stream::CREDENTIALS)
|
7
|
+
AWS::S3::S3Object.stream(filename, bucket_name) do |chunk|
|
8
|
+
$stdout.write chunk
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
desc "store bucket filename", "upload/stream the file from stdin to S3"
|
13
|
+
def store(bucket_name, filename)
|
14
|
+
require "aws-sdk"
|
15
|
+
$stdout.sync = true
|
16
|
+
s3 = AWS::S3.new(S3Stream::CREDENTIALS)
|
17
|
+
bucket = s3.buckets[bucket_name]
|
18
|
+
object = bucket.objects[filename]
|
19
|
+
buffer = ""
|
20
|
+
total = 0
|
21
|
+
buffer_size = INITIAL_BUFFER_SIZE
|
22
|
+
puts "Uploading, please be patient."
|
23
|
+
object.multipart_upload do |upload|
|
24
|
+
(1..MAX_CHUNKS).each do |chunk|
|
25
|
+
if $stdin.eof?
|
26
|
+
puts "End of input."
|
27
|
+
break
|
28
|
+
end
|
29
|
+
if chunk < MAX_CHUNKS
|
30
|
+
print "Buffering input (up to #{buffer_size} bytes) ... "
|
31
|
+
$stdin.read(buffer_size, buffer)
|
32
|
+
puts "done."
|
33
|
+
print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
|
34
|
+
upload.add_part(buffer)
|
35
|
+
puts "done."
|
36
|
+
buffer_size = (buffer_size * BUFFER_GROWTH_FACTOR).to_i
|
37
|
+
else
|
38
|
+
print "Last part (#{chunk})! Buffering input (unlimited) ..."
|
39
|
+
buffer = $stdin.read
|
40
|
+
puts "done."
|
41
|
+
print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
|
42
|
+
upload.add_part(buffer)
|
43
|
+
puts "done."
|
44
|
+
end
|
45
|
+
total += buffer.size
|
46
|
+
end
|
47
|
+
end
|
48
|
+
puts "Done uploading to s3://#{bucket_name}/#{filename} (#{total} bytes)"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
data/lib/s3stream/version.rb
CHANGED
data/lib/s3stream.rb
CHANGED
@@ -1,32 +1,11 @@
|
|
1
|
-
require "s3stream/version"
|
2
1
|
require "thor"
|
3
|
-
require "
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
end
|
13
|
-
|
14
|
-
desc "store bucket filename", "upload/stream the file from stdin to S3"
|
15
|
-
def store(bucket, filename)
|
16
|
-
AWS::S3::S3Object.store(filename, ARGF, bucket)
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def self.usage
|
21
|
-
puts "Please set AMAZON_ACCESS_KEY_ID and AMAZON_SECRET_ACCESS_KEY environmental variables."
|
22
|
-
exit(1)
|
23
|
-
end
|
2
|
+
require "s3stream/version"
|
3
|
+
require "s3stream/constants"
|
4
|
+
require "s3stream/main"
|
5
|
+
require "s3stream/credentials"
|
6
|
+
|
7
|
+
if S3Stream::CREDENTIALS.nil?
|
8
|
+
puts "Environmental variables must be set."
|
9
|
+
puts "See https://github.com/kindkid/s3stream/blob/v#{S3Stream::VERSION}/README"
|
10
|
+
exit(1)
|
24
11
|
end
|
25
|
-
|
26
|
-
AMAZON_ACCESS_KEY_ID = ENV['AMAZON_ACCESS_KEY_ID'] || S3Stream.usage
|
27
|
-
AMAZON_SECRET_ACCESS_KEY = ENV['AMAZON_SECRET_ACCESS_KEY'] || S3Stream.usage
|
28
|
-
|
29
|
-
AWS::S3::Base.establish_connection!(
|
30
|
-
:access_key_id => AMAZON_ACCESS_KEY_ID,
|
31
|
-
:secret_access_key => AMAZON_SECRET_ACCESS_KEY
|
32
|
-
)
|
data/s3stream.gemspec
CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
|
|
7
7
|
s.version = S3Stream::VERSION
|
8
8
|
s.authors = ["Chris Johnson"]
|
9
9
|
s.email = ["chris@kindkid.com"]
|
10
|
-
s.homepage = ""
|
10
|
+
s.homepage = "https://github.com/kindkid/s3stream"
|
11
11
|
s.summary = "Stream files on S3 to stdout or from stdin"
|
12
12
|
s.description = s.summary
|
13
13
|
|
@@ -18,12 +18,10 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
19
|
s.require_paths = ["lib"]
|
20
20
|
|
21
|
-
s.add_runtime_dependency "aws-s3", "~> 0.6.2"
|
22
21
|
s.add_runtime_dependency "thor", "~> 0.14.6"
|
23
|
-
s.add_development_dependency "
|
24
|
-
s.add_development_dependency "
|
25
|
-
s.add_development_dependency
|
26
|
-
s.add_development_dependency "
|
27
|
-
s.add_development_dependency "
|
28
|
-
s.add_development_dependency "guard-rspec", "~> 0.4"
|
22
|
+
s.add_development_dependency "aws-sdk", "~> 1.3.5" #actually a runtime dependency
|
23
|
+
s.add_development_dependency "aws-s3", "~> 0.6.2" #actually a runtime dependency
|
24
|
+
s.add_development_dependency "pry"
|
25
|
+
s.add_development_dependency "pry-doc"
|
26
|
+
s.add_development_dependency "pry-nav"
|
29
27
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3stream
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,22 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-02-
|
12
|
+
date: 2012-02-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
-
- !ruby/object:Gem::Dependency
|
15
|
-
name: aws-s3
|
16
|
-
requirement: &70254260784700 !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
|
-
requirements:
|
19
|
-
- - ~>
|
20
|
-
- !ruby/object:Gem::Version
|
21
|
-
version: 0.6.2
|
22
|
-
type: :runtime
|
23
|
-
prerelease: false
|
24
|
-
version_requirements: *70254260784700
|
25
14
|
- !ruby/object:Gem::Dependency
|
26
15
|
name: thor
|
27
|
-
requirement: &
|
16
|
+
requirement: &70321780913920 !ruby/object:Gem::Requirement
|
28
17
|
none: false
|
29
18
|
requirements:
|
30
19
|
- - ~>
|
@@ -32,73 +21,62 @@ dependencies:
|
|
32
21
|
version: 0.14.6
|
33
22
|
type: :runtime
|
34
23
|
prerelease: false
|
35
|
-
version_requirements: *
|
24
|
+
version_requirements: *70321780913920
|
36
25
|
- !ruby/object:Gem::Dependency
|
37
|
-
name:
|
38
|
-
requirement: &
|
26
|
+
name: aws-sdk
|
27
|
+
requirement: &70321780912980 !ruby/object:Gem::Requirement
|
39
28
|
none: false
|
40
29
|
requirements:
|
41
30
|
- - ~>
|
42
31
|
- !ruby/object:Gem::Version
|
43
|
-
version:
|
32
|
+
version: 1.3.5
|
44
33
|
type: :development
|
45
34
|
prerelease: false
|
46
|
-
version_requirements: *
|
35
|
+
version_requirements: *70321780912980
|
47
36
|
- !ruby/object:Gem::Dependency
|
48
|
-
name:
|
49
|
-
requirement: &
|
50
|
-
none: false
|
51
|
-
requirements:
|
52
|
-
- - ~>
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '0.4'
|
55
|
-
type: :development
|
56
|
-
prerelease: false
|
57
|
-
version_requirements: *70254260780500
|
58
|
-
- !ruby/object:Gem::Dependency
|
59
|
-
name: rb-fsevent
|
60
|
-
requirement: &70254260779540 !ruby/object:Gem::Requirement
|
37
|
+
name: aws-s3
|
38
|
+
requirement: &70321780923920 !ruby/object:Gem::Requirement
|
61
39
|
none: false
|
62
40
|
requirements:
|
63
41
|
- - ~>
|
64
42
|
- !ruby/object:Gem::Version
|
65
|
-
version:
|
43
|
+
version: 0.6.2
|
66
44
|
type: :development
|
67
45
|
prerelease: false
|
68
|
-
version_requirements: *
|
46
|
+
version_requirements: *70321780923920
|
69
47
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
71
|
-
requirement: &
|
48
|
+
name: pry
|
49
|
+
requirement: &70321780923240 !ruby/object:Gem::Requirement
|
72
50
|
none: false
|
73
51
|
requirements:
|
74
|
-
- -
|
52
|
+
- - ! '>='
|
75
53
|
- !ruby/object:Gem::Version
|
76
|
-
version: '0
|
54
|
+
version: '0'
|
77
55
|
type: :development
|
78
56
|
prerelease: false
|
79
|
-
version_requirements: *
|
57
|
+
version_requirements: *70321780923240
|
80
58
|
- !ruby/object:Gem::Dependency
|
81
|
-
name:
|
82
|
-
requirement: &
|
59
|
+
name: pry-doc
|
60
|
+
requirement: &70321780922600 !ruby/object:Gem::Requirement
|
83
61
|
none: false
|
84
62
|
requirements:
|
85
|
-
- -
|
63
|
+
- - ! '>='
|
86
64
|
- !ruby/object:Gem::Version
|
87
|
-
version: '0
|
65
|
+
version: '0'
|
88
66
|
type: :development
|
89
67
|
prerelease: false
|
90
|
-
version_requirements: *
|
68
|
+
version_requirements: *70321780922600
|
91
69
|
- !ruby/object:Gem::Dependency
|
92
|
-
name:
|
93
|
-
requirement: &
|
70
|
+
name: pry-nav
|
71
|
+
requirement: &70321780922040 !ruby/object:Gem::Requirement
|
94
72
|
none: false
|
95
73
|
requirements:
|
96
|
-
- -
|
74
|
+
- - ! '>='
|
97
75
|
- !ruby/object:Gem::Version
|
98
|
-
version: '0
|
76
|
+
version: '0'
|
99
77
|
type: :development
|
100
78
|
prerelease: false
|
101
|
-
version_requirements: *
|
79
|
+
version_requirements: *70321780922040
|
102
80
|
description: Stream files on S3 to stdout or from stdin
|
103
81
|
email:
|
104
82
|
- chris@kindkid.com
|
@@ -110,16 +88,17 @@ files:
|
|
110
88
|
- .gitignore
|
111
89
|
- .rvmrc
|
112
90
|
- Gemfile
|
113
|
-
- Guardfile
|
114
91
|
- README
|
115
92
|
- Rakefile
|
116
93
|
- bin/s3stream
|
117
94
|
- lib/s3stream.rb
|
95
|
+
- lib/s3stream/buffer_growth.rb
|
96
|
+
- lib/s3stream/constants.rb
|
97
|
+
- lib/s3stream/credentials.rb
|
98
|
+
- lib/s3stream/main.rb
|
118
99
|
- lib/s3stream/version.rb
|
119
100
|
- s3stream.gemspec
|
120
|
-
|
121
|
-
- spec/spec_helper.rb
|
122
|
-
homepage: ''
|
101
|
+
homepage: https://github.com/kindkid/s3stream
|
123
102
|
licenses: []
|
124
103
|
post_install_message:
|
125
104
|
rdoc_options: []
|
@@ -143,6 +122,5 @@ rubygems_version: 1.8.10
|
|
143
122
|
signing_key:
|
144
123
|
specification_version: 3
|
145
124
|
summary: Stream files on S3 to stdout or from stdin
|
146
|
-
test_files:
|
147
|
-
|
148
|
-
- spec/spec_helper.rb
|
125
|
+
test_files: []
|
126
|
+
has_rdoc:
|
data/Guardfile
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
# A sample Guardfile
|
2
|
-
# More info at https://github.com/guard/guard#readme
|
3
|
-
|
4
|
-
guard 'bundler' do
|
5
|
-
watch('Gemfile')
|
6
|
-
watch(/^.+\.gemspec/)
|
7
|
-
end
|
8
|
-
|
9
|
-
guard 'rspec', :cli => '-c --format documentation -r ./spec/spec_helper.rb',
|
10
|
-
:version => 2 do
|
11
|
-
watch(%r{^spec/.+_spec\.rb})
|
12
|
-
watch(%r{^lib/(.+)\.rb}) { |m| "spec/lib/#{m[1]}_spec.rb" }
|
13
|
-
watch('spec/spec_helper.rb') { "spec" }
|
14
|
-
end
|
data/spec/lib/s3stream_spec.rb
DELETED