s3stream 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README CHANGED
@@ -6,23 +6,33 @@ Installation
6
6
  Configuration
7
7
  =============
8
8
 
9
- export AMAZON_ACCESS_KEY_ID='abcdefghijklmnop'
10
- export AMAZON_SECRET_ACCESS_KEY='1234567891012345'
9
+ Log into Amazon Web Services, and lookup your access key ID and secret.
10
+ https://aws-portal.amazon.com/gp/aws/developer/account/index.html?action=access-key
11
+
12
+ Here, I'll pretend that your access key is "ABCDEFGHIJKLMNOPQ" and your secret
13
+ is "12345678901234567890abcde". Alright then...
14
+
15
+ % echo 'AWSAccessKeyId=ABCDEFGHIJKLMNOPQ' > ~/.aws-credentials
16
+ % echo 'AWSSecretKey=12345678901234567890abcde' >> ~/.aws-credentials
17
+ % export AWS_CREDENTIAL_FILE=~/.aws-credentials
18
+
19
+ - OR -
20
+
21
+ % export AMAZON_ACCESS_KEY_ID='ABCDEFGHIJKLMNOPQ'
22
+ % export AMAZON_SECRET_ACCESS_KEY='12345678901234567890abcde'
23
+
24
+ - OR -
25
+
26
+ % export AWS_ACCESS_KEY='ABCDEFGHIJKLMNOPQ'
27
+ % export AWS_SECRET_KEY='12345678901234567890abcde'
28
+
11
29
 
12
30
  Usage
13
31
  =====
14
- From the command line:
15
32
 
16
33
  % s3stream fetch mybucket myfile.dat > /tmp/myfile.dat
17
34
  % s3stream store mybucket myfile.dat < /tmp/myfile.dat
18
35
 
19
36
  And if you want to get fancy:
20
37
 
21
- % cat <(s3stream fetch mybucket log0.log.gz | gunzip) <(s3stream fetch mybucket log1.log.gz | gunzip) | gzip > s3stream store mybucket combined.log.gz
22
-
23
-
24
- Development
25
- ===========
26
- gem install bundler
27
- bundle install
28
- guard
38
+ % cat <(s3stream fetch mybucket log0.log.gz | gunzip) <(s3stream fetch mybucket log1.log.gz | gunzip) | lzop -9 | s3stream store mybucket combined.log.lzo
data/Rakefile CHANGED
@@ -1,24 +1,7 @@
1
1
  require "bundler/gem_tasks"
2
- require 'rspec/core/rake_task'
3
-
4
- desc "Run all specs"
5
- RSpec::Core::RakeTask.new(:spec) do |t|
6
- t.rspec_opts = [
7
- '-c',
8
- '--format documentation',
9
- '-r ./spec/spec_helper.rb'
10
- ]
11
- t.pattern = 'spec/**/*_spec.rb'
12
- end
13
-
14
- desc "open coverage report"
15
- task :coverage do
16
- system 'rake spec'
17
- system 'open coverage/index.html'
18
- end
19
2
 
20
3
  desc "Open development console"
21
4
  task :console do
22
5
  puts "Loading development console..."
23
- system "irb -I #{File.join('.', 'lib')} -r #{File.join('.', 'lib', 's3stream')}"
6
+ system "pry -I #{File.join('.', 'lib')} -r #{File.join('.', 'lib', 's3stream')}"
24
7
  end
@@ -0,0 +1,42 @@
1
+ module S3Stream
2
+ class BufferGrowth
3
+
4
+ EPSILON = 1.0e-16
5
+
6
+ def initialize(initial_buffer_size, max_file_size, max_chunks)
7
+ @max_chunks = max_chunks
8
+ @max_file_size = max_file_size
9
+ @initial_buffer_size = initial_buffer_size
10
+ end
11
+
12
+ def solve
13
+ lower_bound = 1.0
14
+ upper_bound = 1.0
15
+ upper_bound *= 2.0 until too_big?(upper_bound)
16
+ begin
17
+ puts "bounds: #{[lower_bound, upper_bound]}"
18
+ return lower_bound if (upper_bound - lower_bound) < EPSILON
19
+ guess = (lower_bound + upper_bound) / 2.0
20
+ return lower_bound if [lower_bound,upper_bound].include?(guess)
21
+ if too_big?(guess)
22
+ upper_bound = guess
23
+ else
24
+ lower_bound = guess
25
+ end
26
+ end while true
27
+ end
28
+
29
+ private
30
+
31
+ def too_big?(growth_factor)
32
+ total = 0
33
+ buffer_size = @initial_buffer_size
34
+ (1..@max_chunks).each do
35
+ total += buffer_size
36
+ buffer_size = (buffer_size * growth_factor).to_i
37
+ return true if total > @max_file_size
38
+ end
39
+ false
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,16 @@
1
+ module S3Stream
2
+ KB = 1024
3
+ MB = 1024 * KB
4
+ GB = 1024 * MB
5
+ TB = 1024 * GB
6
+
7
+ MAX_CHUNKS = 10_000
8
+ MAX_FILE_SIZE = 5 * TB
9
+ INITIAL_BUFFER_SIZE = 5 * MB
10
+
11
+ BUFFER_GROWTH_FACTOR = 1.0006533241143831
12
+ #BUFFER_GROWTH_FACTOR = begin
13
+ # require 's3stream/buffer_growth'
14
+ # BufferGrowth.new(INITIAL_BUFFER_SIZE, MAX_FILE_SIZE, MAX_CHUNKS).solve
15
+ #end
16
+ end
@@ -0,0 +1,34 @@
1
+ module S3Stream
2
+
3
+ private
4
+
5
+ def self.credentials
6
+ access_key_id, secret_access_key = nil
7
+
8
+ # Try AWS tools' standard way
9
+ if File.file?(ENV['AWS_CREDENTIAL_FILE'])
10
+ File.open(ENV['AWS_CREDENTIAL_FILE']) do |f|
11
+ f.lines.each do |line|
12
+ access_key_id = $1 if line =~ /^AWSAccessKeyId=(.*)$/
13
+ secret_access_key = $1 if line =~ /^AWSSecretKey=(.*)$/
14
+ end
15
+ end
16
+ end
17
+
18
+ # Try aws-s3 gem's way
19
+ access_key_id ||= ENV['AMAZON_ACCESS_KEY_ID']
20
+ secret_access_key ||= ENV['AMAZON_SECRET_ACCESS_KEY']
21
+
22
+ # Try s3cmd gem's way
23
+ access_key_id ||= ENV['AWS_ACCESS_KEY']
24
+ secret_access_key ||= ENV['AWS_SECRET_KEY']
25
+
26
+ if access_key_id.nil? || secret_access_key.nil?
27
+ nil
28
+ else
29
+ {:access_key_id => access_key_id, :secret_access_key => secret_access_key}
30
+ end
31
+ end
32
+
33
+ CREDENTIALS = S3Stream.credentials
34
+ end
@@ -0,0 +1,51 @@
1
+ module S3Stream
2
+ class Main < Thor
3
+ desc "fetch bucket filename", "download/stream the file from S3 to stdout"
4
+ def fetch(bucket_name, filename)
5
+ require "aws/s3"
6
+ AWS::S3::Base.establish_connection!(S3Stream::CREDENTIALS)
7
+ AWS::S3::S3Object.stream(filename, bucket_name) do |chunk|
8
+ $stdout.write chunk
9
+ end
10
+ end
11
+
12
+ desc "store bucket filename", "upload/stream the file from stdin to S3"
13
+ def store(bucket_name, filename)
14
+ require "aws-sdk"
15
+ $stdout.sync = true
16
+ s3 = AWS::S3.new(S3Stream::CREDENTIALS)
17
+ bucket = s3.buckets[bucket_name]
18
+ object = bucket.objects[filename]
19
+ buffer = ""
20
+ total = 0
21
+ buffer_size = INITIAL_BUFFER_SIZE
22
+ puts "Uploading, please be patient."
23
+ object.multipart_upload do |upload|
24
+ (1..MAX_CHUNKS).each do |chunk|
25
+ if $stdin.eof?
26
+ puts "End of input."
27
+ break
28
+ end
29
+ if chunk < MAX_CHUNKS
30
+ print "Buffering input (up to #{buffer_size} bytes) ... "
31
+ $stdin.read(buffer_size, buffer)
32
+ puts "done."
33
+ print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
34
+ upload.add_part(buffer)
35
+ puts "done."
36
+ buffer_size = (buffer_size * BUFFER_GROWTH_FACTOR).to_i
37
+ else
38
+ print "Last part (#{chunk})! Buffering input (unlimited) ..."
39
+ buffer = $stdin.read
40
+ puts "done."
41
+ print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
42
+ upload.add_part(buffer)
43
+ puts "done."
44
+ end
45
+ total += buffer.size
46
+ end
47
+ end
48
+ puts "Done uploading to s3://#{bucket_name}/#{filename} (#{total} bytes)"
49
+ end
50
+ end
51
+ end
@@ -1,3 +1,3 @@
1
1
  module S3Stream
2
- VERSION = "0.0.2"
2
+ VERSION = "0.0.3"
3
3
  end
data/lib/s3stream.rb CHANGED
@@ -1,32 +1,11 @@
1
- require "s3stream/version"
2
1
  require "thor"
3
- require "aws/s3"
4
-
5
- module S3Stream
6
- class Main < Thor
7
- desc "fetch bucket filename", "download/stream the file from S3 to stdout"
8
- def fetch(bucket, filename)
9
- AWS::S3::S3Object.stream(filename, bucket) do |chunk|
10
- $stdout.write chunk
11
- end
12
- end
13
-
14
- desc "store bucket filename", "upload/stream the file from stdin to S3"
15
- def store(bucket, filename)
16
- AWS::S3::S3Object.store(filename, ARGF, bucket)
17
- end
18
- end
19
-
20
- def self.usage
21
- puts "Please set AMAZON_ACCESS_KEY_ID and AMAZON_SECRET_ACCESS_KEY environmental variables."
22
- exit(1)
23
- end
2
+ require "s3stream/version"
3
+ require "s3stream/constants"
4
+ require "s3stream/main"
5
+ require "s3stream/credentials"
6
+
7
+ if S3Stream::CREDENTIALS.nil?
8
+ puts "Environmental variables must be set."
9
+ puts "See https://github.com/kindkid/s3stream/blob/v#{S3Stream::VERSION}/README"
10
+ exit(1)
24
11
  end
25
-
26
- AMAZON_ACCESS_KEY_ID = ENV['AMAZON_ACCESS_KEY_ID'] || S3Stream.usage
27
- AMAZON_SECRET_ACCESS_KEY = ENV['AMAZON_SECRET_ACCESS_KEY'] || S3Stream.usage
28
-
29
- AWS::S3::Base.establish_connection!(
30
- :access_key_id => AMAZON_ACCESS_KEY_ID,
31
- :secret_access_key => AMAZON_SECRET_ACCESS_KEY
32
- )
data/s3stream.gemspec CHANGED
@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
7
7
  s.version = S3Stream::VERSION
8
8
  s.authors = ["Chris Johnson"]
9
9
  s.email = ["chris@kindkid.com"]
10
- s.homepage = ""
10
+ s.homepage = "https://github.com/kindkid/s3stream"
11
11
  s.summary = "Stream files on S3 to stdout or from stdin"
12
12
  s.description = s.summary
13
13
 
@@ -18,12 +18,10 @@ Gem::Specification.new do |s|
18
18
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
19
  s.require_paths = ["lib"]
20
20
 
21
- s.add_runtime_dependency "aws-s3", "~> 0.6.2"
22
21
  s.add_runtime_dependency "thor", "~> 0.14.6"
23
- s.add_development_dependency "rspec", "~> 2.6"
24
- s.add_development_dependency "simplecov", "~> 0.4"
25
- s.add_development_dependency("rb-fsevent", "~> 0.4") if RUBY_PLATFORM =~ /darwin/i
26
- s.add_development_dependency "guard", "~> 0.5"
27
- s.add_development_dependency "guard-bundler", "~> 0.1"
28
- s.add_development_dependency "guard-rspec", "~> 0.4"
22
+ s.add_development_dependency "aws-sdk", "~> 1.3.5" #actually a runtime dependency
23
+ s.add_development_dependency "aws-s3", "~> 0.6.2" #actually a runtime dependency
24
+ s.add_development_dependency "pry"
25
+ s.add_development_dependency "pry-doc"
26
+ s.add_development_dependency "pry-nav"
29
27
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: s3stream
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,22 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-02-23 00:00:00.000000000 Z
12
+ date: 2012-02-27 00:00:00.000000000 Z
13
13
  dependencies:
14
- - !ruby/object:Gem::Dependency
15
- name: aws-s3
16
- requirement: &70254260784700 !ruby/object:Gem::Requirement
17
- none: false
18
- requirements:
19
- - - ~>
20
- - !ruby/object:Gem::Version
21
- version: 0.6.2
22
- type: :runtime
23
- prerelease: false
24
- version_requirements: *70254260784700
25
14
  - !ruby/object:Gem::Dependency
26
15
  name: thor
27
- requirement: &70254260782160 !ruby/object:Gem::Requirement
16
+ requirement: &70321780913920 !ruby/object:Gem::Requirement
28
17
  none: false
29
18
  requirements:
30
19
  - - ~>
@@ -32,73 +21,62 @@ dependencies:
32
21
  version: 0.14.6
33
22
  type: :runtime
34
23
  prerelease: false
35
- version_requirements: *70254260782160
24
+ version_requirements: *70321780913920
36
25
  - !ruby/object:Gem::Dependency
37
- name: rspec
38
- requirement: &70254260781400 !ruby/object:Gem::Requirement
26
+ name: aws-sdk
27
+ requirement: &70321780912980 !ruby/object:Gem::Requirement
39
28
  none: false
40
29
  requirements:
41
30
  - - ~>
42
31
  - !ruby/object:Gem::Version
43
- version: '2.6'
32
+ version: 1.3.5
44
33
  type: :development
45
34
  prerelease: false
46
- version_requirements: *70254260781400
35
+ version_requirements: *70321780912980
47
36
  - !ruby/object:Gem::Dependency
48
- name: simplecov
49
- requirement: &70254260780500 !ruby/object:Gem::Requirement
50
- none: false
51
- requirements:
52
- - - ~>
53
- - !ruby/object:Gem::Version
54
- version: '0.4'
55
- type: :development
56
- prerelease: false
57
- version_requirements: *70254260780500
58
- - !ruby/object:Gem::Dependency
59
- name: rb-fsevent
60
- requirement: &70254260779540 !ruby/object:Gem::Requirement
37
+ name: aws-s3
38
+ requirement: &70321780923920 !ruby/object:Gem::Requirement
61
39
  none: false
62
40
  requirements:
63
41
  - - ~>
64
42
  - !ruby/object:Gem::Version
65
- version: '0.4'
43
+ version: 0.6.2
66
44
  type: :development
67
45
  prerelease: false
68
- version_requirements: *70254260779540
46
+ version_requirements: *70321780923920
69
47
  - !ruby/object:Gem::Dependency
70
- name: guard
71
- requirement: &70254260778500 !ruby/object:Gem::Requirement
48
+ name: pry
49
+ requirement: &70321780923240 !ruby/object:Gem::Requirement
72
50
  none: false
73
51
  requirements:
74
- - - ~>
52
+ - - ! '>='
75
53
  - !ruby/object:Gem::Version
76
- version: '0.5'
54
+ version: '0'
77
55
  type: :development
78
56
  prerelease: false
79
- version_requirements: *70254260778500
57
+ version_requirements: *70321780923240
80
58
  - !ruby/object:Gem::Dependency
81
- name: guard-bundler
82
- requirement: &70254260777160 !ruby/object:Gem::Requirement
59
+ name: pry-doc
60
+ requirement: &70321780922600 !ruby/object:Gem::Requirement
83
61
  none: false
84
62
  requirements:
85
- - - ~>
63
+ - - ! '>='
86
64
  - !ruby/object:Gem::Version
87
- version: '0.1'
65
+ version: '0'
88
66
  type: :development
89
67
  prerelease: false
90
- version_requirements: *70254260777160
68
+ version_requirements: *70321780922600
91
69
  - !ruby/object:Gem::Dependency
92
- name: guard-rspec
93
- requirement: &70254260775520 !ruby/object:Gem::Requirement
70
+ name: pry-nav
71
+ requirement: &70321780922040 !ruby/object:Gem::Requirement
94
72
  none: false
95
73
  requirements:
96
- - - ~>
74
+ - - ! '>='
97
75
  - !ruby/object:Gem::Version
98
- version: '0.4'
76
+ version: '0'
99
77
  type: :development
100
78
  prerelease: false
101
- version_requirements: *70254260775520
79
+ version_requirements: *70321780922040
102
80
  description: Stream files on S3 to stdout or from stdin
103
81
  email:
104
82
  - chris@kindkid.com
@@ -110,16 +88,17 @@ files:
110
88
  - .gitignore
111
89
  - .rvmrc
112
90
  - Gemfile
113
- - Guardfile
114
91
  - README
115
92
  - Rakefile
116
93
  - bin/s3stream
117
94
  - lib/s3stream.rb
95
+ - lib/s3stream/buffer_growth.rb
96
+ - lib/s3stream/constants.rb
97
+ - lib/s3stream/credentials.rb
98
+ - lib/s3stream/main.rb
118
99
  - lib/s3stream/version.rb
119
100
  - s3stream.gemspec
120
- - spec/lib/s3stream_spec.rb
121
- - spec/spec_helper.rb
122
- homepage: ''
101
+ homepage: https://github.com/kindkid/s3stream
123
102
  licenses: []
124
103
  post_install_message:
125
104
  rdoc_options: []
@@ -143,6 +122,5 @@ rubygems_version: 1.8.10
143
122
  signing_key:
144
123
  specification_version: 3
145
124
  summary: Stream files on S3 to stdout or from stdin
146
- test_files:
147
- - spec/lib/s3stream_spec.rb
148
- - spec/spec_helper.rb
125
+ test_files: []
126
+ has_rdoc:
data/Guardfile DELETED
@@ -1,14 +0,0 @@
1
- # A sample Guardfile
2
- # More info at https://github.com/guard/guard#readme
3
-
4
- guard 'bundler' do
5
- watch('Gemfile')
6
- watch(/^.+\.gemspec/)
7
- end
8
-
9
- guard 'rspec', :cli => '-c --format documentation -r ./spec/spec_helper.rb',
10
- :version => 2 do
11
- watch(%r{^spec/.+_spec\.rb})
12
- watch(%r{^lib/(.+)\.rb}) { |m| "spec/lib/#{m[1]}_spec.rb" }
13
- watch('spec/spec_helper.rb') { "spec" }
14
- end
@@ -1,3 +0,0 @@
1
- describe S3Stream do
2
- it "should have specs"
3
- end
data/spec/spec_helper.rb DELETED
@@ -1,7 +0,0 @@
1
- require 'simplecov'
2
- SimpleCov.start
3
-
4
- require 'rubygems'
5
- require 'bundler'
6
-
7
- Bundler.require(:default, :test, :development)