s3stream 0.0.2 → 0.0.3

data/README

@@ -6,23 +6,33 @@ Installation
 Configuration
 =============
 
-  export AMAZON_ACCESS_KEY_ID='abcdefghijklmnop'
-  export AMAZON_SECRET_ACCESS_KEY='1234567891012345'
+Log into Amazon Web Services, and lookup your access key ID and secret.
+https://aws-portal.amazon.com/gp/aws/developer/account/index.html?action=access-key
+
+Here, I'll pretend that your access key is "ABCDEFGHIJKLMNOPQ" and your secret
+is "12345678901234567890abcde". Alright then...
+
+    % echo 'AWSAccessKeyId=ABCDEFGHIJKLMNOPQ' > ~/.aws-credentials
+    % echo 'AWSSecretKey=12345678901234567890abcde' >> ~/.aws-credentials
+    % export AWS_CREDENTIAL_FILE=~/.aws-credentials
+
+  - OR -
+
+    % export AMAZON_ACCESS_KEY_ID='ABCDEFGHIJKLMNOPQ'
+    % export AMAZON_SECRET_ACCESS_KEY='12345678901234567890abcde'
+
+  - OR -
+
+    % export AWS_ACCESS_KEY='ABCDEFGHIJKLMNOPQ'
+    % export AWS_SECRET_KEY='12345678901234567890abcde'
+
 
 Usage
 =====
-From the command line:
 
     % s3stream fetch mybucket myfile.dat > /tmp/myfile.dat
     % s3stream store mybucket myfile.dat < /tmp/myfile.dat
 
 And if you want to get fancy:
 
-    % cat <(s3stream fetch mybucket log0.log.gz | gunzip) <(s3stream fetch mybucket log1.log.gz | gunzip) | gzip > s3stream store mybucket combined.log.gz
-
-
-Development
-===========
-    gem install bundler
-    bundle install
-    guard
+    % cat <(s3stream fetch mybucket log0.log.gz | gunzip) <(s3stream fetch mybucket log1.log.gz | gunzip) | lzop -9 | s3stream store mybucket combined.log.lzo

data/Rakefile

@@ -1,24 +1,7 @@
 require "bundler/gem_tasks"
-require 'rspec/core/rake_task'
-
-desc "Run all specs"
-RSpec::Core::RakeTask.new(:spec) do |t|
-  t.rspec_opts = [
-    '-c',
-    '--format documentation',
-    '-r ./spec/spec_helper.rb'
-  ]
-  t.pattern = 'spec/**/*_spec.rb'
-end
-
-desc "open coverage report"
-task :coverage do
-  system 'rake spec'
-  system 'open coverage/index.html'
-end
 
 desc "Open development console"
 task :console do
   puts "Loading development console..."
-  system "irb -I #{File.join('.', 'lib')} -r #{File.join('.', 'lib', 's3stream')}"
+  system "pry -I #{File.join('.', 'lib')} -r #{File.join('.', 'lib', 's3stream')}"
 end

data/lib/s3stream/buffer_growth.rb

@@ -0,0 +1,42 @@
+module S3Stream
+  class BufferGrowth
+  
+    EPSILON = 1.0e-16
+
+    def initialize(initial_buffer_size, max_file_size, max_chunks)
+      @max_chunks = max_chunks
+      @max_file_size = max_file_size
+      @initial_buffer_size = initial_buffer_size
+    end
+
+    def solve
+      lower_bound = 1.0
+      upper_bound = 1.0
+      upper_bound *= 2.0 until too_big?(upper_bound)
+      begin
+        puts "bounds: #{[lower_bound, upper_bound]}"
+        return lower_bound if (upper_bound - lower_bound) < EPSILON
+        guess = (lower_bound + upper_bound) / 2.0
+        return lower_bound if [lower_bound,upper_bound].include?(guess)
+        if too_big?(guess)
+          upper_bound = guess
+        else
+          lower_bound = guess
+        end
+      end while true
+    end
+
+    private
+
+    def too_big?(growth_factor)
+      total = 0
+      buffer_size = @initial_buffer_size
+      (1..@max_chunks).each do
+        total += buffer_size
+        buffer_size = (buffer_size * growth_factor).to_i
+        return true if total > @max_file_size
+      end
+      false
+    end
+  end
+end

data/lib/s3stream/constants.rb

@@ -0,0 +1,16 @@
+module S3Stream
+  KB = 1024
+  MB = 1024 * KB
+  GB = 1024 * MB
+  TB = 1024 * GB
+
+  MAX_CHUNKS = 10_000
+  MAX_FILE_SIZE = 5 * TB
+  INITIAL_BUFFER_SIZE = 5 * MB
+
+  BUFFER_GROWTH_FACTOR = 1.0006533241143831
+  #BUFFER_GROWTH_FACTOR = begin
+  #  require 's3stream/buffer_growth'
+  #  BufferGrowth.new(INITIAL_BUFFER_SIZE, MAX_FILE_SIZE, MAX_CHUNKS).solve
+  #end
+end

data/lib/s3stream/credentials.rb

@@ -0,0 +1,34 @@
+module S3Stream
+
+  private
+
+  def self.credentials
+    access_key_id, secret_access_key = nil
+
+    # Try AWS tools' standard way
+    if File.file?(ENV['AWS_CREDENTIAL_FILE'])
+      File.open(ENV['AWS_CREDENTIAL_FILE']) do |f|
+        f.lines.each do |line|
+          access_key_id = $1 if line =~ /^AWSAccessKeyId=(.*)$/
+          secret_access_key = $1 if line =~ /^AWSSecretKey=(.*)$/
+        end
+      end
+    end
+
+    # Try aws-s3 gem's way
+    access_key_id     ||= ENV['AMAZON_ACCESS_KEY_ID']
+    secret_access_key ||= ENV['AMAZON_SECRET_ACCESS_KEY']
+
+    # Try s3cmd gem's way
+    access_key_id     ||= ENV['AWS_ACCESS_KEY']
+    secret_access_key ||= ENV['AWS_SECRET_KEY']
+
+    if access_key_id.nil? || secret_access_key.nil?
+      nil
+    else
+      {:access_key_id => access_key_id, :secret_access_key => secret_access_key}
+    end
+  end
+
+  CREDENTIALS = S3Stream.credentials
+end

data/lib/s3stream/main.rb

@@ -0,0 +1,51 @@
+module S3Stream
+  class Main < Thor
+    desc "fetch bucket filename", "download/stream the file from S3 to stdout"
+    def fetch(bucket_name, filename)
+      require "aws/s3"
+      AWS::S3::Base.establish_connection!(S3Stream::CREDENTIALS)
+      AWS::S3::S3Object.stream(filename, bucket_name) do |chunk|
+        $stdout.write chunk
+      end
+    end
+    
+    desc "store bucket filename", "upload/stream the file from stdin to S3"
+    def store(bucket_name, filename)
+      require "aws-sdk"
+      $stdout.sync = true
+      s3 = AWS::S3.new(S3Stream::CREDENTIALS)
+      bucket = s3.buckets[bucket_name]
+      object = bucket.objects[filename]
+      buffer = ""
+      total = 0
+      buffer_size = INITIAL_BUFFER_SIZE
+      puts "Uploading, please be patient."
+      object.multipart_upload do |upload|
+        (1..MAX_CHUNKS).each do |chunk|
+          if $stdin.eof?
+            puts "End of input."
+            break
+          end
+          if chunk < MAX_CHUNKS
+            print "Buffering input (up to #{buffer_size} bytes) ... "
+            $stdin.read(buffer_size, buffer)
+            puts "done."
+            print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
+            upload.add_part(buffer)
+            puts "done."
+            buffer_size = (buffer_size * BUFFER_GROWTH_FACTOR).to_i
+          else
+            print "Last part (#{chunk})! Buffering input (unlimited) ..."
+            buffer = $stdin.read
+            puts "done."
+            print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
+            upload.add_part(buffer)
+            puts "done."
+          end
+          total += buffer.size
+        end
+      end
+      puts "Done uploading to s3://#{bucket_name}/#{filename} (#{total} bytes)"
+    end
+  end
+end

data/lib/s3stream/version.rb

@@ -1,3 +1,3 @@
 module S3Stream
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/s3stream.rb

@@ -1,32 +1,11 @@
-require "s3stream/version"
 require "thor"
-require "aws/s3"
-
-module S3Stream
-  class Main < Thor
-    desc "fetch bucket filename", "download/stream the file from S3 to stdout"
-    def fetch(bucket, filename)
-      AWS::S3::S3Object.stream(filename, bucket) do |chunk|
-        $stdout.write chunk
-      end
-    end
-    
-    desc "store bucket filename", "upload/stream the file from stdin to S3"
-    def store(bucket, filename)
-      AWS::S3::S3Object.store(filename, ARGF, bucket)
-    end
-  end
-
-  def self.usage
-    puts "Please set AMAZON_ACCESS_KEY_ID and AMAZON_SECRET_ACCESS_KEY environmental variables."
-    exit(1)
-  end
+require "s3stream/version"
+require "s3stream/constants"
+require "s3stream/main"
+require "s3stream/credentials"
+
+if S3Stream::CREDENTIALS.nil?
+  puts "Environmental variables must be set."
+  puts "See https://github.com/kindkid/s3stream/blob/v#{S3Stream::VERSION}/README"
+  exit(1)
 end
-
-AMAZON_ACCESS_KEY_ID     = ENV['AMAZON_ACCESS_KEY_ID'] || S3Stream.usage
-AMAZON_SECRET_ACCESS_KEY = ENV['AMAZON_SECRET_ACCESS_KEY'] || S3Stream.usage
-
-AWS::S3::Base.establish_connection!(
-  :access_key_id     => AMAZON_ACCESS_KEY_ID,
-  :secret_access_key => AMAZON_SECRET_ACCESS_KEY
-)

data/s3stream.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   s.version     = S3Stream::VERSION
   s.authors     = ["Chris Johnson"]
   s.email       = ["chris@kindkid.com"]
-  s.homepage    = ""
+  s.homepage    = "https://github.com/kindkid/s3stream"
   s.summary     = "Stream files on S3 to stdout or from stdin"
   s.description = s.summary
 
@@ -18,12 +18,10 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency "aws-s3", "~> 0.6.2"
   s.add_runtime_dependency "thor", "~> 0.14.6"
-  s.add_development_dependency "rspec", "~> 2.6"
-  s.add_development_dependency "simplecov", "~> 0.4"
-  s.add_development_dependency("rb-fsevent", "~> 0.4") if RUBY_PLATFORM =~ /darwin/i
-  s.add_development_dependency "guard", "~> 0.5"
-  s.add_development_dependency "guard-bundler", "~> 0.1"
-  s.add_development_dependency "guard-rspec", "~> 0.4"
+  s.add_development_dependency "aws-sdk", "~> 1.3.5" #actually a runtime dependency
+  s.add_development_dependency "aws-s3", "~> 0.6.2"  #actually a runtime dependency
+  s.add_development_dependency "pry"
+  s.add_development_dependency "pry-doc"
+  s.add_development_dependency "pry-nav"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3stream
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-23 00:00:00.000000000 Z
+date: 2012-02-27 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: aws-s3
-  requirement: &70254260784700 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.6.2
-  type: :runtime
-  prerelease: false
-  version_requirements: *70254260784700
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &70254260782160 !ruby/object:Gem::Requirement
+  requirement: &70321780913920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,73 +21,62 @@ dependencies:
         version: 0.14.6
   type: :runtime
   prerelease: false
-  version_requirements: *70254260782160
+  version_requirements: *70321780913920
 - !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: &70254260781400 !ruby/object:Gem::Requirement
+  name: aws-sdk
+  requirement: &70321780912980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.6'
+        version: 1.3.5
   type: :development
   prerelease: false
-  version_requirements: *70254260781400
+  version_requirements: *70321780912980
 - !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: &70254260780500 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.4'
-  type: :development
-  prerelease: false
-  version_requirements: *70254260780500
-- !ruby/object:Gem::Dependency
-  name: rb-fsevent
-  requirement: &70254260779540 !ruby/object:Gem::Requirement
+  name: aws-s3
+  requirement: &70321780923920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: 0.6.2
   type: :development
   prerelease: false
-  version_requirements: *70254260779540
+  version_requirements: *70321780923920
 - !ruby/object:Gem::Dependency
-  name: guard
-  requirement: &70254260778500 !ruby/object:Gem::Requirement
+  name: pry
+  requirement: &70321780923240 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0.5'
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70254260778500
+  version_requirements: *70321780923240
 - !ruby/object:Gem::Dependency
-  name: guard-bundler
-  requirement: &70254260777160 !ruby/object:Gem::Requirement
+  name: pry-doc
+  requirement: &70321780922600 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70254260777160
+  version_requirements: *70321780922600
 - !ruby/object:Gem::Dependency
-  name: guard-rspec
-  requirement: &70254260775520 !ruby/object:Gem::Requirement
+  name: pry-nav
+  requirement: &70321780922040 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70254260775520
+  version_requirements: *70321780922040
 description: Stream files on S3 to stdout or from stdin
 email:
 - chris@kindkid.com
@@ -110,16 +88,17 @@ files:
 - .gitignore
 - .rvmrc
 - Gemfile
-- Guardfile
 - README
 - Rakefile
 - bin/s3stream
 - lib/s3stream.rb
+- lib/s3stream/buffer_growth.rb
+- lib/s3stream/constants.rb
+- lib/s3stream/credentials.rb
+- lib/s3stream/main.rb
 - lib/s3stream/version.rb
 - s3stream.gemspec
-- spec/lib/s3stream_spec.rb
-- spec/spec_helper.rb
-homepage: ''
+homepage: https://github.com/kindkid/s3stream
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -143,6 +122,5 @@ rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Stream files on S3 to stdout or from stdin
-test_files:
-- spec/lib/s3stream_spec.rb
-- spec/spec_helper.rb
+test_files: []
+has_rdoc: 

data/Guardfile

@@ -1,14 +0,0 @@
-# A sample Guardfile
-# More info at https://github.com/guard/guard#readme
-
-guard 'bundler' do
-  watch('Gemfile')
-  watch(/^.+\.gemspec/)
-end
-
-guard 'rspec', :cli => '-c --format documentation -r ./spec/spec_helper.rb',
-               :version => 2 do
-  watch(%r{^spec/.+_spec\.rb})
-  watch(%r{^lib/(.+)\.rb})        { |m| "spec/lib/#{m[1]}_spec.rb" }
-  watch('spec/spec_helper.rb')    { "spec" }
-end

data/spec/lib/s3stream_spec.rb

@@ -1,3 +0,0 @@
-describe S3Stream do
-  it "should have specs"
-end

data/spec/spec_helper.rb

@@ -1,7 +0,0 @@
-require 'simplecov'
-SimpleCov.start
-
-require 'rubygems'
-require 'bundler'
-
-Bundler.require(:default, :test, :development)