s3stream 0.0.6 → 0.0.7

data/README

@@ -33,6 +33,10 @@ Usage
     % s3stream fetch mybucket myfile.dat > /tmp/myfile.dat
     % s3stream store mybucket myfile.dat < /tmp/myfile.dat
 
+If you want to try uploading a bunch of data, but don't have any handy:
+
+    % base64 /dev/urandom | head -c $((10 * 1024 * 1024)) | s3stream store mybucket 10mb.txt
+
 And if you want to get fancy:
 
     % cat \
@@ -40,3 +44,11 @@ And if you want to get fancy:
       <(s3stream fetch mybucket log1.log.gz | gunzip) \
       | lzop -9 \
       | s3stream store mybucket combined.log.lzo
+
+
+If you want to stream an upload in ruby:
+
+    S3Stream::Upload.to(:s3object => s3object) do |stream|
+      stream.write("abc")
+      stream.write("123")
+    end

data/bin/s3stream

@@ -1,4 +1,11 @@
 #!/usr/bin/env ruby
 
 require "s3stream"
+
+if S3Stream::CREDENTIALS.nil?
+  puts "Environmental variables must be set."
+  puts "See https://github.com/kindkid/s3stream/blob/v#{S3Stream::VERSION}/README"
+  exit(1)
+end
+
 S3Stream::Main.start

data/lib/s3stream/main.rb

@@ -18,36 +18,13 @@ module S3Stream
     def store(bucket_name, filename)
       $stdout.sync = true
       object = s3object(bucket_name, filename)
-      buffer = ""
-      total = 0
-      buffer_size = INITIAL_BUFFER_SIZE
-      puts "Uploading, please be patient."
-      object.multipart_upload do |upload|
-        (1..MAX_CHUNKS).each do |chunk|
-          if $stdin.eof?
-            puts "End of input."
-            break
-          end
-          if chunk < MAX_CHUNKS
-            print "Buffering input (up to #{buffer_size} bytes) ... "
-            $stdin.read(buffer_size, buffer)
-            puts "done."
-            print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
-            upload.add_part(buffer)
-            puts "done."
-            buffer_size = (buffer_size * BUFFER_GROWTH_FACTOR).to_i
-          else
-            print "Last part (#{chunk})! Buffering input (unlimited) ..."
-            buffer = $stdin.read
-            puts "done."
-            print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
-            upload.add_part(buffer)
-            puts "done."
-          end
-          total += buffer.size
+      S3Stream::Upload.to(:s3object => object, :log_to => $stdout) do |out|
+        buffer = ""
+        until $stdin.eof?
+          $stdin.readpartial(4096, buffer)
+          out.write(buffer)
         end
       end
-      puts "Done uploading to s3://#{bucket_name}/#{filename} (#{total} bytes)"
     end
 
     private

data/lib/s3stream/upload.rb

@@ -0,0 +1,77 @@
+module S3Stream
+  class Upload
+
+    # S3Stream::Upload.to(:s3object => s3object, :log_to => $stdout) do |out|
+    #   out.write("abc")
+    #   out.write("123")
+    # end
+    def self.to(args={})
+      stream = new(args)
+      begin
+        yield stream
+        stream.close
+      rescue => e
+        stream.cancel
+        raise
+      end
+    end
+
+    attr_reader :size
+
+    def initialize(args={})
+      @s3object = args[:s3object] || raise("Missing :s3object")
+      @log = args[:log_to] # optional
+      reset
+      @upload = @s3object.multipart_upload
+    end
+
+    def write(data)
+      @buffer << data
+      flush if @chunk < S3Stream::MAX_CHUNKS && @buffer.size >= @buffer_size
+      @size += data.size
+    end
+
+    def close
+      flush if @buffer.size > 0
+      unless @upload.close.nil?
+        log "Done uploading #{size} bytes to #{location}."
+      end
+    end
+
+    def cancel
+      @upload.abort unless @upload.nil?
+      reset
+      log "Canceled upload to #{location}."
+    end
+
+    private
+
+    def location
+      "s3://#{@s3object.bucket.name}/#{@s3object.key}"
+    end
+
+    def reset
+      @buffer = ""
+      @buffer_size = S3Stream::INITIAL_BUFFER_SIZE
+      @size = 0
+      @chunk = 1
+      @upload = nil
+    end
+
+    def flush
+      log "Uploading part #{@chunk} (#{@buffer.size} bytes)."
+      @upload.add_part(@buffer)
+      @buffer_size = (@buffer_size * S3Stream::BUFFER_GROWTH_FACTOR).to_i
+      @buffer.clear
+      @chunk += 1
+      nil
+    end
+
+    def log(msg)
+      unless @log.nil?
+        @log.puts(msg)
+        @log.flush
+      end
+    end
+  end
+end

data/lib/s3stream/version.rb

@@ -1,3 +1,3 @@
 module S3Stream
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/lib/s3stream.rb

@@ -2,11 +2,6 @@ require "thor"
 require "aws-sdk"
 require "s3stream/version"
 require "s3stream/constants"
+require "s3stream/upload"
 require "s3stream/main"
 require "s3stream/credentials"
-
-if S3Stream::CREDENTIALS.nil?
-  puts "Environmental variables must be set."
-  puts "See https://github.com/kindkid/s3stream/blob/v#{S3Stream::VERSION}/README"
-  exit(1)
-end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3stream
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-01 00:00:00.000000000 Z
+date: 2012-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &70192394775500 !ruby/object:Gem::Requirement
+  requirement: &70361652001540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.14.6
   type: :runtime
   prerelease: false
-  version_requirements: *70192394775500
+  version_requirements: *70361652001540
 - !ruby/object:Gem::Dependency
   name: aws-sdk
-  requirement: &70192394790820 !ruby/object:Gem::Requirement
+  requirement: &70361651998340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 1.3.5
   type: :runtime
   prerelease: false
-  version_requirements: *70192394790820
+  version_requirements: *70361651998340
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &70192394788600 !ruby/object:Gem::Requirement
+  requirement: &70361652012160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70192394788600
+  version_requirements: *70361652012160
 - !ruby/object:Gem::Dependency
   name: pry-doc
-  requirement: &70192394787200 !ruby/object:Gem::Requirement
+  requirement: &70361652011060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70192394787200
+  version_requirements: *70361652011060
 - !ruby/object:Gem::Dependency
   name: pry-nav
-  requirement: &70192394786080 !ruby/object:Gem::Requirement
+  requirement: &70361652010500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70192394786080
+  version_requirements: *70361652010500
 description: Stream files on S3 to stdout or from stdin
 email:
 - chris@kindkid.com
@@ -85,6 +85,7 @@ files:
 - lib/s3stream/constants.rb
 - lib/s3stream/credentials.rb
 - lib/s3stream/main.rb
+- lib/s3stream/upload.rb
 - lib/s3stream/version.rb
 - s3stream.gemspec
 homepage: https://github.com/kindkid/s3stream