s3_streaming_multipart_uploader 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1ba359075a8dcf7ecfb596714f3b50c0effafae2
+  data.tar.gz: f6c2aa214d0953902df54534279baf59eb1a7513
+SHA512:
+  metadata.gz: 294887cafefb74b01caea504c76a04ed2d35576cf5ebd2f45528e55ccb57c1ed84d1a927b7790d70bff864c8964d4e924481fc31008e7444ba00da259d6b4470
+  data.tar.gz: c12892ecc00f6ee94f89e75bb9ef9d4a6337bfd448e325386ddc79d9b8fbc7f35a21bdf6b47dc62451bd12fce17a7da2076bba43f1f028e2070dc613c5360250

data/lib/s3_streaming_multipart_uploader.rb

@@ -0,0 +1,93 @@
+class S3StreamingMultipartUploader
+
+  MIN_PART_SIZE = 1024 * 1024 * 5
+
+  def initialize(s3_client, options = {})
+    @min_part_size = options[:min_part_size] || MIN_PART_SIZE
+    @s3_client = s3_client
+  end
+
+  def stream(bucket, key, enumerator)
+    @uploader = Uploader.new(@s3_client, bucket, key, enumerator, @min_part_size)
+    @uploader.stream
+  end
+
+  private 
+
+  class Uploader
+    def initialize(s3_client, bucket, key, enumerator, min_part_size)
+      @s3_client = s3_client
+      @bucket = bucket
+      @key = key
+      @enumerator = enumerator
+      @min_part_size = min_part_size
+    end
+
+    def stream
+      upload_id = start.upload_id
+      begin
+        parts = stream_parts(upload_id)
+        complete(upload_id, parts)
+      rescue Aws::Errors::ServiceError => e
+        p e
+        abort(upload_id)
+      end
+    end
+
+    private
+
+    def read
+      buffer = ""
+      begin
+        while (chunck = @enumerator.next) && buffer << chunck && buffer.size <= @min_part_size
+          p "READ #{chunck.size} #{buffer.size}"
+        end
+      rescue StopIteration
+      end
+      buffer
+    end
+
+    def options
+      {bucket: @bucket, key: @key}
+    end
+
+    def start
+      @s3_client.create_multipart_upload(options)
+    end
+
+    def stream_parts(upload_id)
+      i = 1
+      parts = []
+      while (chunck = read).size > 0
+        parts << @s3_client.upload_part(
+          options.merge(
+            body: chunck,
+            upload_id: upload_id,
+            part_number: i
+          )
+        )
+        i += 1
+      end
+      parts.each_with_index.map do |part, i|
+        {etag: part.etag, part_number: i + 1}
+      end
+    end
+
+    def complete(upload_id, request_parts)
+      @s3_client.complete_multipart_upload(
+        options.merge(
+          upload_id: upload_id,
+          multipart_upload: {parts: request_parts}
+        )
+      )
+    end
+
+    def abort(upload_id)
+      @s3_client.abort_multipart_upload(
+        options.merge(
+          upload_id: upload_id
+        )
+      )
+    end
+  end
+end

metadata

@@ -0,0 +1,44 @@
+--- !ruby/object:Gem::Specification
+name: s3_streaming_multipart_uploader
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Fonsan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-15 00:00:00.000000000 Z
+dependencies: []
+description: Stream data to S3 via enumerator
+email: fonsan@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/s3_streaming_multipart_uploader.rb
+homepage: http://github.com/Fonsan/s3_streaming_multipart_uploader
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Stream data to S3 via enumerator
+test_files: []