s3_streamer 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 158368a84fcdfe9ac40904f3a922b4da9b4ccae2
-  data.tar.gz: 4d659a39ebde78e65b0b878a0def086789a43299
+  metadata.gz: 08b8b13f35734bff5cbdcb9a4cf1ba0770b0671d
+  data.tar.gz: 570918eb21651ecd7a1bd3545395c1eccd1691db
 SHA512:
-  metadata.gz: 5bcec56c4f9cd00f6873c086fd8b12f5a6e59b511b927b20de747129ebab426c644b8002e23a39634213bb007ac0b28a2d370652be242696c40775738749b58a
-  data.tar.gz: ff0fb774181964ac67cefc507c32c9ba4213751d5267674cfdd53d5eb270b9aaca26a6b5fc790594a6720f61867b5a468f42f1333b468c12ebedc67dca779836
+  metadata.gz: 8a7cfe74390bd782a5f5b7af9e440b8260dc7a8c215d1e1a9fcb573ecc42ea466a1d4ff35976181b4f7e8b3bcbb11238f62754c811ad0fb74bb43e6a6e317083
+  data.tar.gz: 6bc5a341149f6d92ddce682fd5904f9a19260bdb901a166390a28d8d7f79995914d87c8af27f1c30726269750b1bd1361fbf595cbf7b7210eac3958f4e724d81

data/README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/thilonel/s3_streamer.svg?branch=master)](https://travis-ci.org/thilonel/s3_streamer)
+
 # S3 Streamer
 
 Stream a file from a remote url using the Ruby AWS SDK to an Amazon S3 bucket.

data/lib/s3_streamer/buffered_writer.rb

@@ -0,0 +1,35 @@
+module S3Streamer
+  class BufferedWriter
+    attr_reader :options, :buffer
+
+    def initialize(options, &block)
+      @options = options
+      @block = block
+      clear
+    end
+
+    def push(chunk)
+      @buffer << chunk
+      write if @buffer.size > min_part_size
+    end
+
+    def finish
+      write if @buffer.size > 0
+    end
+
+    private
+
+    def write
+      @block.call buffer
+      clear
+    end
+
+    def min_part_size
+      options[:min_part_size] || 1024
+    end
+
+    def clear
+      @buffer = ""
+    end
+  end
+end

data/lib/s3_streamer/client.rb

@@ -1,18 +1,20 @@
 require 'aws-sdk'
 
-require 's3_streamer/client/downstream'
-require 's3_streamer/client/upstream'
+require 's3_streamer/downstream'
+require 's3_streamer/upstream'
 
 module S3Streamer
   class Client
+    attr_reader :client
+
     def initialize(options)
       @client = Aws::S3::Client.new(options)
     end
 
-    def stream(source_uri, bucket, destination_file)
-      upstream = Upstream.new @client, bucket, destination_file
+    def stream(source_uri, bucket, destination_file, options={})
+      upstream = Upstream.new client, bucket, destination_file
 
-      Downstream.new(URI(source_uri)).each do |chunk|
+      Downstream.new(URI(source_uri), options).each do |chunk|
         upstream.upload chunk
       end
 

data/lib/s3_streamer/downstream.rb

@@ -0,0 +1,33 @@
+require 'net/http'
+require 's3_streamer/buffered_writer'
+
+module S3Streamer
+  class Downstream
+    DEFAULT_PART_SIZE = 10 * 1024 * 1024
+
+    def initialize(uri, options={})
+      @uri = uri
+      @options = options
+    end
+
+    def each
+      Net::HTTP.start(@uri.host) do |http|
+        http.request(Net::HTTP::Get.new @uri) do |response|
+          buffer = BufferedWriter.new(min_part_size: part_size) do |part|
+            yield part
+          end
+
+          response.read_body { |chunk| buffer.push chunk }
+
+          buffer.finish
+        end
+      end
+    end
+
+    private
+
+    def part_size
+      @options[:part_size] || DEFAULT_PART_SIZE
+    end
+  end
+end

data/lib/s3_streamer/upstream.rb

@@ -0,0 +1,37 @@
+module S3Streamer
+  class Upstream
+    def initialize(client, bucket, destination_file)
+      @client = client
+      @bucket = bucket
+      @destination_file = destination_file
+      @parts = []
+      @part_number = 1
+      @upload_id = create
+    end
+
+    def upload(chunk)
+      uploaded_part = @client.upload_part(body: chunk,
+                                          bucket: @bucket,
+                                          key: @destination_file,
+                                          part_number: @part_number,
+                                          upload_id: @upload_id)
+
+      @parts << {etag: uploaded_part.etag, part_number: @part_number}
+
+      @part_number += 1
+    end
+
+    def complete
+      @client.complete_multipart_upload(bucket: @bucket,
+                                        key: @destination_file,
+                                        upload_id: @upload_id,
+                                        multipart_upload: {parts: @parts})
+    end
+
+    private
+
+    def create
+      @client.create_multipart_upload(bucket: @bucket, key: @destination_file).upload_id
+    end
+  end
+end

data/lib/s3_streamer/version.rb

@@ -1,3 +1,3 @@
 module S3Streamer
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_streamer
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - thilonel
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-02 00:00:00.000000000 Z
+date: 2016-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -99,10 +99,10 @@ files:
 - bin/console
 - bin/setup
 - lib/s3_streamer.rb
+- lib/s3_streamer/buffered_writer.rb
 - lib/s3_streamer/client.rb
-- lib/s3_streamer/client/downstream.rb
-- lib/s3_streamer/client/downstream/chunk_buffer.rb
-- lib/s3_streamer/client/upstream.rb
+- lib/s3_streamer/downstream.rb
+- lib/s3_streamer/upstream.rb
 - lib/s3_streamer/version.rb
 - s3_streamer.gemspec
 homepage: https://github.com/thilonel/s3_streamer

data/lib/s3_streamer/client/downstream/chunk_buffer.rb

@@ -1,18 +0,0 @@
-class ChunkBuffer
-  def initialize(response, chunks_per_buffer = 10240)
-    @response = response
-    @chunks_per_buffer = chunks_per_buffer
-  end
-
-  def each
-    buffer = []
-    @response.read_body do |chunk|
-      buffer << chunk
-      if buffer.size >= @chunks_per_buffer
-        yield buffer.join
-        buffer = []
-      end
-    end
-    yield buffer.join if buffer.size > 0
-  end
-end

data/lib/s3_streamer/client/downstream.rb

@@ -1,18 +0,0 @@
-require 'net/http'
-require_relative 'downstream/chunk_buffer'
-
-class Downstream
-  def initialize(uri)
-    @uri = uri
-  end
-
-  def each
-    Net::HTTP.start(@uri.host) do |http|
-      http.request(Net::HTTP::Get.new @uri) do |response|
-        ChunkBuffer.new(response).each do |chunk|
-          yield chunk
-        end
-      end
-    end
-  end
-end

data/lib/s3_streamer/client/upstream.rb

@@ -1,35 +0,0 @@
-class Upstream
-  def initialize(client, bucket, destination_file)
-    @client = client
-    @bucket = bucket
-    @destination_file = destination_file
-    @parts = []
-    @part_number = 1
-    @upload_id = create
-  end
-
-  def upload(chunk)
-    uploaded_part = @client.upload_part(body: chunk,
-                                        bucket: @bucket,
-                                        key: @destination_file,
-                                        part_number: @part_number,
-                                        upload_id: @upload_id)
-
-    @parts << {etag: uploaded_part.etag, part_number: @part_number}
-
-    @part_number += 1
-  end
-
-  def complete
-    @client.complete_multipart_upload(bucket: @bucket,
-                                      key: @destination_file,
-                                      upload_id: @upload_id,
-                                      multipart_upload: {parts: @parts})
-  end
-
-  private
-
-  def create
-    @client.create_multipart_upload(bucket: @bucket, key: @destination_file).upload_id
-  end
-end