faster_s3 1.0.0 → 1.1.0

data/faster_s3.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["patrick@redbubble.com"]
   spec.description   = %q{Download files from s3 in parallel}
   spec.summary       = %q{Faster s3 downloads}
-  spec.homepage      = ""
+  spec.homepage      = "https://github.com/stiak/faster_s3"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files`.split($/)

data/lib/faster_s3/download.rb

@@ -0,0 +1,68 @@
+require 'aws/s3'
+require 'parallel'
+
+module FasterS3
+  class Download
+    DEFAULT_PARTS = 8
+    CHUNK_SIZE = 1024*1024
+
+    attr_reader :target_path, :config, :parts
+
+    def initialize(target_path, config = {})
+      @target_path = target_path
+      @config = config
+      @parts = config[:parallel] || DEFAULT_PARTS
+    end
+
+    def download
+      reconstitute(download_parts)
+    end
+
+    def download_parts
+      part_objects = create_parts
+      Parallel.map(part_objects, in_threads: part_objects.length) do |file_part|
+        file_part.download(s3_object)
+        file_part.part_path
+      end
+    end
+
+    def create_parts
+      length = s3_object.content_length
+      raise "File is too small to download in parallel" if length < parts
+
+      # Split into parts
+      part_length = length / parts
+      remainder = length % parts
+
+      part_objects = []
+      parts.times do |index|
+        # Last part includes any remaining bytes
+        extra_bytes = (index == (parts - 1)) ? remainder : 0
+        part_objects << Part.new(target_path, index, part_length, extra_bytes)
+      end
+
+      part_objects
+    end
+
+    def reconstitute(file_parts)
+      File.open(target_path, 'wb') do |file|
+        file_parts.each do |part_path|
+          File.open(part_path, 'rb') do |part_file|
+            file.write(part_file.read(CHUNK_SIZE)) until part_file.eof?
+          end
+          File.delete(part_path)
+        end
+      end
+    end
+
+    private
+
+    def s3_object
+      @s3_object ||= begin
+        s3 = AWS::S3.new({:access_key_id => config[:access_key_id], :secret_access_key => config[:secret_access_key]})
+        bucket = s3.buckets[config[:bucket_name]]
+        bucket.objects[config[:path]]
+      end
+    end
+  end
+end

data/lib/faster_s3/part.rb

@@ -0,0 +1,33 @@
+class FasterS3::Part
+  attr_accessor :file_path, :index, :part_length, :extra_bytes
+
+  def initialize(file_path, index, part_length, extra_bytes)
+    self.file_path = file_path
+    self.index = index
+    self.part_length = part_length
+    self.extra_bytes = extra_bytes
+  end
+
+  def part_path
+    "#{file_path}.part.#{index}"
+  end
+
+  def byte_range
+    start = index * part_length
+    end_pos = start + part_length + extra_bytes
+
+    (start + existing_size)...end_pos
+  end
+
+  def existing_size
+    File.exists?(part_path) ? File.size(part_path) : 0
+  end
+
+  def download(s3_object)
+    File.open(part_path, 'ab') do |file|
+      s3_object.read(range: byte_range) do |chunk|
+        file.write(chunk)
+      end
+    end
+  end
+end

data/lib/faster_s3/version.rb

@@ -1,3 +1,3 @@
 module FasterS3
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/lib/faster_s3.rb

@@ -1,6 +1,6 @@
 require "faster_s3/version"
-require 'aws/s3'
-require 'parallel'
+require "faster_s3/part"
+require "faster_s3/download"
 
 module FasterS3
   # @param [String] destination File will be written to this destination
@@ -17,69 +17,7 @@ module FasterS3
   # @option options [String] :parallel Optional - number of parallel parts to download.  Default is 8
   #
   def self.download(destination, options)
-    Download.new(options).download_to(destination)
-  end
-
-  class Download
-    DEFAULT_PARTS = 8
-    CHUNK_SIZE = 1024*1024
-
-    attr_reader :config, :parts
-
-    def initialize(config = {})
-      @config = config
-      @parts = config[:parallel] || DEFAULT_PARTS
-    end
-
-    def download_to(path)
-      s3_object = remote_object
-      length = s3_object.content_length
-      raise "File is too small to download in parallel" if length < parts
-
-      # Split into parts
-      part_length = length / parts
-      remainder = length % parts
-
-      part_array = Array(1..parts)
-      file_parts = Parallel.map(part_array, in_threads: parts) do |part|
-        start = (part - 1) * part_length
-        end_pos = start + part_length
-
-        if part == part_array.last
-          end_pos += remainder
-          byte_range = start..end_pos
-        else
-          byte_range = start...end_pos
-        end
-
-        part_path = "#{path}.part.#{part}"
-        File.open("#{path}.part.#{part}", 'wb') do |file|
-          s3_object.read(range: byte_range) do |chunk|
-            file.write(chunk)
-          end
-        end
-        part_path
-      end
-
-      reconstitute_parts(path, file_parts)
-    end
-
-    def reconstitute_parts(path, file_parts)
-      File.open(path, 'wb') do |file|
-        file_parts.each do |part_path|
-          File.open(part_path, 'rb') do |part_file|
-            file.write(part_file.read(CHUNK_SIZE)) until part_file.eof?
-          end
-          File.delete(part_path)
-        end
-      end
-    end
-
-    def remote_object
-      s3 = AWS::S3.new({:access_key_id => config[:access_key_id], :secret_access_key => config[:secret_access_key]})
-      bucket = s3.buckets[config[:bucket_name]]
-      bucket.objects[config[:path]]
-    end
+    Download.new(destination, options).download
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: faster_s3
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70297353186560 !ruby/object:Gem::Requirement
+  requirement: &70342546745840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '1.3'
   type: :development
   prerelease: false
-  version_requirements: *70297353186560
+  version_requirements: *70342546745840
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70297353185680 !ruby/object:Gem::Requirement
+  requirement: &70342546745360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70297353185680
+  version_requirements: *70342546745360
 - !ruby/object:Gem::Dependency
   name: aws-sdk
-  requirement: &70297353184260 !ruby/object:Gem::Requirement
+  requirement: &70342546744860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70297353184260
+  version_requirements: *70342546744860
 - !ruby/object:Gem::Dependency
   name: parallel
-  requirement: &70297353183640 !ruby/object:Gem::Requirement
+  requirement: &70342546744400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70297353183640
+  version_requirements: *70342546744400
 description: Download files from s3 in parallel
 email:
 - patrick@redbubble.com
@@ -71,8 +71,10 @@ files:
 - Rakefile
 - faster_s3.gemspec
 - lib/faster_s3.rb
+- lib/faster_s3/download.rb
+- lib/faster_s3/part.rb
 - lib/faster_s3/version.rb
-homepage: ''
+homepage: https://github.com/stiak/faster_s3
 licenses:
 - MIT
 post_install_message: