aws-sdk-s3 1.0.0.rc3 → 1.0.0.rc4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 47082bcd6514ee19071b0e65b3614849bc841028
-  data.tar.gz: d0a6a1b92982a0cfb6a1df90d2abf54d9870a947
+  metadata.gz: 5fbb74b560c218d4dda7e2f060f0b87dd468e095
+  data.tar.gz: 6e888b327c450752ccdde576d35a9567f854cd44
 SHA512:
-  metadata.gz: 321f5f82ee5ec0c7537a8578e6f26c1a22c17722c3debd411ca3f7f34ed66a8792147960c8e0499285c2b48b290f6f12d487a1aa1d2ea5e56b57b5882caaa2b0
-  data.tar.gz: c3fbf872aa1413afcfc62116b0aed6c6cfaa703ea0d2130dcb50896b905a84731a6a580223a79830d42d1a8452319aea8dc47ff010814ba81f332041cbdca599
+  metadata.gz: e2ee1313f3356b25e7ea6527b8c1dbf0b6e16db9e19181728852fa7738887af59a313b09c637a621923c29ce190291dea6290ea57af43e0bde868f95b7e64d8b
+  data.tar.gz: bf7c18e08d97a1f64ca0d432c0fa6ca55d18d198bd8740c722dbd7320e95695725ad73b6b8a7640878f1a572eedf640afcec625d9a956bbc8d689c15289c803d

data/lib/aws-sdk-s3.rb

@@ -61,6 +61,6 @@ require_relative 'aws-sdk-s3/customizations'
 # @service
 module Aws::S3
 
-  GEM_VERSION = '1.0.0.rc2'
+  GEM_VERSION = '1.0.0.rc4'
 
 end

data/lib/aws-sdk-s3/client.rb

@@ -18,6 +18,7 @@ require 'aws-sdk-core/plugins/regional_endpoint.rb'
 require 'aws-sdk-core/plugins/response_paging.rb'
 require 'aws-sdk-core/plugins/stub_responses.rb'
 require 'aws-sdk-core/plugins/idempotency_token.rb'
+require 'aws-sdk-core/plugins/jsonvalue_converter.rb'
 require 'aws-sdk-core/plugins/protocols/rest_xml.rb'
 require 'aws-sdk-s3/plugins/accelerate.rb'
 require 'aws-sdk-s3/plugins/dualstack.rb'
@@ -57,6 +58,7 @@ module Aws::S3
     add_plugin(Aws::Plugins::ResponsePaging)
     add_plugin(Aws::Plugins::StubResponses)
     add_plugin(Aws::Plugins::IdempotencyToken)
+    add_plugin(Aws::Plugins::JsonvalueConverter)
     add_plugin(Aws::Plugins::Protocols::RestXml)
     add_plugin(Aws::S3::Plugins::Accelerate)
     add_plugin(Aws::S3::Plugins::Dualstack)
@@ -4361,7 +4363,7 @@ module Aws::S3
         params: params,
         config: config)
       context[:gem_name] = 'aws-sdk-s3'
-      context[:gem_version] = '1.0.0.rc2'
+      context[:gem_version] = '1.0.0.rc4'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/aws-sdk-s3/customizations.rb

@@ -3,6 +3,7 @@ require 'aws-sdk-s3/bucket_region_cache'
 require 'aws-sdk-s3/encryption'
 require 'aws-sdk-s3/file_part'
 require 'aws-sdk-s3/file_uploader'
+require 'aws-sdk-s3/file_downloader'
 require 'aws-sdk-s3/legacy_signer'
 require 'aws-sdk-s3/multipart_file_uploader'
 require 'aws-sdk-s3/multipart_upload_error'

data/lib/aws-sdk-s3/customizations/object.rb

@@ -253,6 +253,38 @@ module Aws
         true
       end
 
+      # Downloads a file in S3 to a path on disk.
+      #
+      #     # small files (< 5MB) are downloaded in a single API call
+      #     obj.download_file('/path/to/file')
+      #
+      # Files larger than 5MB are downloaded using multipart method
+      #
+      #     # large files are split into parts
+      #     # and the parts are downloaded in parallel
+      #     obj.download_file('/path/to/very_large_file')
+      #
+      # @param [String] destination Where to download the file to
+      #
+      # @option options [String] mode `auto`, `single_request`, `get_range`
+      #  `single_request` mode forces only 1 GET request is made in download,
+      #  `get_range` mode allows `chunk_size` parameter to configured in
+      #  customizing each range size in multipart_download,
+      #  By default, `auto` mode is enabled, which performs multipart_download
+      #
+      # @option options [String] chunk_size required in get_range mode
+      #
+      # @option options [String] thread_count Customize threads used in multipart
+      #   download, if not provided, 10 is default value
+      #
+      # @return [Boolean] Returns `true` when the file is downloaded
+      #   without any errors.
+      def download_file(destination, options = {})
+        downloader = FileDownloader.new(client: client)
+        downloader.download(
+          destination, options.merge(bucket: bucket_name, key: key))
+        true
+      end
     end
   end
 end

data/lib/aws-sdk-s3/customizations/object_summary.rb

@@ -60,6 +60,14 @@ module Aws
         object.upload_file(source, options)
       end
 
+      # @param (see Object#download_file)
+      # @options (see Object#download_file)
+      # @return (see Object#download_file)
+      # @see Object#download_file
+      def download_file(destination, options = {})
+        object.download_file(destination, options)
+      end
+
     end
   end
 end

data/lib/aws-sdk-s3/file_downloader.rb

@@ -0,0 +1,173 @@
+require 'pathname'
+require 'thread'
+require 'set'
+require 'tmpdir'
+
+module Aws
+  module S3
+    # @api private
+    class FileDownloader
+
+      MIN_CHUNK_SIZE = 5 * 1024 * 1024
+      MAX_PARTS = 10_000
+      THREAD_COUNT = 10
+
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      def download(destination, options = {})
+        @path = destination
+        @mode = options[:mode] || "auto"
+        @thread_count = options[:thread_count] || THREAD_COUNT
+        @chunk_size = options[:chunk_size]
+        @bucket = options[:bucket]
+        @key = options[:key]
+
+        case @mode
+        when "auto" then multipart_download
+        when "single_request" then single_request
+        when "get_range"
+          if @chunk_size
+            resp = @client.head_object(bucket: @bucket, key: @key)
+            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+          else
+            msg = "In :get_range mode, :chunk_size must be provided"
+            raise ArgumentError, msg
+          end
+        else
+          msg = "Invalid mode #{@mode} provided, "\
+            "mode should be :single_request, :get_range or :auto"
+          raise ArgumentError, msg
+        end
+      end
+
+      private
+
+      def multipart_download
+        resp = @client.head_object(bucket: @bucket, key: @key, part_number: 1)
+        count = resp.parts_count
+        if count.nil? || count <= 1
+          resp.content_length < MIN_CHUNK_SIZE ?
+            single_request :
+            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+        else
+          # partNumber is an option
+          resp = @client.head_object(bucket: @bucket, key: @key)
+          resp.content_length < MIN_CHUNK_SIZE ?
+            single_request :
+            compute_mode(resp.content_length, count)
+        end
+      end
+
+      def compute_mode(file_size, count)
+        chunk_size = compute_chunk(file_size)
+        part_size = (file_size.to_f / count.to_f).ceil
+        if chunk_size < part_size
+          multithreaded_get_by_ranges(construct_chunks(file_size))
+        else
+          multithreaded_get_by_parts(count)
+        end
+      end
+
+      def construct_chunks(file_size)
+        offset = 0
+        default_chunk_size = compute_chunk(file_size)
+        chunks = []
+        while offset <= file_size
+          progress = offset + default_chunk_size
+          chunks << "bytes=#{offset}-#{progress < file_size ? progress : file_size}"
+          offset = progress + 1
+        end
+        chunks
+      end
+
+      def compute_chunk(file_size)
+        if @chunk_size && @chunk_size > file_size
+          raise ArgumentError, ":chunk_size shouldn't exceed total file size."
+        else
+          default_chunk_size = @chunk_size || [(file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE].max.to_i
+        end
+      end
+
+      def sort_files(files)
+        # sort file by start range count or part number
+        files.sort do |a, b|
+          a[/([^\=]+)$/].split('-')[0].to_i <=> b[/([^\=]+)$/].split('-')[0].to_i
+        end
+      end
+
+      def concatenate_parts(fileparts)
+        File.open(@path, 'wb')do |output_path|
+          sort_files(fileparts).each {|part| IO.copy_stream(part, output_path)}
+        end
+      end
+
+      def clean_up_parts(parts)
+        parts.each do |filename|
+          File.unlink(filename) if File.exists?(filename)
+        end
+      end
+
+      def file_batches(chunks, mode)
+        batches = []
+        dir = Dir.tmpdir
+        chunks = (1..chunks) if mode.eql? 'part_number'
+        chunks.each_slice(@thread_count) do |slice|
+          batches << map_files(slice, dir, mode)
+        end
+        batches
+      end
+
+      def map_files(slice, dir, mode)
+        case mode
+        when 'range'
+          slice.inject({}) {|h, chunk| h[chunk] = File.join(dir, chunk); h}
+        when 'part_number'
+          slice.inject({}) {|h, part| h[part] = File.join(dir, "part_number=#{part}"); h}
+        end
+      end
+
+      def multithreaded_get_by_ranges(chunks)
+        thread_batches(chunks, 'range')
+      end
+
+      def multithreaded_get_by_parts(parts)
+        thread_batches(parts, 'part_number')
+      end
+
+      def thread_batches(chunks, param)
+        batches = file_batches(chunks, param)
+        parts = batches.flat_map(&:values)
+        begin
+          batches.each do |batch|
+            threads = []
+            batch.each do |chunk, file|
+              threads << Thread.new do
+                resp = @client.get_object(
+                  :bucket => @bucket,
+                  :key => @key,
+                  param.to_sym => chunk,
+                  :response_target => file
+                )
+              end
+            end
+            threads.each(&:join)
+          end
+          concatenate_parts(parts)
+        ensure
+          clean_up_parts(parts)
+        end
+      end
+
+      def single_request
+        @client.get_object(
+          bucket: @bucket, key: @key, response_target: @path
+        )
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/legacy_signer.rb

@@ -1,6 +1,7 @@
 require 'set'
 require 'time'
 require 'openssl'
+require 'cgi/util'
 require 'webrick/httputils'
 require 'aws-sdk-core/query'
 

data/lib/aws-sdk-s3/object_multipart_copier.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'cgi/util'
 
 module Aws
   module S3

data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb

@@ -1,4 +1,5 @@
 require 'uri'
+require 'cgi/util'
 
 module Aws
   module S3

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-s3
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc3
+  version: 1.0.0.rc4
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-09 00:00:00.000000000 Z
+date: 2017-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0.rc3
+        version: 1.0.0.rc4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.0.0.rc3
+        version: 1.0.0.rc4
 - !ruby/object:Gem::Dependency
   name: aws-sigv4
   requirement: !ruby/object:Gem::Requirement
@@ -96,6 +96,7 @@ files:
 - lib/aws-sdk-s3/encryption/materials.rb
 - lib/aws-sdk-s3/encryption/utils.rb
 - lib/aws-sdk-s3/errors.rb
+- lib/aws-sdk-s3/file_downloader.rb
 - lib/aws-sdk-s3/file_part.rb
 - lib/aws-sdk-s3/file_uploader.rb
 - lib/aws-sdk-s3/legacy_signer.rb