fog-aws 3.6.6 → 3.10.0

data/lib/fog/aws/models/storage/file.rb

@@ -4,6 +4,10 @@ module Fog
   module AWS
     class Storage
       class File < Fog::Model
+        MIN_MULTIPART_CHUNK_SIZE = 5242880
+        MAX_SINGLE_PUT_SIZE = 5368709120
+        MULTIPART_COPY_THRESHOLD = 15728640
+
         # @see AWS Object docs http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html
 
         identity  :key,             :aliases => 'Key'
@@ -27,14 +31,54 @@ module Fog
         attribute :kms_key_id,          :aliases => 'x-amz-server-side-encryption-aws-kms-key-id'
         attribute :tags,                :aliases => 'x-amz-tagging'
 
+        UploadPartData = Struct.new(:part_number, :upload_options, :etag)
+
+        class PartList
+          def initialize(parts = [])
+            @parts = parts
+            @mutex = Mutex.new
+          end
+
+          def push(part)
+            @mutex.synchronize { @parts.push(part) }
+          end
+
+          def shift
+            @mutex.synchronize { @parts.shift }
+          end
+
+          def clear!
+            @mutex.synchronize { @parts.clear }
+          end
+
+          def size
+            @mutex.synchronize { @parts.size }
+          end
+
+          def to_a
+            @mutex.synchronize { @parts.dup }
+          end
+        end
+
         # @note Chunk size to use for multipart uploads.
         #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
         attr_reader :multipart_chunk_size
         def multipart_chunk_size=(mp_chunk_size)
-          raise ArgumentError.new("minimum multipart_chunk_size is 5242880") if mp_chunk_size < 5242880
+          raise ArgumentError.new("minimum multipart_chunk_size is #{MIN_MULTIPART_CHUNK_SIZE}") if mp_chunk_size < MIN_MULTIPART_CHUNK_SIZE
           @multipart_chunk_size = mp_chunk_size
         end
 
+        # @note Number of threads used to copy files.
+        def concurrency=(concurrency)
+          raise ArgumentError.new('minimum concurrency is 1') if concurrency.to_i < 1
+
+          @concurrency = concurrency.to_i
+        end
+
+        def concurrency
+          @concurrency || 1
+        end
+
         def acl
           requires :directory, :key
           service.get_object_acl(directory.key, key).body['AccessControlList']
@@ -99,7 +143,18 @@ module Fog
         #
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+
+          # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
+          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE * 2 if !multipart_chunk_size && self.content_length.to_i > MAX_SINGLE_PUT_SIZE
+
+          if multipart_chunk_size && self.content_length.to_i >= multipart_chunk_size
+            upload_part_options = options.select { |key, _| ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym) }
+            upload_part_options = upload_part_options.merge({ 'x-amz-copy-source' => "#{directory.key}/#{key}" })
+            multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          else
+            service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+          end
+
           target_directory = service.directories.new(:key => target_directory_key)
           target_directory.files.head(target_file_key)
         end
@@ -214,7 +269,7 @@ module Fog
           options.merge!(encryption_headers)
 
           # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
-          self.multipart_chunk_size = 5242880 if !multipart_chunk_size && Fog::Storage.get_body_size(body) > 5368709120
+          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE if !multipart_chunk_size && Fog::Storage.get_body_size(body) > MAX_SINGLE_PUT_SIZE
 
           if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
             data = multipart_save(options)
@@ -294,6 +349,30 @@ module Fog
           service.complete_multipart_upload(directory.key, key, upload_id, part_tags)
         end
 
+        def multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          # Initiate the upload
+          res = service.initiate_multipart_upload(target_directory_key, target_file_key, options)
+          upload_id = res.body["UploadId"]
+
+          # Store ETags of upload parts
+          part_tags = []
+          pending = PartList.new(create_part_list(upload_part_options))
+          thread_count = self.concurrency
+          completed = PartList.new
+          errors = upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+
+          raise errors.first if errors.any?
+
+          part_tags = completed.to_a.sort_by { |part| part.part_number }.map(&:etag)
+        rescue => e
+          # Abort the upload & reraise
+          service.abort_multipart_upload(target_directory_key, target_file_key, upload_id) if upload_id
+          raise
+        else
+          # Complete the upload
+          service.complete_multipart_upload(target_directory_key, target_file_key, upload_id, part_tags)
+        end
+
         def encryption_headers
           if encryption && encryption_key
             encryption_customer_key_headers
@@ -318,6 +397,49 @@ module Fog
             'x-amz-server-side-encryption-customer-key-md5' => Base64.encode64(OpenSSL::Digest::MD5.digest(encryption_key.to_s)).chomp!
           }
         end
+
+        def create_part_list(upload_part_options)
+          current_pos = 0
+          count = 0
+          pending = []
+
+          while current_pos < self.content_length do
+            start_pos = current_pos
+            end_pos = [current_pos + self.multipart_chunk_size, self.content_length - 1].min
+            range = "bytes=#{start_pos}-#{end_pos}"
+            part_options = upload_part_options.dup
+            part_options['x-amz-copy-source-range'] = range
+            pending << UploadPartData.new(count + 1, part_options, nil)
+            count += 1
+            current_pos = end_pos + 1
+          end
+
+          pending
+        end
+
+        def upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+          threads = []
+
+          thread_count.times do
+            thread = Thread.new do
+              begin
+                while part = pending.shift
+                  part_upload = service.upload_part_copy(target_directory_key, target_file_key, upload_id, part.part_number, part.upload_options)
+                  part.etag = part_upload.body['ETag']
+                  completed.push(part)
+                end
+              rescue => error
+                pending.clear!
+                error
+              end
+            end
+
+            thread.abort_on_exception = true
+            threads << thread
+          end
+
+          threads.map(&:value).compact
+        end
       end
     end
   end

data/lib/fog/aws/models/storage/files.rb

@@ -17,6 +17,15 @@ module Fog
 
         model Fog::AWS::Storage::File
 
+        DASHED_HEADERS = %w(
+          Cache-Control
+          Content-Disposition
+          Content-Encoding
+          Content-Length
+          Content-MD5
+          Content-Type
+        ).freeze
+
         def all(options = {})
           requires :directory
           options = {
@@ -114,8 +123,29 @@ module Fog
         end
 
         def normalize_headers(data)
-          data.headers['Last-Modified'] = Time.parse(data.get_header('Last-Modified'))
-          data.headers['ETag'] = data.get_header('ETag').gsub('"','')
+          data.headers['Last-Modified'] = Time.parse(fetch_and_delete_header(data, 'Last-Modified'))
+
+          etag = fetch_and_delete_header(data, 'ETag').gsub('"','')
+          data.headers['ETag'] = etag
+
+          DASHED_HEADERS.each do |header|
+            value = fetch_and_delete_header(data, header)
+            data.headers[header] = value if value
+          end
+        end
+
+        private
+
+        def fetch_and_delete_header(response, header)
+          value = response.get_header(header)
+
+          return unless value
+
+          response.headers.keys.each do |key|
+            response.headers.delete(key) if key.downcase == header.downcase
+          end
+
+          value
         end
       end
     end

data/lib/fog/aws/parsers/storage/upload_part_copy_object.rb

@@ -0,0 +1,18 @@
+module Fog
+  module Parsers
+    module AWS
+      module Storage
+        class UploadPartCopyObject < Fog::Parsers::Base
+          def end_element(name)
+            case name
+            when 'ETag'
+              @response[name] = value.gsub('"', '')
+            when 'LastModified'
+              @response[name] = Time.parse(value)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/requests/compute/request_spot_instances.rb

@@ -113,7 +113,7 @@ module Fog
               raise Fog::AWS::Compute::Error.new(message)
             end
 
-            if !image_id.match(/^ami-[a-f0-9]{8}$/)
+            if !image_id.match(/^ami-[a-f0-9]{8,17}$/)
               message = "The image id '[#{image_id}]' does not exist"
               raise Fog::AWS::Compute::NotFound.new(message)
             end

data/lib/fog/aws/requests/storage/delete_multiple_objects.rb

@@ -36,13 +36,20 @@ module Fog
           data << "<Quiet>true</Quiet>" if headers.delete(:quiet)
           version_ids = headers.delete('versionId')
           object_names.each do |object_name|
-            data << "<Object>"
-            data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
             object_version = version_ids.nil? ? nil : version_ids[object_name]
             if object_version
-              data << "<VersionId>#{CGI.escapeHTML(object_version)}</VersionId>"
+              object_version = object_version.is_a?(String) ? [object_version] : object_version
+              object_version.each do |version_id|
+                data << "<Object>"
+                data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
+                data << "<VersionId>#{CGI.escapeHTML(version_id)}</VersionId>"
+                data << "</Object>"
+              end
+            else
+              data << "<Object>"
+              data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
+              data << "</Object>"
             end
-            data << "</Object>"
           end
           data << "</Delete>"
 
@@ -72,10 +79,13 @@ module Fog
             response.body = { 'DeleteResult' => [] }
             version_ids = headers.delete('versionId')
             object_names.each do |object_name|
-              object_version = version_ids.nil? ? nil : version_ids[object_name]
-              response.body['DeleteResult'] << delete_object_helper(bucket,
-                                                                object_name,
-                                                                object_version)
+              object_version = version_ids.nil? ? [nil] : version_ids[object_name]
+              object_version = object_version.is_a?(String) ? [object_version] : object_version
+              object_version.each do |version_id|
+                response.body['DeleteResult'] << delete_object_helper(bucket,
+                                                                  object_name,
+                                                                  version_id)
+              end
             end
           else
             response.status = 404

data/lib/fog/aws/requests/storage/shared_mock_methods.rb

@@ -15,6 +15,7 @@ module Fog
         def parse_mock_data(data)
           data = Fog::Storage.parse_data(data)
           unless data[:body].is_a?(String)
+            data[:body].rewind if data[:body].eof?
             data[:body] = data[:body].read
           end
           data

data/lib/fog/aws/requests/storage/upload_part_copy.rb

@@ -0,0 +1,119 @@
+module Fog
+  module AWS
+    class Storage
+      # From https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+      ALLOWED_UPLOAD_PART_OPTIONS = %i(
+        x-amz-copy-source
+        x-amz-copy-source-if-match
+        x-amz-copy-source-if-modified-since
+        x-amz-copy-source-if-none-match
+        x-amz-copy-source-if-unmodified-since
+        x-amz-copy-source-range
+        x-amz-copy-source-server-side-encryption-customer-algorithm
+        x-amz-copy-source-server-side-encryption-customer-key
+        x-amz-copy-source-server-side-encryption-customer-key-MD5
+        x-amz-expected-bucket-owner
+        x-amz-request-payer
+        x-amz-server-side-encryption-customer-algorithm
+        x-amz-server-side-encryption-customer-key
+        x-amz-server-side-encryption-customer-key-MD5
+        x-amz-source-expected-bucket-owner
+      ).freeze
+
+      class Real
+        require 'fog/aws/parsers/storage/upload_part_copy_object'
+
+        # Upload a part for a multipart copy
+        #
+        # @param target_bucket_name [String] Name of bucket to create copy in
+        # @param target_object_name [String] Name for new copy of object
+        # @param upload_id [String] Id of upload to add part to
+        # @param part_number [String] Index of part in upload
+        # @param options [Hash]:
+        # @option options [String] x-amz-metadata-directive Specifies whether to copy metadata from source or replace with data in request.  Must be in ['COPY', 'REPLACE']
+        # @option options [String] x-amz-copy_source-if-match Copies object if its etag matches this value
+        # @option options [Time] x-amz-copy_source-if-modified_since Copies object it it has been modified since this time
+        # @option options [String] x-amz-copy_source-if-none-match Copies object if its etag does not match this value
+        # @option options [Time] x-amz-copy_source-if-unmodified-since Copies object it it has not been modified since this time
+        # @option options [Time] x-amz-copy-source-range Specifes the range of bytes to copy from the source object
+        #
+        # @return [Excon::Response]
+        #   * body [Hash]:
+        #     * ETag [String] - etag of new object
+        #     * LastModified [Time] - date object was last modified
+        #
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+        #
+        def upload_part_copy(target_bucket_name, target_object_name, upload_id, part_number, options = {})
+          headers = options
+          request({
+            :expects    => 200,
+            :idempotent => true,
+            :headers    => headers,
+            :bucket_name => target_bucket_name,
+            :object_name => target_object_name,
+            :method     => 'PUT',
+            :query      => {'uploadId' => upload_id, 'partNumber' => part_number},
+            :parser   => Fog::Parsers::AWS::Storage::UploadPartCopyObject.new,
+          })
+        end
+      end # Real
+
+      class Mock # :nodoc:all
+        require 'fog/aws/requests/storage/shared_mock_methods'
+        include Fog::AWS::Storage::SharedMockMethods
+
+        def upload_part_copy(target_bucket_name, target_object_name, upload_id, part_number, options = {})
+          validate_options!(options)
+
+          copy_source = options['x-amz-copy-source']
+          copy_range = options['x-amz-copy-source-range']
+
+          raise 'No x-amz-copy-source header provided' unless copy_source
+          raise 'No x-amz-copy-source-range header provided' unless copy_range
+
+          source_bucket_name, source_object_name = copy_source.split('/', 2)
+          verify_mock_bucket_exists(source_bucket_name)
+
+          source_bucket = self.data[:buckets][source_bucket_name]
+          source_object = source_bucket && source_bucket[:objects][source_object_name] && source_bucket[:objects][source_object_name].first
+          upload_info = get_upload_info(target_bucket_name, upload_id)
+
+          response = Excon::Response.new
+
+          if source_object
+            start_pos, end_pos = byte_range(copy_range, source_object[:body].length)
+            upload_info[:parts][part_number] = source_object[:body][start_pos..end_pos]
+
+            response.status = 200
+            response.body = {
+              # just use the part number as the ETag, for simplicity
+              'ETag'          => part_number.to_i,
+              'LastModified'  => Time.parse(source_object['Last-Modified'])
+            }
+            response
+          else
+            response.status = 404
+            raise(Excon::Errors.status_error({:expects => 200}, response))
+          end
+        end
+
+        def byte_range(range, size)
+          matches = range.match(/bytes=(\d*)-(\d*)/)
+
+          return nil unless matches
+
+          end_pos = [matches[2].to_i, size].min
+
+          [matches[1].to_i, end_pos]
+        end
+
+        def validate_options!(options)
+          options.keys.each do |key|
+            raise "Invalid UploadPart option: #{key}" unless ::Fog::AWS::Storage::ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym)
+          end
+        end
+      end # Mock
+    end # Storage
+  end # AWS
+end # Fog

data/lib/fog/aws/storage.rb

@@ -112,6 +112,7 @@ module Fog
       request :put_request_payment
       request :sync_clock
       request :upload_part
+      request :upload_part_copy
 
       module Utils
         attr_accessor :region
@@ -227,7 +228,7 @@ module Fog
           when %r{\Acn-.*}
             "s3.#{region}.amazonaws.com.cn"
           else
-            "s3-#{region}.amazonaws.com"
+            "s3.#{region}.amazonaws.com"
           end
         end
 
@@ -283,10 +284,10 @@ module Fog
               path_style = params.fetch(:path_style, @path_style)
               if !path_style
                 if COMPLIANT_BUCKET_NAMES !~ bucket_name
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 elsif scheme == 'https' && !path_style && bucket_name =~ /\./
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 end
               end
@@ -297,6 +298,8 @@ module Fog
                 host = params.fetch(:cname, bucket_name)
               elsif path_style
                 path = bucket_to_path bucket_name, path
+              elsif host.start_with?("#{bucket_name}.")
+                # no-op
               else
                 host = [bucket_name, host].join('.')
               end