fog-aws 3.6.4 → 3.8.0

data/lib/fog/aws/models/compute/server.rb

@@ -14,6 +14,7 @@ module Fog
         attribute :associate_public_ip,      :aliases => 'associatePublicIP'
         attribute :availability_zone,        :aliases => 'availabilityZone'
         attribute :block_device_mapping,     :aliases => 'blockDeviceMapping'
+        attribute :hibernation_options,      :aliases => 'hibernationOptions'
         attribute :network_interfaces,       :aliases => 'networkInterfaces'
         attribute :client_token,             :aliases => 'clientToken'
         attribute :disable_api_termination,  :aliases => 'disableApiTermination'
@@ -144,6 +145,7 @@ module Fog
 
           options = {
             'BlockDeviceMapping'          => block_device_mapping,
+            'HibernationOptions'          => hibernation_options,
             'NetworkInterfaces'           => network_interfaces,
             'ClientToken'                 => client_token,
             'DisableApiTermination'       => disable_api_termination,
@@ -227,9 +229,9 @@ module Fog
           true
         end
 
-        def stop(force = false)
+        def stop(options = {})
           requires :id
-          service.stop_instances(id, force)
+          service.stop_instances(id, options)
           true
         end
 

data/lib/fog/aws/models/compute/servers.rb

@@ -22,6 +22,7 @@ module Fog
         #    ami_launch_index=nil,
         #    availability_zone=nil,
         #    block_device_mapping=nil,
+        #    hibernation_options=nil,
         #    network_interfaces=nil,
         #    client_token=nil,
         #    dns_name=nil,
@@ -119,6 +120,7 @@ module Fog
         #    ami_launch_index=0,
         #    availability_zone="us-east-1b",
         #    block_device_mapping=[],
+        #    hibernation_options=[],
         #    client_token=nil,
         #    dns_name="ec2-25-2-474-44.compute-1.amazonaws.com",
         #    groups=["default"],

data/lib/fog/aws/models/compute/vpc.rb

@@ -39,7 +39,7 @@ module Fog
         end
 
         def is_default?
-          require :is_default
+          requires :is_default
           is_default
         end
 

data/lib/fog/aws/models/storage/file.rb

@@ -4,6 +4,10 @@ module Fog
   module AWS
     class Storage
       class File < Fog::Model
+        MIN_MULTIPART_CHUNK_SIZE = 5242880
+        MAX_SINGLE_PUT_SIZE = 5368709120
+        MULTIPART_COPY_THRESHOLD = 15728640
+
         # @see AWS Object docs http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html
 
         identity  :key,             :aliases => 'Key'
@@ -27,14 +31,54 @@ module Fog
         attribute :kms_key_id,          :aliases => 'x-amz-server-side-encryption-aws-kms-key-id'
         attribute :tags,                :aliases => 'x-amz-tagging'
 
+        UploadPartData = Struct.new(:part_number, :upload_options, :etag)
+
+        class PartList
+          def initialize(parts = [])
+            @parts = parts
+            @mutex = Mutex.new
+          end
+
+          def push(part)
+            @mutex.synchronize { @parts.push(part) }
+          end
+
+          def shift
+            @mutex.synchronize { @parts.shift }
+          end
+
+          def clear!
+            @mutex.synchronize { @parts.clear }
+          end
+
+          def size
+            @mutex.synchronize { @parts.size }
+          end
+
+          def to_a
+            @mutex.synchronize { @parts.dup }
+          end
+        end
+
         # @note Chunk size to use for multipart uploads.
         #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
         attr_reader :multipart_chunk_size
         def multipart_chunk_size=(mp_chunk_size)
-          raise ArgumentError.new("minimum multipart_chunk_size is 5242880") if mp_chunk_size < 5242880
+          raise ArgumentError.new("minimum multipart_chunk_size is #{MIN_MULTIPART_CHUNK_SIZE}") if mp_chunk_size < MIN_MULTIPART_CHUNK_SIZE
           @multipart_chunk_size = mp_chunk_size
         end
 
+        # @note Number of threads used to copy files.
+        def concurrency=(concurrency)
+          raise ArgumentError.new('minimum concurrency is 1') if concurrency.to_i < 1
+
+          @concurrency = concurrency.to_i
+        end
+
+        def concurrency
+          @concurrency || 1
+        end
+
         def acl
           requires :directory, :key
           service.get_object_acl(directory.key, key).body['AccessControlList']
@@ -99,7 +143,18 @@ module Fog
         #
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
-          service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+
+          # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
+          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE * 2 if !multipart_chunk_size && self.content_length.to_i > MAX_SINGLE_PUT_SIZE
+
+          if multipart_chunk_size && self.content_length.to_i >= multipart_chunk_size
+            upload_part_options = options.select { |key, _| ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym) }
+            upload_part_options = upload_part_options.merge({ 'x-amz-copy-source' => "#{directory.key}/#{key}" })
+            multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          else
+            service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
+          end
+
           target_directory = service.directories.new(:key => target_directory_key)
           target_directory.files.head(target_file_key)
         end
@@ -214,7 +269,7 @@ module Fog
           options.merge!(encryption_headers)
 
           # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
-          self.multipart_chunk_size = 5242880 if !multipart_chunk_size && Fog::Storage.get_body_size(body) > 5368709120
+          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE if !multipart_chunk_size && Fog::Storage.get_body_size(body) > MAX_SINGLE_PUT_SIZE
 
           if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
             data = multipart_save(options)
@@ -294,6 +349,30 @@ module Fog
           service.complete_multipart_upload(directory.key, key, upload_id, part_tags)
         end
 
+        def multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
+          # Initiate the upload
+          res = service.initiate_multipart_upload(target_directory_key, target_file_key, options)
+          upload_id = res.body["UploadId"]
+
+          # Store ETags of upload parts
+          part_tags = []
+          pending = PartList.new(create_part_list(upload_part_options))
+          thread_count = self.concurrency
+          completed = PartList.new
+          errors = upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+
+          raise errors.first if errors.any?
+
+          part_tags = completed.to_a.sort_by { |part| part.part_number }.map(&:etag)
+        rescue => e
+          # Abort the upload & reraise
+          service.abort_multipart_upload(target_directory_key, target_file_key, upload_id) if upload_id
+          raise
+        else
+          # Complete the upload
+          service.complete_multipart_upload(target_directory_key, target_file_key, upload_id, part_tags)
+        end
+
         def encryption_headers
           if encryption && encryption_key
             encryption_customer_key_headers
@@ -318,6 +397,49 @@ module Fog
             'x-amz-server-side-encryption-customer-key-md5' => Base64.encode64(OpenSSL::Digest::MD5.digest(encryption_key.to_s)).chomp!
           }
         end
+
+        def create_part_list(upload_part_options)
+          current_pos = 0
+          count = 0
+          pending = []
+
+          while current_pos < self.content_length do
+            start_pos = current_pos
+            end_pos = [current_pos + self.multipart_chunk_size, self.content_length - 1].min
+            range = "bytes=#{start_pos}-#{end_pos}"
+            part_options = upload_part_options.dup
+            part_options['x-amz-copy-source-range'] = range
+            pending << UploadPartData.new(count + 1, part_options, nil)
+            count += 1
+            current_pos = end_pos + 1
+          end
+
+          pending
+        end
+
+        def upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
+          threads = []
+
+          thread_count.times do
+            thread = Thread.new do
+              begin
+                while part = pending.shift
+                  part_upload = service.upload_part_copy(target_directory_key, target_file_key, upload_id, part.part_number, part.upload_options)
+                  part.etag = part_upload.body['ETag']
+                  completed.push(part)
+                end
+              rescue => error
+                pending.clear!
+                error
+              end
+            end
+
+            thread.abort_on_exception = true
+            threads << thread
+          end
+
+          threads.map(&:value).compact
+        end
       end
     end
   end

data/lib/fog/aws/parsers/storage/upload_part_copy_object.rb

@@ -0,0 +1,18 @@
+module Fog
+  module Parsers
+    module AWS
+      module Storage
+        class UploadPartCopyObject < Fog::Parsers::Base
+          def end_element(name)
+            case name
+            when 'ETag'
+              @response[name] = value.gsub('"', '')
+            when 'LastModified'
+              @response[name] = Time.parse(value)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/requests/compute/request_spot_instances.rb

@@ -113,7 +113,7 @@ module Fog
               raise Fog::AWS::Compute::Error.new(message)
             end
 
-            if !image_id.match(/^ami-[a-f0-9]{8}$/)
+            if !image_id.match(/^ami-[a-f0-9]{8,17}$/)
               message = "The image id '[#{image_id}]' does not exist"
               raise Fog::AWS::Compute::NotFound.new(message)
             end

data/lib/fog/aws/requests/compute/run_instances.rb

@@ -30,6 +30,8 @@ module Fog
         #     * 'Ebs.Encrypted'<~Boolean> - specifies whether or not the volume is to be encrypted unless snapshot is specified
         #     * 'Ebs.VolumeType'<~String> - Type of EBS volue. Valid options in ['standard', 'io1'] default is 'standard'.
         #     * 'Ebs.Iops'<~String> - The number of I/O operations per second (IOPS) that the volume supports. Required when VolumeType is 'io1'
+        #   * 'HibernationOptions'<~Array>: array of hashes
+        #     * 'Configured'<~Boolean> - specifies whether or not the instance is configued for hibernation.  This parameter is valid only if the instance meets the hibernation prerequisites.  
         #   * 'NetworkInterfaces'<~Array>: array of hashes
         #     * 'NetworkInterfaceId'<~String> - An existing interface to attach to a single instance
         #     * 'DeviceIndex'<~String> - The device index. Applies both to attaching an existing network interface and creating a network interface
@@ -75,6 +77,8 @@ module Fog
         #           * 'deviceName'<~String> - specifies how volume is exposed to instance
         #           * 'status'<~String> - status of attached volume
         #           * 'volumeId'<~String> - Id of attached volume
+        #         * 'hibernationOptions'<~Array>
+        #           * 'configured'<~Boolean> - whether or not the instance is enabled for hibernation               
         #         * 'dnsName'<~String> - public dns name, blank until instance is running
         #         * 'imageId'<~String> - image id of ami used to launch instance
         #         * 'instanceId'<~String> - id of the instance
@@ -111,6 +115,13 @@ module Fog
               end
             end
           end
+          if hibernation_options = options.delete('HibernationOptions')
+            hibernation_options.each_with_index do |mapping, index|
+              for key, value in mapping
+                options.merge!({ format("HibernationOptions.%d.#{key}", index) => value })
+              end
+            end
+          end
           if security_groups = options.delete('SecurityGroup')
             options.merge!(Fog::AWS.indexed_param('SecurityGroup', [*security_groups]))
           end
@@ -182,6 +193,14 @@ module Fog
               }
             end
 
+	    hibernation_options = (options['HibernationOptions'] || []).reduce([]) do |mapping, device|
+              configure = device.fetch("Configure", true)
+
+              mapping << {
+                "Configure" => configure,
+              }
+            end
+
             if options['SubnetId']
               if options['PrivateIpAddress']
                 ni_options = {'PrivateIpAddress' => options['PrivateIpAddress']}
@@ -221,6 +240,7 @@ module Fog
               'associatePublicIP'     => options['associatePublicIP'] || false,
               'architecture'          => 'i386',
               'blockDeviceMapping'    => block_device_mapping,
+              'hibernationOptions'    => hibernation_options,
               'networkInterfaces'     => network_interfaces,
               'clientToken'           => options['clientToken'],
               'dnsName'               => nil,

data/lib/fog/aws/requests/compute/stop_instances.rb

@@ -16,9 +16,17 @@ module Fog
         #     * TODO: fill in the blanks
         #
         # {Amazon API Reference}[http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-StopInstances.html]
-        def stop_instances(instance_id, force = false)
+        def stop_instances(instance_id, options = {})
           params = Fog::AWS.indexed_param('InstanceId', instance_id)
-          params.merge!('Force' => 'true') if force
+          unless options.is_a?(Hash)
+            Fog::Logger.warning("stop_instances with #{options.class} param is deprecated, use stop_instances('force' => boolean) instead [light_black](#{caller.first})[/]")
+            options = {'force' => options}
+          end
+          params.merge!('Force' => 'true') if options['force']
+          if options['hibernate']
+            params.merge!('Hibernate' => 'true')
+            params.merge!('Force' => 'false')
+          end
           request({
             'Action'    => 'StopInstances',
             :idempotent => true,
@@ -28,7 +36,7 @@ module Fog
       end
 
       class Mock
-        def stop_instances(instance_id, force = false)
+        def stop_instances(instance_id, options = {})
           instance_ids = Array(instance_id)
 
           instance_set = self.data[:instances].values

data/lib/fog/aws/requests/storage/shared_mock_methods.rb

@@ -15,6 +15,7 @@ module Fog
         def parse_mock_data(data)
           data = Fog::Storage.parse_data(data)
           unless data[:body].is_a?(String)
+            data[:body].rewind if data[:body].eof?
             data[:body] = data[:body].read
           end
           data

data/lib/fog/aws/requests/storage/upload_part_copy.rb

@@ -0,0 +1,119 @@
+module Fog
+  module AWS
+    class Storage
+      # From https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+      ALLOWED_UPLOAD_PART_OPTIONS = %i(
+        x-amz-copy-source
+        x-amz-copy-source-if-match
+        x-amz-copy-source-if-modified-since
+        x-amz-copy-source-if-none-match
+        x-amz-copy-source-if-unmodified-since
+        x-amz-copy-source-range
+        x-amz-copy-source-server-side-encryption-customer-algorithm
+        x-amz-copy-source-server-side-encryption-customer-key
+        x-amz-copy-source-server-side-encryption-customer-key-MD5
+        x-amz-expected-bucket-owner
+        x-amz-request-payer
+        x-amz-server-side-encryption-customer-algorithm
+        x-amz-server-side-encryption-customer-key
+        x-amz-server-side-encryption-customer-key-MD5
+        x-amz-source-expected-bucket-owner
+      ).freeze
+
+      class Real
+        require 'fog/aws/parsers/storage/upload_part_copy_object'
+
+        # Upload a part for a multipart copy
+        #
+        # @param target_bucket_name [String] Name of bucket to create copy in
+        # @param target_object_name [String] Name for new copy of object
+        # @param upload_id [String] Id of upload to add part to
+        # @param part_number [String] Index of part in upload
+        # @param options [Hash]:
+        # @option options [String] x-amz-metadata-directive Specifies whether to copy metadata from source or replace with data in request.  Must be in ['COPY', 'REPLACE']
+        # @option options [String] x-amz-copy_source-if-match Copies object if its etag matches this value
+        # @option options [Time] x-amz-copy_source-if-modified_since Copies object it it has been modified since this time
+        # @option options [String] x-amz-copy_source-if-none-match Copies object if its etag does not match this value
+        # @option options [Time] x-amz-copy_source-if-unmodified-since Copies object it it has not been modified since this time
+        # @option options [Time] x-amz-copy-source-range Specifes the range of bytes to copy from the source object
+        #
+        # @return [Excon::Response]
+        #   * body [Hash]:
+        #     * ETag [String] - etag of new object
+        #     * LastModified [Time] - date object was last modified
+        #
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+        #
+        def upload_part_copy(target_bucket_name, target_object_name, upload_id, part_number, options = {})
+          headers = options
+          request({
+            :expects    => 200,
+            :idempotent => true,
+            :headers    => headers,
+            :bucket_name => target_bucket_name,
+            :object_name => target_object_name,
+            :method     => 'PUT',
+            :query      => {'uploadId' => upload_id, 'partNumber' => part_number},
+            :parser   => Fog::Parsers::AWS::Storage::UploadPartCopyObject.new,
+          })
+        end
+      end # Real
+
+      class Mock # :nodoc:all
+        require 'fog/aws/requests/storage/shared_mock_methods'
+        include Fog::AWS::Storage::SharedMockMethods
+
+        def upload_part_copy(target_bucket_name, target_object_name, upload_id, part_number, options = {})
+          validate_options!(options)
+
+          copy_source = options['x-amz-copy-source']
+          copy_range = options['x-amz-copy-source-range']
+
+          raise 'No x-amz-copy-source header provided' unless copy_source
+          raise 'No x-amz-copy-source-range header provided' unless copy_range
+
+          source_bucket_name, source_object_name = copy_source.split('/', 2)
+          verify_mock_bucket_exists(source_bucket_name)
+
+          source_bucket = self.data[:buckets][source_bucket_name]
+          source_object = source_bucket && source_bucket[:objects][source_object_name] && source_bucket[:objects][source_object_name].first
+          upload_info = get_upload_info(target_bucket_name, upload_id)
+
+          response = Excon::Response.new
+
+          if source_object
+            start_pos, end_pos = byte_range(copy_range, source_object[:body].length)
+            upload_info[:parts][part_number] = source_object[:body][start_pos..end_pos]
+
+            response.status = 200
+            response.body = {
+              # just use the part number as the ETag, for simplicity
+              'ETag'          => part_number.to_i,
+              'LastModified'  => Time.parse(source_object['Last-Modified'])
+            }
+            response
+          else
+            response.status = 404
+            raise(Excon::Errors.status_error({:expects => 200}, response))
+          end
+        end
+
+        def byte_range(range, size)
+          matches = range.match(/bytes=(\d*)-(\d*)/)
+
+          return nil unless matches
+
+          end_pos = [matches[2].to_i, size].min
+
+          [matches[1].to_i, end_pos]
+        end
+
+        def validate_options!(options)
+          options.keys.each do |key|
+            raise "Invalid UploadPart option: #{key}" unless ::Fog::AWS::Storage::ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym)
+          end
+        end
+      end # Mock
+    end # Storage
+  end # AWS
+end # Fog