fog-aws 3.7.0 → 3.11.0

data/README.md

@@ -1,7 +1,7 @@
 # Fog::Aws
 
 ![Gem Version](https://badge.fury.io/rb/fog-aws.svg)
-[![Build Status](https://travis-ci.org/fog/fog-aws.svg?branch=master)](https://travis-ci.org/fog/fog-aws)
+[![Build Status](https://github.com/fog/fog-aws/actions/workflows/ruby.yml/badge.svg)](https://github.com/fog/fog-aws/actions/workflows/ruby.yml)
 [![Test Coverage](https://codeclimate.com/github/fog/fog-aws/badges/coverage.svg)](https://codeclimate.com/github/fog/fog-aws)
 [![Code Climate](https://codeclimate.com/github/fog/fog-aws.svg)](https://codeclimate.com/github/fog/fog-aws)
 

data/fog-aws.gemspec

@@ -14,17 +14,18 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/fog/fog-aws"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0")
+  spec.files         = Dir['lib/**/*.rb', 'tests/**/*', 'CHANGELOG.md', 'CONTRIBUTING.md',
+                           'CONTRIBUTORS.md', 'LICENSE.md', 'README.md', 'fog-aws.gemspec',]
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
   spec.required_ruby_version = '>= 2.0.0'
 
-  spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'github_changelog_generator', '~> 1.14'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'github_changelog_generator', '~> 1.16'
   spec.add_development_dependency 'rake',    '>= 12.3.3'
-  spec.add_development_dependency 'rubyzip', '~> 1.3.0'
+  spec.add_development_dependency 'rubyzip', '~> 2.3.0'
   spec.add_development_dependency 'shindo',  '~> 0.3'
 
   spec.add_dependency 'fog-core',  '~> 2.1'

data/lib/fog/aws/credential_fetcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'securerandom'
+
 module Fog
   module AWS
     module CredentialFetcher
@@ -38,7 +40,7 @@ module Fog
                 params = {
                   :Action => "AssumeRoleWithWebIdentity",
                   :RoleArn => options[:role_arn] || ENV.fetch("AWS_ROLE_ARN"),
-                  :RoleSessionName => options[:role_session_name] || ENV.fetch("AWS_ROLE_SESSION_NAME"),
+                  :RoleSessionName => options[:role_session_name] || ENV["AWS_ROLE_SESSION_NAME"] || "fog-aws-#{SecureRandom.hex}",
                   :WebIdentityToken => File.read(options[:aws_web_identity_token_file] || ENV.fetch("AWS_WEB_IDENTITY_TOKEN_FILE")),
                   :Version => "2011-06-15",
                 }

data/lib/fog/aws/models/compute/server.rb

@@ -50,6 +50,7 @@ module Fog
         attribute :subnet_id,                :aliases => 'subnetId'
         attribute :tenancy
         attribute :tags,                     :aliases => 'tagSet'
+        attribute :tag_specifications,       :aliases => 'tagSpecifications'
         attribute :user_data
         attribute :virtualization_type,      :aliases => 'virtualizationType'
         attribute :vpc_id,                   :aliases => 'vpcId'
@@ -166,6 +167,7 @@ module Fog
             'SecurityGroupId'             => security_group_ids,
             'SubnetId'                    => subnet_id,
             'UserData'                    => user_data,
+            'TagSpecifications'           => tag_specifications,
           }
           options.delete_if {|key, value| value.nil?}
 

data/lib/fog/aws/models/storage/file.rb

@@ -4,8 +4,11 @@ module Fog
   module AWS
     class Storage
       class File < Fog::Model
-        MIN_MULTIPART_CHUNK_SIZE = 5242880
-        MAX_SINGLE_PUT_SIZE = 5368709120
+        # @deprecated use {Fog::AWS::Storage::MIN_MULTIPART_CHUNK_SIZE} instead
+        MIN_MULTIPART_CHUNK_SIZE = Fog::AWS::Storage::MIN_MULTIPART_CHUNK_SIZE
+        # @deprecated use {Fog::AWS::Storage::MAX_SINGLE_PUT_SIZE} instead
+        MAX_SINGLE_PUT_SIZE = Fog::AWS::Storage::MAX_SINGLE_PUT_SIZE
+        # @deprecated not used for anything
         MULTIPART_COPY_THRESHOLD = 15728640
 
         # @see AWS Object docs http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html
@@ -30,6 +33,7 @@ module Fog
         attribute :version,             :aliases => 'x-amz-version-id'
         attribute :kms_key_id,          :aliases => 'x-amz-server-side-encryption-aws-kms-key-id'
         attribute :tags,                :aliases => 'x-amz-tagging'
+        attribute :website_redirect_location, :aliases => 'x-amz-website-redirect-location'
 
         UploadPartData = Struct.new(:part_number, :upload_options, :etag)
 
@@ -64,7 +68,7 @@ module Fog
         #     Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb
         attr_reader :multipart_chunk_size
         def multipart_chunk_size=(mp_chunk_size)
-          raise ArgumentError.new("minimum multipart_chunk_size is #{MIN_MULTIPART_CHUNK_SIZE}") if mp_chunk_size < MIN_MULTIPART_CHUNK_SIZE
+          service.validate_chunk_size(mp_chunk_size, 'multipart_chunk_size')
           @multipart_chunk_size = mp_chunk_size
         end
 
@@ -144,11 +148,11 @@ module Fog
         def copy(target_directory_key, target_file_key, options = {})
           requires :directory, :key
 
-          # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
-          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE * 2 if !multipart_chunk_size && self.content_length.to_i > MAX_SINGLE_PUT_SIZE
+          self.multipart_chunk_size = service.max_copy_chunk_size if multipart_chunk_size.nil?
 
-          if multipart_chunk_size && self.content_length.to_i >= multipart_chunk_size
-            upload_part_options = options.merge({ 'x-amz-copy-source' => "#{directory.key}/#{key}" })
+          if multipart_chunk_size > 0 && self.content_length.to_i >= multipart_chunk_size
+            upload_part_options = options.select { |key, _| ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym) }
+            upload_part_options = upload_part_options.merge({ 'x-amz-copy-source' => "#{directory.key}/#{key}" })
             multipart_copy(options, upload_part_options, target_directory_key, target_file_key)
           else
             service.copy_object(directory.key, key, target_directory_key, target_file_key, options)
@@ -248,6 +252,7 @@ module Fog
         # @option options [String] storage_class sets x-amz-storage-class HTTP header. Defaults to 'STANDARD'. Or, 'REDUCED_REDUNDANCY'
         # @option options [String] encryption sets HTTP encryption header. Set to 'AES256' to encrypt files at rest on S3
         # @option options [String] tags sets x-amz-tagging HTTP header. For example, 'Org-Id=1' or 'Org-Id=1&Service=MyService'
+        # @option options [String] website_redirect_location sets x-amz-website-redirect-location HTTP header. For example, 'website_redirect_location=http://www.rubydoc.info/github/fog/fog-aws'
         # @return [Boolean] true if no errors
         #
         def save(options = {})
@@ -265,12 +270,11 @@ module Fog
           options.merge!(metadata)
           options['x-amz-storage-class'] = storage_class if storage_class
           options['x-amz-tagging'] = tags if tags
+          options['x-amz-website-redirect-location'] = website_redirect_location if website_redirect_location
           options.merge!(encryption_headers)
 
-          # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects.
-          self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE if !multipart_chunk_size && Fog::Storage.get_body_size(body) > MAX_SINGLE_PUT_SIZE
-
-          if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
+          self.multipart_chunk_size = service.max_put_chunk_size if multipart_chunk_size.nil?
+          if multipart_chunk_size > 0 && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read)
             data = multipart_save(options)
             merge_attributes(data.body)
           else
@@ -360,7 +364,7 @@ module Fog
           completed = PartList.new
           errors = upload_in_threads(target_directory_key, target_file_key, upload_id, pending, completed, thread_count)
 
-          raise error.first if errors.any?
+          raise errors.first if errors.any?
 
           part_tags = completed.to_a.sort_by { |part| part.part_number }.map(&:etag)
         rescue => e

data/lib/fog/aws/models/storage/files.rb

@@ -17,6 +17,15 @@ module Fog
 
         model Fog::AWS::Storage::File
 
+        DASHED_HEADERS = %w(
+          Cache-Control
+          Content-Disposition
+          Content-Encoding
+          Content-Length
+          Content-MD5
+          Content-Type
+        ).freeze
+
         def all(options = {})
           requires :directory
           options = {
@@ -114,8 +123,29 @@ module Fog
         end
 
         def normalize_headers(data)
-          data.headers['Last-Modified'] = Time.parse(data.get_header('Last-Modified'))
-          data.headers['ETag'] = data.get_header('ETag').gsub('"','')
+          data.headers['Last-Modified'] = Time.parse(fetch_and_delete_header(data, 'Last-Modified'))
+
+          etag = fetch_and_delete_header(data, 'ETag').gsub('"','')
+          data.headers['ETag'] = etag
+
+          DASHED_HEADERS.each do |header|
+            value = fetch_and_delete_header(data, header)
+            data.headers[header] = value if value
+          end
+        end
+
+        private
+
+        def fetch_and_delete_header(response, header)
+          value = response.get_header(header)
+
+          return unless value
+
+          response.headers.keys.each do |key|
+            response.headers.delete(key) if key.downcase == header.downcase
+          end
+
+          value
         end
       end
     end

data/lib/fog/aws/requests/compute/run_instances.rb

@@ -44,6 +44,11 @@ module Fog
         #     * 'PrivateIpAddresses.Primary'<~Bool> - Indicates whether the private IP address is the primary private IP address.
         #     * 'SecondaryPrivateIpAddressCount'<~Bool> - The number of private IP addresses to assign to the network interface.
         #     * 'AssociatePublicIpAddress'<~String> - Indicates whether to assign a public IP address to an instance in a VPC. The public IP address is assigned to a specific network interface
+        #   * 'TagSpecifications'<~Array>: array of hashes
+        #     * 'ResourceType'<~String> - Type of resource to apply tags on, e.g: instance or volume
+        #     * 'Tags'<~Array> - List of hashs reprensenting tag to be set
+        #       * 'Key'<~String> - Tag name
+        #       * 'Value'<~String> - Tag value
         #   * 'ClientToken'<~String> - unique case-sensitive token for ensuring idempotency
         #   * 'DisableApiTermination'<~Boolean> - specifies whether or not to allow termination of the instance from the api
         #   * 'SecurityGroup'<~Array> or <~String> - Name of security group(s) for instances (not supported for VPC)
@@ -144,6 +149,45 @@ module Fog
               end
             end
           end
+          if tag_specifications = options.delete('TagSpecifications')
+            # From https://docs.aws.amazon.com/sdk-for-ruby/v2/api/Aws/EC2/Client.html#run_instances-instance_method
+            # And https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html
+            # Discussed at https://github.com/fog/fog-aws/issues/603
+            #
+            # Example
+            #
+            # TagSpecifications: [
+            #     {
+            #       ResourceType: "instance",
+            #       Tags: [
+            #         {
+            #           Key: "Project",
+            #           Value: "MyProject",
+            #         },
+            #       ],
+            #     },
+            #     {
+            #       ResourceType: "volume",
+            #       Tags: [
+            #         {
+            #           Key: "Project",
+            #           Value: "MyProject",
+            #         },
+            #       ],
+            #     },
+            # ]
+            tag_specifications.each_with_index do |val, idx|
+              resource_type = val["ResourceType"]
+              tags = val["Tags"]
+              options["TagSpecification.#{idx}.ResourceType"] = resource_type
+              tags.each_with_index do |tag, tag_idx|
+                aws_tag_key = "TagSpecification.#{idx}.Tag.#{tag_idx}.Key"
+                aws_tag_value = "TagSpecification.#{idx}.Tag.#{tag_idx}.Value"
+                options[aws_tag_key] = tag["Key"]
+                options[aws_tag_value] = tag["Value"]
+              end
+            end
+          end
 
           idempotent = !(options['ClientToken'].nil? || options['ClientToken'].empty?)
 

data/lib/fog/aws/requests/storage/delete_multiple_objects.rb

@@ -36,13 +36,20 @@ module Fog
           data << "<Quiet>true</Quiet>" if headers.delete(:quiet)
           version_ids = headers.delete('versionId')
           object_names.each do |object_name|
-            data << "<Object>"
-            data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
             object_version = version_ids.nil? ? nil : version_ids[object_name]
             if object_version
-              data << "<VersionId>#{CGI.escapeHTML(object_version)}</VersionId>"
+              object_version = object_version.is_a?(String) ? [object_version] : object_version
+              object_version.each do |version_id|
+                data << "<Object>"
+                data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
+                data << "<VersionId>#{CGI.escapeHTML(version_id)}</VersionId>"
+                data << "</Object>"
+              end
+            else
+              data << "<Object>"
+              data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
+              data << "</Object>"
             end
-            data << "</Object>"
           end
           data << "</Delete>"
 
@@ -72,10 +79,13 @@ module Fog
             response.body = { 'DeleteResult' => [] }
             version_ids = headers.delete('versionId')
             object_names.each do |object_name|
-              object_version = version_ids.nil? ? nil : version_ids[object_name]
-              response.body['DeleteResult'] << delete_object_helper(bucket,
-                                                                object_name,
-                                                                object_version)
+              object_version = version_ids.nil? ? [nil] : version_ids[object_name]
+              object_version = object_version.is_a?(String) ? [object_version] : object_version
+              object_version.each do |version_id|
+                response.body['DeleteResult'] << delete_object_helper(bucket,
+                                                                  object_name,
+                                                                  version_id)
+              end
             end
           else
             response.status = 404

data/lib/fog/aws/requests/storage/get_object.rb

@@ -50,7 +50,7 @@ module Fog
 
           idempotent = true
           if block_given?
-            params[:response_block] = Proc.new
+            params[:response_block] = Proc.new(&block)
             idempotent = false
           end
 

data/lib/fog/aws/requests/storage/shared_mock_methods.rb

@@ -15,6 +15,7 @@ module Fog
         def parse_mock_data(data)
           data = Fog::Storage.parse_data(data)
           unless data[:body].is_a?(String)
+            data[:body].rewind if data[:body].eof?
             data[:body] = data[:body].read
           end
           data

data/lib/fog/aws/requests/storage/upload_part_copy.rb

@@ -1,6 +1,25 @@
 module Fog
   module AWS
     class Storage
+      # From https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+      ALLOWED_UPLOAD_PART_OPTIONS = %i(
+        x-amz-copy-source
+        x-amz-copy-source-if-match
+        x-amz-copy-source-if-modified-since
+        x-amz-copy-source-if-none-match
+        x-amz-copy-source-if-unmodified-since
+        x-amz-copy-source-range
+        x-amz-copy-source-server-side-encryption-customer-algorithm
+        x-amz-copy-source-server-side-encryption-customer-key
+        x-amz-copy-source-server-side-encryption-customer-key-MD5
+        x-amz-expected-bucket-owner
+        x-amz-request-payer
+        x-amz-server-side-encryption-customer-algorithm
+        x-amz-server-side-encryption-customer-key
+        x-amz-server-side-encryption-customer-key-MD5
+        x-amz-source-expected-bucket-owner
+      ).freeze
+
       class Real
         require 'fog/aws/parsers/storage/upload_part_copy_object'
 
@@ -45,6 +64,8 @@ module Fog
         include Fog::AWS::Storage::SharedMockMethods
 
         def upload_part_copy(target_bucket_name, target_object_name, upload_id, part_number, options = {})
+          validate_options!(options)
+
           copy_source = options['x-amz-copy-source']
           copy_range = options['x-amz-copy-source-range']
 
@@ -86,6 +107,12 @@ module Fog
 
           [matches[1].to_i, end_pos]
         end
+
+        def validate_options!(options)
+          options.keys.each do |key|
+            raise "Invalid UploadPart option: #{key}" unless ::Fog::AWS::Storage::ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym)
+          end
+        end
       end # Mock
     end # Storage
   end # AWS

data/lib/fog/aws/storage.rb

@@ -14,6 +14,9 @@ module Fog
         'https' => 443
       }
 
+      MIN_MULTIPART_CHUNK_SIZE = 5242880
+      MAX_SINGLE_PUT_SIZE = 5368709120
+
       VALID_QUERY_KEYS = %w[
         acl
         cors
@@ -43,7 +46,7 @@ module Fog
       ]
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname
+      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname, :max_put_chunk_size, :max_copy_chunk_size
 
       secrets    :aws_secret_access_key, :hmac
 
@@ -117,6 +120,17 @@ module Fog
       module Utils
         attr_accessor :region
 
+        # Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB.
+        # Other S3-compatible storages (like, Ceph) do not have such limit.
+        # Ceph shows much better performance when file is copied as a whole, in a single request.
+        # fog-aws user can use these settings to configure chunk sizes.
+        # A non-positive value will tell fog-aws to use a single put/copy request regardless of file size.
+        #
+        # @return [Integer]
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/copy-object.html
+        attr_reader :max_put_chunk_size
+        attr_reader :max_copy_chunk_size
+
         def cdn
           @cdn ||= Fog::AWS::CDN.new(
             :aws_access_key_id => @aws_access_key_id,
@@ -171,6 +185,12 @@ module Fog
           params_to_url(params)
         end
 
+        # @param value [int]
+        # @param description [str]
+        def validate_chunk_size(value, description)
+          raise "#{description} (#{value}) is less than minimum #{MIN_MULTIPART_CHUNK_SIZE}" unless value <= 0 || value >= MIN_MULTIPART_CHUNK_SIZE
+        end
+
         private
 
         def validate_signature_version!
@@ -179,6 +199,16 @@ module Fog
           end
         end
 
+        def init_max_put_chunk_size!(options = {})
+          @max_put_chunk_size = options.fetch(:max_put_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_put_chunk_size, 'max_put_chunk_size')
+        end
+
+        def init_max_copy_chunk_size!(options = {})
+          @max_copy_chunk_size = options.fetch(:max_copy_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_copy_chunk_size, 'max_copy_chunk_size')
+        end
+
         def v4_signed_params_for_url(params, expires)
           now = Fog::Time.now
 
@@ -284,10 +314,10 @@ module Fog
               path_style = params.fetch(:path_style, @path_style)
               if !path_style
                 if COMPLIANT_BUCKET_NAMES !~ bucket_name
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 elsif scheme == 'https' && !path_style && bucket_name =~ /\./
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 end
               end
@@ -298,6 +328,8 @@ module Fog
                 host = params.fetch(:cname, bucket_name)
               elsif path_style
                 path = bucket_to_path bucket_name, path
+              elsif host.start_with?("#{bucket_name}.")
+                # no-op
               else
                 host = [bucket_name, host].join('.')
               end
@@ -450,6 +482,10 @@ module Fog
 
 
           @path_style = options[:path_style] || false
+
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
           @signature_version = options.fetch(:aws_signature_version, 4)
           validate_signature_version!
           setup_credentials(options)
@@ -513,6 +549,9 @@ module Fog
           validate_signature_version!
           @path_style = options[:path_style]  || false
 
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
           @region = options[:region] || DEFAULT_REGION
 
           if @endpoint = options[:endpoint]