fog-aws 3.5.2 → 3.19.0

data/lib/fog/aws/requests/rds/remove_tags_from_resource.rb

@@ -11,11 +11,12 @@ module Fog
         # * response<~Excon::Response>:
         #   * body<~Hash>:
         def remove_tags_from_resource(rds_id, keys)
+          resource_name = "arn:aws:rds:#{@region}:#{owner_id}:db:#{rds_id}"
+          %w[us-gov-west-1 us-gov-east-1].include?(@region) ? resource_name.insert(7, '-us-gov') : resource_name
           request(
-            { 'Action'        => 'RemoveTagsFromResource',
-              'ResourceName'  => "arn:aws:rds:#{@region}:#{owner_id}:db:#{rds_id}",
-              :parser         => Fog::Parsers::AWS::RDS::Base.new,
-            }.merge(Fog::AWS.indexed_param('TagKeys.member.%d', keys))
+            { 'Action' => 'RemoveTagsFromResource',
+              'ResourceName' => resource_name,
+              :parser => Fog::Parsers::AWS::RDS::Base.new }.merge(Fog::AWS.indexed_param('TagKeys.member.%d', keys))
           )
         end
       end
@@ -23,15 +24,15 @@ module Fog
       class Mock
         def remove_tags_from_resource(rds_id, keys)
           response = Excon::Response.new
-          if server = self.data[:servers][rds_id]
-            keys.each {|key| self.data[:tags][rds_id].delete key}
+          if server = data[:servers][rds_id]
+            keys.each { |key| data[:tags][rds_id].delete key }
             response.status = 200
             response.body = {
-              "ResponseMetadata"=>{ "RequestId"=> Fog::AWS::Mock.request_id }
+              'ResponseMetadata' => { 'RequestId' => Fog::AWS::Mock.request_id }
             }
             response
           else
-            raise Fog::AWS::RDS::NotFound.new("DBInstance #{rds_id} not found")
+            raise Fog::AWS::RDS::NotFound, "DBInstance #{rds_id} not found"
           end
         end
       end

data/lib/fog/aws/requests/storage/delete_multiple_objects.rb

@@ -36,13 +36,20 @@ module Fog
           data << "<Quiet>true</Quiet>" if headers.delete(:quiet)
           version_ids = headers.delete('versionId')
           object_names.each do |object_name|
-            data << "<Object>"
-            data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
             object_version = version_ids.nil? ? nil : version_ids[object_name]
             if object_version
-              data << "<VersionId>#{CGI.escapeHTML(object_version)}</VersionId>"
+              object_version = object_version.is_a?(String) ? [object_version] : object_version
+              object_version.each do |version_id|
+                data << "<Object>"
+                data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
+                data << "<VersionId>#{CGI.escapeHTML(version_id)}</VersionId>"
+                data << "</Object>"
+              end
+            else
+              data << "<Object>"
+              data << "<Key>#{CGI.escapeHTML(object_name)}</Key>"
+              data << "</Object>"
             end
-            data << "</Object>"
           end
           data << "</Delete>"
 
@@ -72,10 +79,13 @@ module Fog
             response.body = { 'DeleteResult' => [] }
             version_ids = headers.delete('versionId')
             object_names.each do |object_name|
-              object_version = version_ids.nil? ? nil : version_ids[object_name]
-              response.body['DeleteResult'] << delete_object_helper(bucket,
-                                                                object_name,
-                                                                object_version)
+              object_version = version_ids.nil? ? [nil] : version_ids[object_name]
+              object_version = object_version.is_a?(String) ? [object_version] : object_version
+              object_version.each do |version_id|
+                response.body['DeleteResult'] << delete_object_helper(bucket,
+                                                                  object_name,
+                                                                  version_id)
+              end
             end
           else
             response.status = 404

data/lib/fog/aws/requests/storage/get_object.rb

@@ -50,7 +50,7 @@ module Fog
 
           idempotent = true
           if block_given?
-            params[:response_block] = Proc.new
+            params[:response_block] = Proc.new(&block)
             idempotent = false
           end
 

data/lib/fog/aws/requests/storage/get_object_tagging.rb

@@ -0,0 +1,41 @@
+module Fog
+  module AWS
+    class Storage
+      class Real
+        require 'fog/aws/parsers/storage/get_object_tagging'
+
+        # Get tags for an S3 object
+        #
+        # @param bucket_name [String] Name of bucket to read from
+        # @param object_name [String] Name of object to get tags for
+        #
+        # @return [Excon::Response] response:
+        #   * body [Hash]:
+        #     * ObjectTagging [Hash]:
+        #       * Key [String] - tag key
+        #       * Value [String] - tag value
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html
+
+        def get_object_tagging(bucket_name, object_name)
+          unless bucket_name
+            raise ArgumentError.new('bucket_name is required')
+          end
+          unless object_name
+            raise ArgumentError.new('object_name is required')
+          end
+
+          request({
+            :expects => 200,
+            :headers => {},
+            :bucket_name => bucket_name,
+            :object_name => object_name,
+            :idempotent => true,
+            :method => 'GET',
+            :parser => Fog::Parsers::AWS::Storage::GetObjectTagging.new,
+            :query => {'tagging' => nil}
+          })
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/requests/storage/get_service.rb

@@ -21,7 +21,7 @@ module Fog
           request({
             :expects  => 200,
             :headers  => {},
-            :host     => 's3.amazonaws.com',
+            :host     => @host || region_to_host(DEFAULT_REGION),
             :idempotent => true,
             :method   => 'GET',
             :parser   => Fog::Parsers::AWS::Storage::GetService.new

data/lib/fog/aws/requests/storage/put_object_tagging.rb

@@ -0,0 +1,42 @@
+module Fog
+  module AWS
+    class Storage
+      class Real
+        # Change tag set for an S3 object
+        #
+        # @param bucket_name [String] Name of bucket to modify object in
+        # @param object_name [String] Name of object to modify
+        #
+        # @param tags [Hash]:
+        #   * Key [String]: tag key
+        #   * Value [String]: tag value
+        #
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html
+
+        def put_object_tagging(bucket_name, object_name, tags)
+          tagging = tags.map do |k,v|
+            "<Tag><Key>#{k}</Key><Value>#{v}</Value></Tag>"
+          end.join("\n")
+          data =
+              <<-DATA
+<Tagging xmlns="http://doc.s3.amazonaws.com/2006-03-01" >
+  <TagSet>
+    #{tagging}
+  </TagSet>
+</Tagging>
+          DATA
+
+          request({
+            :body     => data,
+            :expects  => 200,
+            :headers  => {'Content-MD5' => Base64.encode64(OpenSSL::Digest::MD5.digest(data)).chomp!, 'Content-Type' => 'application/xml'},
+            :bucket_name => bucket_name,
+            :object_name => object_name,
+            :method   => 'PUT',
+            :query    => {'tagging' => nil}
+          })
+        end
+      end
+    end
+  end
+end

data/lib/fog/aws/requests/storage/shared_mock_methods.rb

@@ -15,6 +15,7 @@ module Fog
         def parse_mock_data(data)
           data = Fog::Storage.parse_data(data)
           unless data[:body].is_a?(String)
+            data[:body].rewind if data[:body].eof?
             data[:body] = data[:body].read
           end
           data

data/lib/fog/aws/requests/storage/sync_clock.rb

@@ -6,12 +6,20 @@ module Fog
         #
         def sync_clock
           response = begin
-            get_service
+            Excon.get(sync_clock_url)
           rescue Excon::Errors::HTTPStatusError => error
             error.response
           end
           Fog::Time.now = Time.parse(response.headers['Date'])
         end
+
+        private
+
+        def sync_clock_url
+          host = @acceleration ? region_to_host(@region) : @host
+
+          "#{@scheme}://#{host}"
+        end
       end # Real
 
       class Mock # :nodoc:all

data/lib/fog/aws/requests/storage/upload_part_copy.rb

@@ -0,0 +1,119 @@
+module Fog
+  module AWS
+    class Storage
+      # From https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+      ALLOWED_UPLOAD_PART_OPTIONS = %i(
+        x-amz-copy-source
+        x-amz-copy-source-if-match
+        x-amz-copy-source-if-modified-since
+        x-amz-copy-source-if-none-match
+        x-amz-copy-source-if-unmodified-since
+        x-amz-copy-source-range
+        x-amz-copy-source-server-side-encryption-customer-algorithm
+        x-amz-copy-source-server-side-encryption-customer-key
+        x-amz-copy-source-server-side-encryption-customer-key-MD5
+        x-amz-expected-bucket-owner
+        x-amz-request-payer
+        x-amz-server-side-encryption-customer-algorithm
+        x-amz-server-side-encryption-customer-key
+        x-amz-server-side-encryption-customer-key-MD5
+        x-amz-source-expected-bucket-owner
+      ).freeze
+
+      class Real
+        require 'fog/aws/parsers/storage/upload_part_copy_object'
+
+        # Upload a part for a multipart copy
+        #
+        # @param target_bucket_name [String] Name of bucket to create copy in
+        # @param target_object_name [String] Name for new copy of object
+        # @param upload_id [String] Id of upload to add part to
+        # @param part_number [String] Index of part in upload
+        # @param options [Hash]:
+        # @option options [String] x-amz-metadata-directive Specifies whether to copy metadata from source or replace with data in request.  Must be in ['COPY', 'REPLACE']
+        # @option options [String] x-amz-copy_source-if-match Copies object if its etag matches this value
+        # @option options [Time] x-amz-copy_source-if-modified_since Copies object it it has been modified since this time
+        # @option options [String] x-amz-copy_source-if-none-match Copies object if its etag does not match this value
+        # @option options [Time] x-amz-copy_source-if-unmodified-since Copies object it it has not been modified since this time
+        # @option options [Time] x-amz-copy-source-range Specifes the range of bytes to copy from the source object
+        #
+        # @return [Excon::Response]
+        #   * body [Hash]:
+        #     * ETag [String] - etag of new object
+        #     * LastModified [Time] - date object was last modified
+        #
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html
+        #
+        def upload_part_copy(target_bucket_name, target_object_name, upload_id, part_number, options = {})
+          headers = options
+          request({
+            :expects    => 200,
+            :idempotent => true,
+            :headers    => headers,
+            :bucket_name => target_bucket_name,
+            :object_name => target_object_name,
+            :method     => 'PUT',
+            :query      => {'uploadId' => upload_id, 'partNumber' => part_number},
+            :parser   => Fog::Parsers::AWS::Storage::UploadPartCopyObject.new,
+          })
+        end
+      end # Real
+
+      class Mock # :nodoc:all
+        require 'fog/aws/requests/storage/shared_mock_methods'
+        include Fog::AWS::Storage::SharedMockMethods
+
+        def upload_part_copy(target_bucket_name, target_object_name, upload_id, part_number, options = {})
+          validate_options!(options)
+
+          copy_source = options['x-amz-copy-source']
+          copy_range = options['x-amz-copy-source-range']
+
+          raise 'No x-amz-copy-source header provided' unless copy_source
+          raise 'No x-amz-copy-source-range header provided' unless copy_range
+
+          source_bucket_name, source_object_name = copy_source.split('/', 2)
+          verify_mock_bucket_exists(source_bucket_name)
+
+          source_bucket = self.data[:buckets][source_bucket_name]
+          source_object = source_bucket && source_bucket[:objects][source_object_name] && source_bucket[:objects][source_object_name].first
+          upload_info = get_upload_info(target_bucket_name, upload_id)
+
+          response = Excon::Response.new
+
+          if source_object
+            start_pos, end_pos = byte_range(copy_range, source_object[:body].length)
+            upload_info[:parts][part_number] = source_object[:body][start_pos..end_pos]
+
+            response.status = 200
+            response.body = {
+              # just use the part number as the ETag, for simplicity
+              'ETag'          => part_number.to_i,
+              'LastModified'  => Time.parse(source_object['Last-Modified'])
+            }
+            response
+          else
+            response.status = 404
+            raise(Excon::Errors.status_error({:expects => 200}, response))
+          end
+        end
+
+        def byte_range(range, size)
+          matches = range.match(/bytes=(\d*)-(\d*)/)
+
+          return nil unless matches
+
+          end_pos = [matches[2].to_i, size].min
+
+          [matches[1].to_i, end_pos]
+        end
+
+        def validate_options!(options)
+          options.keys.each do |key|
+            raise "Invalid UploadPart option: #{key}" unless ::Fog::AWS::Storage::ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym)
+          end
+        end
+      end # Mock
+    end # Storage
+  end # AWS
+end # Fog

data/lib/fog/aws/requests/sts/assume_role_with_web_identity.rb

@@ -6,12 +6,13 @@ module Fog
 
         def assume_role_with_web_identity(role_arn, web_identity_token, role_session_name, options={})
           request_unsigned(
-            'Action'          => 'AssumeRoleWithWebIdentity',
-            'RoleArn'         => role_arn,
-            'RoleSessionName' => role_session_name,
-            'DurationSeconds' => options[:duration] || 3600,
-            :idempotent       => true,
-            :parser           => Fog::Parsers::AWS::STS::AssumeRoleWithWebIdentity.new
+            'Action'            => 'AssumeRoleWithWebIdentity',
+            'RoleArn'           => role_arn,
+            'RoleSessionName'   => role_session_name,
+            'WebIdentityToken'  => web_identity_token,
+            'DurationSeconds'   => options[:duration] || 3600,
+            :idempotent         => true,
+            :parser             => Fog::Parsers::AWS::STS::AssumeRoleWithWebIdentity.new
           )
         end
       end

data/lib/fog/aws/storage.rb

@@ -14,6 +14,9 @@ module Fog
         'https' => 443
       }
 
+      MIN_MULTIPART_CHUNK_SIZE = 5242880
+      MAX_SINGLE_PUT_SIZE = 5368709120
+
       VALID_QUERY_KEYS = %w[
         acl
         cors
@@ -43,7 +46,7 @@ module Fog
       ]
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname
+      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname, :max_put_chunk_size, :max_copy_chunk_size, :aws_credentials_refresh_threshold_seconds, :disable_content_md5_validation
 
       secrets    :aws_secret_access_key, :hmac
 
@@ -84,6 +87,7 @@ module Fog
       request :get_object_http_url
       request :get_object_https_url
       request :get_object_url
+      request :get_object_tagging
       request :get_request_payment
       request :get_service
       request :head_bucket
@@ -107,12 +111,26 @@ module Fog
       request :put_object
       request :put_object_acl
       request :put_object_url
+      request :put_object_tagging
       request :put_request_payment
       request :sync_clock
       request :upload_part
+      request :upload_part_copy
 
       module Utils
         attr_accessor :region
+        attr_accessor :disable_content_md5_validation
+
+        # Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB.
+        # Other S3-compatible storages (like, Ceph) do not have such limit.
+        # Ceph shows much better performance when file is copied as a whole, in a single request.
+        # fog-aws user can use these settings to configure chunk sizes.
+        # A non-positive value will tell fog-aws to use a single put/copy request regardless of file size.
+        #
+        # @return [Integer]
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/copy-object.html
+        attr_reader :max_put_chunk_size
+        attr_reader :max_copy_chunk_size
 
         def cdn
           @cdn ||= Fog::AWS::CDN.new(
@@ -168,6 +186,12 @@ module Fog
           params_to_url(params)
         end
 
+        # @param value [int]
+        # @param description [str]
+        def validate_chunk_size(value, description)
+          raise "#{description} (#{value}) is less than minimum #{MIN_MULTIPART_CHUNK_SIZE}" unless value <= 0 || value >= MIN_MULTIPART_CHUNK_SIZE
+        end
+
         private
 
         def validate_signature_version!
@@ -176,6 +200,16 @@ module Fog
           end
         end
 
+        def init_max_put_chunk_size!(options = {})
+          @max_put_chunk_size = options.fetch(:max_put_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_put_chunk_size, 'max_put_chunk_size')
+        end
+
+        def init_max_copy_chunk_size!(options = {})
+          @max_copy_chunk_size = options.fetch(:max_copy_chunk_size, MAX_SINGLE_PUT_SIZE)
+          validate_chunk_size(@max_copy_chunk_size, 'max_copy_chunk_size')
+        end
+
         def v4_signed_params_for_url(params, expires)
           now = Fog::Time.now
 
@@ -225,7 +259,7 @@ module Fog
           when %r{\Acn-.*}
             "s3.#{region}.amazonaws.com.cn"
           else
-            "s3-#{region}.amazonaws.com"
+            "s3.#{region}.amazonaws.com"
           end
         end
 
@@ -281,10 +315,10 @@ module Fog
               path_style = params.fetch(:path_style, @path_style)
               if !path_style
                 if COMPLIANT_BUCKET_NAMES !~ bucket_name
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 elsif scheme == 'https' && !path_style && bucket_name =~ /\./
-                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: http://docs.amazonwebservices.com/AmazonS3/latest/dev/BucketRestrictions.html")
+                  Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
                   path_style = true
                 end
               end
@@ -295,6 +329,8 @@ module Fog
                 host = params.fetch(:cname, bucket_name)
               elsif path_style
                 path = bucket_to_path bucket_name, path
+              elsif host.start_with?("#{bucket_name}.")
+                # no-op
               else
                 host = [bucket_name, host].join('.')
               end
@@ -447,6 +483,12 @@ module Fog
 
 
           @path_style = options[:path_style] || false
+
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
+          @disable_content_md5_validation = options[:disable_content_md5_validation] || false
+
           @signature_version = options.fetch(:aws_signature_version, 4)
           validate_signature_version!
           setup_credentials(options)
@@ -461,6 +503,8 @@ module Fog
         end
 
         def setup_credentials(options)
+          @aws_credentials_refresh_threshold_seconds = options[:aws_credentials_refresh_threshold_seconds]
+
           @aws_access_key_id = options[:aws_access_key_id]
           @aws_secret_access_key = options[:aws_secret_access_key]
           @aws_session_token     = options[:aws_session_token]
@@ -502,7 +546,7 @@ module Fog
           @use_iam_profile = options[:use_iam_profile]
           @instrumentor       = options[:instrumentor]
           @instrumentor_name  = options[:instrumentor_name] || 'fog.aws.storage'
-          @connection_options     = options[:connection_options] || {}
+          @connection_options     = options[:connection_options] || { retry_limit: 5, retry_interval: 1 }
           @persistent = options.fetch(:persistent, false)
           @acceleration = options.fetch(:acceleration, false)
           @signature_version = options.fetch(:aws_signature_version, 4)
@@ -510,6 +554,11 @@ module Fog
           validate_signature_version!
           @path_style = options[:path_style]  || false
 
+          init_max_put_chunk_size!(options)
+          init_max_copy_chunk_size!(options)
+
+          @disable_content_md5_validation = options[:disable_content_md5_validation] || false
+
           @region = options[:region] || DEFAULT_REGION
 
           if @endpoint = options[:endpoint]
@@ -535,6 +584,8 @@ module Fog
 
 
         def setup_credentials(options)
+          @aws_credentials_refresh_threshold_seconds = options[:aws_credentials_refresh_threshold_seconds]
+
           @aws_access_key_id     = options[:aws_access_key_id]
           @aws_secret_access_key = options[:aws_secret_access_key]
           @aws_session_token     = options[:aws_session_token]

data/lib/fog/aws/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AWS
-    VERSION = "3.5.2"
+    VERSION = "3.19.0"
   end
 end

data/lib/fog/aws.rb

@@ -26,6 +26,7 @@ module Fog
     autoload :ECS,              File.expand_path('../aws/ecs', __FILE__)
     autoload :EFS,              File.expand_path('../aws/efs', __FILE__)
     autoload :ELB,              File.expand_path('../aws/elb', __FILE__)
+    autoload :ELBV2,            File.expand_path('../aws/elbv2', __FILE__)
     autoload :EMR,              File.expand_path('../aws/emr', __FILE__)
     autoload :ElasticBeanstalk, File.expand_path('../aws/beanstalk', __FILE__)
     autoload :Elasticache,      File.expand_path('../aws/elasticache', __FILE__)
@@ -58,6 +59,7 @@ module Fog
     service(:ecs,             'ECS')
     service(:efs,             'EFS')
     service(:elb,             'ELB')
+    service(:elbv2,           'ELBV2')
     service(:emr,             'EMR')
     service(:federation,      'Federation')
     service(:glacier,         'Glacier')
@@ -218,19 +220,22 @@ module Fog
 
     def self.regions
       @regions ||= [
+        'af-south-1',
         'ap-east-1',
         'ap-northeast-1', 'ap-northeast-2', 'ap-northeast-3',
         'ap-south-1',
-        'ap-southeast-1', 'ap-southeast-2',
+        'ap-southeast-1', 'ap-southeast-2', 'ap-southeast-3', 'ap-southeast-4',
         'ca-central-1',
         'cn-north-1',
         'cn-northwest-1',
         'eu-central-1',
         'eu-north-1',
-        'eu-west-1', 'eu-west-2', 'eu-west-3',
+        'eu-west-1', 'eu-west-2', 'eu-west-3', 'eu-south-1', 'eu-south-2',
+        'me-south-1','me-central-1',
         'us-east-1', 'us-east-2',
         'us-west-1', 'us-west-2',
         'sa-east-1',
+        'us-gov-east-1',
         'us-gov-west-1'
       ]
     end