fog-aws 3.13.0 → 3.21.0

data/lib/fog/aws/models/storage/directories.rb

@@ -11,6 +11,7 @@ module Fog
           load(data)
         end
 
+        # Warning! This retrieves and caches meta data for the first 10,000 objects in the bucket, which can be very expensive. When possible use directories.new
         def get(key, options = {})
           remap_attributes(options, {
             :delimiter  => 'delimiter',

data/lib/fog/aws/models/storage/file.rb

@@ -180,7 +180,8 @@ module Fog
 
         remove_method :metadata
         def metadata
-          attributes.reject {|key, value| !(key.to_s =~ /^x-amz-/)}
+          attributes.reject {|key, value| !(key.to_s =~ /^x-amz-/) }
+                    .reject {|key, value| ['x-amz-id-2', 'x-amz-request-id'].include?(key) }
         end
 
         remove_method :metadata=
@@ -335,12 +336,12 @@ module Fog
             body.rewind  rescue nil
           end
           while (chunk = body.read(multipart_chunk_size)) do
-            part_upload = service.upload_part(directory.key, key, upload_id, part_tags.size + 1, chunk, part_headers(chunk, options))
+            part_upload = service.upload_part(directory.key, key, upload_id, part_tags.size + 1, chunk, part_headers(chunk))
             part_tags << part_upload.headers["ETag"]
           end
 
           if part_tags.empty? #it is an error to have a multipart upload with no parts
-            part_upload = service.upload_part(directory.key, key, upload_id, 1, '', part_headers('', options))
+            part_upload = service.upload_part(directory.key, key, upload_id, 1, '', part_headers(''))
             part_tags << part_upload.headers["ETag"]
           end
 
@@ -387,11 +388,18 @@ module Fog
           end
         end
 
-        def part_headers(chunk, options)
-          md5 = Base64.encode64(OpenSSL::Digest::MD5.digest(chunk)).strip
-          encryption_keys = encryption_customer_key_headers.keys
-          encryption_headers = options.select { |key| encryption_keys.include?(key) }
-          { 'Content-MD5' => md5 }.merge(encryption_headers)
+        def part_headers(chunk)
+          base_headers = part_checksum_headers(chunk)
+
+          # Only SSE-C headers needed in the UploadPart request. [1]
+          # x-amz-server-side-encryption and
+          # x-amz-server-side-encryption-aws-kms-key-id are only needed
+          # in the CreateMultipartUpload request. [2]
+          # [1] https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html
+          # [2] https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html
+          base_headers.merge!(encryption_customer_key_headers) if encryption && encryption_key
+
+          base_headers
         end
 
         def encryption_customer_key_headers
@@ -402,6 +410,14 @@ module Fog
           }
         end
 
+        def part_checksum_headers(chunk)
+          if service.disable_content_md5_validation
+            {}
+          else
+            { 'Content-MD5' => Base64.encode64(OpenSSL::Digest::MD5.digest(chunk)).strip }
+          end
+        end
+
         def create_part_list(upload_part_options)
           current_pos = 0
           count = 0

data/lib/fog/aws/parsers/iam/get_group_policy.rb

@@ -14,7 +14,7 @@ module Fog
             when 'GroupName', 'PolicyName'
               @response[name] = value
             when 'PolicyDocument'
-              @response['Policy'][name] = if decoded_string = URI.decode(value)
+              @response['Policy'][name] = if decoded_string = URI.decode_www_form_component(value)
                                   Fog::JSON.decode(decoded_string) rescue value
                                 else
                                   value

data/lib/fog/aws/parsers/iam/get_role_policy.rb

@@ -12,7 +12,7 @@ module Fog
             when 'RoleName', 'PolicyName'
               @response['Policy'][name] = value
             when 'PolicyDocument'
-              @response['Policy'][name] = if decoded_string = URI.decode(value)
+              @response['Policy'][name] = if decoded_string = URI.decode_www_form_component(value)
                                   Fog::JSON.decode(decoded_string) rescue value
                                 else
                                   value

data/lib/fog/aws/parsers/iam/get_user_policy.rb

@@ -14,7 +14,7 @@ module Fog
             when 'UserName', 'PolicyName'
               @response['Policy'][name] = value
             when 'PolicyDocument'
-              @response['Policy'][name] = if decoded_string = URI.decode(value)
+              @response['Policy'][name] = if decoded_string = URI.decode_www_form_component(value)
                                   Fog::JSON.decode(decoded_string) rescue value
                                 else
                                   value

data/lib/fog/aws/parsers/iam/policy_version.rb

@@ -18,7 +18,7 @@ module Fog
             when 'IsDefaultVersion'
               @version[name] = (value == 'true')
             when 'Document'
-              @version[name] = if decoded_string = URI.decode(value)
+              @version[name] = if decoded_string = URI.decode_www_form_component(value)
                                  Fog::JSON.decode(decoded_string) rescue value
                                else
                                  value

data/lib/fog/aws/rds.rb

@@ -9,7 +9,7 @@ module Fog
       class AuthorizationAlreadyExists < Fog::Errors::Error; end
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :version, :instrumentor, :instrumentor_name
+      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :version, :instrumentor, :instrumentor_name, :sts_endpoint
 
       request_path 'fog/aws/requests/rds'
       request :describe_events

data/lib/fog/aws/redshift.rb

@@ -4,7 +4,7 @@ module Fog
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name
+      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :sts_endpoint
 
       request_path 'fog/aws/requests/redshift'
 

data/lib/fog/aws/requests/rds/add_tags_to_resource.rb

@@ -12,28 +12,30 @@ module Fog
         #   * body<~Hash>:
         def add_tags_to_resource(rds_id, tags)
           keys    = tags.keys.sort
-          values  = keys.map {|key| tags[key]}
+          values  = keys.map { |key| tags[key] }
+          resource_name = "arn:aws:rds:#{@region}:#{owner_id}:db:#{rds_id}"
+          %w[us-gov-west-1 us-gov-east-1].include?(@region) ? resource_name.insert(7, '-us-gov') : resource_name
           request({
-              'Action'        => 'AddTagsToResource',
-              'ResourceName'  => "arn:aws:rds:#{@region}:#{owner_id}:db:#{rds_id}",
-              :parser         => Fog::Parsers::AWS::RDS::Base.new,
-            }.merge(Fog::AWS.indexed_param('Tags.member.%d.Key', keys)).
-              merge(Fog::AWS.indexed_param('Tags.member.%d.Value', values)))
+            'Action' => 'AddTagsToResource',
+            'ResourceName' => resource_name,
+            :parser => Fog::Parsers::AWS::RDS::Base.new
+          }.merge(Fog::AWS.indexed_param('Tags.member.%d.Key', keys))
+              .merge(Fog::AWS.indexed_param('Tags.member.%d.Value', values)))
         end
       end
 
       class Mock
         def add_tags_to_resource(rds_id, tags)
           response = Excon::Response.new
-          if server = self.data[:servers][rds_id]
-            self.data[:tags][rds_id].merge! tags
+          if server = data[:servers][rds_id]
+            data[:tags][rds_id].merge! tags
             response.status = 200
             response.body = {
-              "ResponseMetadata"=>{ "RequestId"=> Fog::AWS::Mock.request_id }
+              'ResponseMetadata' => { 'RequestId' => Fog::AWS::Mock.request_id }
             }
             response
           else
-            raise Fog::AWS::RDS::NotFound.new("DBInstance #{rds_id} not found")
+            raise Fog::AWS::RDS::NotFound, "DBInstance #{rds_id} not found"
           end
         end
       end

data/lib/fog/aws/requests/rds/list_tags_for_resource.rb

@@ -11,11 +11,14 @@ module Fog
         # ==== Returns
         # * response<~Excon::Response>:
         #   * body<~Hash>:
+
         def list_tags_for_resource(rds_id)
+          resource_name = "arn:aws:rds:#{@region}:#{owner_id}:db:#{rds_id}"
+          %w[us-gov-west-1 us-gov-east-1].include?(@region) ? resource_name.insert(7, '-us-gov') : resource_name
           request(
-            'Action'        => 'ListTagsForResource',
-            'ResourceName'  => "arn:aws:rds:#{@region}:#{owner_id}:db:#{rds_id}",
-            :parser         => Fog::Parsers::AWS::RDS::TagListParser.new
+            'Action' => 'ListTagsForResource',
+            'ResourceName' => resource_name,
+            :parser => Fog::Parsers::AWS::RDS::TagListParser.new
           )
         end
       end
@@ -23,15 +26,15 @@ module Fog
       class Mock
         def list_tags_for_resource(rds_id)
           response = Excon::Response.new
-          if server = self.data[:servers][rds_id]
+          if server = data[:servers][rds_id]
             response.status = 200
             response.body = {
-              "ListTagsForResourceResult" =>
-                {"TagList" =>  self.data[:tags][rds_id]}
+              'ListTagsForResourceResult' =>
+                { 'TagList' => data[:tags][rds_id] }
             }
             response
           else
-            raise Fog::AWS::RDS::NotFound.new("DBInstance #{rds_id} not found")
+            raise Fog::AWS::RDS::NotFound, "DBInstance #{rds_id} not found"
           end
         end
       end

data/lib/fog/aws/requests/rds/remove_tags_from_resource.rb

@@ -11,11 +11,12 @@ module Fog
         # * response<~Excon::Response>:
         #   * body<~Hash>:
         def remove_tags_from_resource(rds_id, keys)
+          resource_name = "arn:aws:rds:#{@region}:#{owner_id}:db:#{rds_id}"
+          %w[us-gov-west-1 us-gov-east-1].include?(@region) ? resource_name.insert(7, '-us-gov') : resource_name
           request(
-            { 'Action'        => 'RemoveTagsFromResource',
-              'ResourceName'  => "arn:aws:rds:#{@region}:#{owner_id}:db:#{rds_id}",
-              :parser         => Fog::Parsers::AWS::RDS::Base.new,
-            }.merge(Fog::AWS.indexed_param('TagKeys.member.%d', keys))
+            { 'Action' => 'RemoveTagsFromResource',
+              'ResourceName' => resource_name,
+              :parser => Fog::Parsers::AWS::RDS::Base.new }.merge(Fog::AWS.indexed_param('TagKeys.member.%d', keys))
           )
         end
       end
@@ -23,15 +24,15 @@ module Fog
       class Mock
         def remove_tags_from_resource(rds_id, keys)
           response = Excon::Response.new
-          if server = self.data[:servers][rds_id]
-            keys.each {|key| self.data[:tags][rds_id].delete key}
+          if server = data[:servers][rds_id]
+            keys.each { |key| data[:tags][rds_id].delete key }
             response.status = 200
             response.body = {
-              "ResponseMetadata"=>{ "RequestId"=> Fog::AWS::Mock.request_id }
+              'ResponseMetadata' => { 'RequestId' => Fog::AWS::Mock.request_id }
             }
             response
           else
-            raise Fog::AWS::RDS::NotFound.new("DBInstance #{rds_id} not found")
+            raise Fog::AWS::RDS::NotFound, "DBInstance #{rds_id} not found"
           end
         end
       end

data/lib/fog/aws/requests/storage/get_service.rb

@@ -21,7 +21,7 @@ module Fog
           request({
             :expects  => 200,
             :headers  => {},
-            :host     => 's3.amazonaws.com',
+            :host     => @host || region_to_host(DEFAULT_REGION),
             :idempotent => true,
             :method   => 'GET',
             :parser   => Fog::Parsers::AWS::Storage::GetService.new

data/lib/fog/aws/requests/storage/sync_clock.rb

@@ -6,12 +6,20 @@ module Fog
         #
         def sync_clock
           response = begin
-            get_service
+            Excon.get(sync_clock_url)
           rescue Excon::Errors::HTTPStatusError => error
             error.response
           end
           Fog::Time.now = Time.parse(response.headers['Date'])
         end
+
+        private
+
+        def sync_clock_url
+          host = @acceleration ? region_to_host(@region) : @host
+
+          "#{@scheme}://#{host}:#{@port}"
+        end
       end # Real
 
       class Mock # :nodoc:all

data/lib/fog/aws/ses.rb

@@ -7,7 +7,7 @@ module Fog
       class MessageRejected < Fog::Errors::Error; end
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name
+      recognizes :region, :host, :path, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :sts_endpoint
 
       request_path 'fog/aws/requests/ses'
       request :delete_verified_email_address

data/lib/fog/aws/simpledb.rb

@@ -4,7 +4,7 @@ module Fog
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :host, :nil_string, :path, :port, :scheme, :persistent, :region, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name
+      recognizes :host, :nil_string, :path, :port, :scheme, :persistent, :region, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :sts_endpoint
 
       request_path 'fog/aws/requests/simpledb'
       request :batch_put_attributes

data/lib/fog/aws/sns.rb

@@ -4,7 +4,7 @@ module Fog
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :host, :path, :port, :scheme, :persistent, :region, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name
+      recognizes :host, :path, :port, :scheme, :persistent, :region, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :sts_endpoint
 
       request_path 'fog/aws/requests/sns'
       request :add_permission

data/lib/fog/aws/sqs.rb

@@ -4,7 +4,7 @@ module Fog
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :region, :host, :path, :port, :scheme, :persistent, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name
+      recognizes :region, :host, :path, :port, :scheme, :persistent, :aws_session_token, :use_iam_profile, :aws_credentials_expire_at, :instrumentor, :instrumentor_name, :sts_endpoint
 
       request_path 'fog/aws/requests/sqs'
       request :change_message_visibility

data/lib/fog/aws/storage.rb

@@ -14,6 +14,14 @@ module Fog
         'https' => 443
       }
 
+      DEFAULT_CONNECTION_OPTIONS = {
+        retry_limit: 5,
+        retry_interval: 1,
+        retry_errors: [
+          Excon::Error::Timeout, Excon::Error::Socket, Excon::Error::Server
+        ]
+      }
+
       MIN_MULTIPART_CHUNK_SIZE = 5242880
       MAX_SINGLE_PUT_SIZE = 5368709120
 
@@ -46,7 +54,7 @@ module Fog
       ]
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname, :max_put_chunk_size, :max_copy_chunk_size
+      recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname, :max_put_chunk_size, :max_copy_chunk_size, :aws_credentials_refresh_threshold_seconds, :disable_content_md5_validation, :sts_endpoint
 
       secrets    :aws_secret_access_key, :hmac
 
@@ -119,6 +127,7 @@ module Fog
 
       module Utils
         attr_accessor :region
+        attr_accessor :disable_content_md5_validation
 
         # Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB.
         # Other S3-compatible storages (like, Ceph) do not have such limit.
@@ -486,6 +495,8 @@ module Fog
           init_max_put_chunk_size!(options)
           init_max_copy_chunk_size!(options)
 
+          @disable_content_md5_validation = options[:disable_content_md5_validation] || false
+
           @signature_version = options.fetch(:aws_signature_version, 4)
           validate_signature_version!
           setup_credentials(options)
@@ -500,6 +511,8 @@ module Fog
         end
 
         def setup_credentials(options)
+          @aws_credentials_refresh_threshold_seconds = options[:aws_credentials_refresh_threshold_seconds]
+
           @aws_access_key_id = options[:aws_access_key_id]
           @aws_secret_access_key = options[:aws_secret_access_key]
           @aws_session_token     = options[:aws_session_token]
@@ -541,7 +554,8 @@ module Fog
           @use_iam_profile = options[:use_iam_profile]
           @instrumentor       = options[:instrumentor]
           @instrumentor_name  = options[:instrumentor_name] || 'fog.aws.storage'
-          @connection_options     = options[:connection_options] || {}
+          @connection_options =
+            DEFAULT_CONNECTION_OPTIONS.merge(options[:connection_options] || {})
           @persistent = options.fetch(:persistent, false)
           @acceleration = options.fetch(:acceleration, false)
           @signature_version = options.fetch(:aws_signature_version, 4)
@@ -552,6 +566,8 @@ module Fog
           init_max_put_chunk_size!(options)
           init_max_copy_chunk_size!(options)
 
+          @disable_content_md5_validation = options[:disable_content_md5_validation] || false
+
           @region = options[:region] || DEFAULT_REGION
 
           if @endpoint = options[:endpoint]
@@ -577,6 +593,8 @@ module Fog
 
 
         def setup_credentials(options)
+          @aws_credentials_refresh_threshold_seconds = options[:aws_credentials_refresh_threshold_seconds]
+
           @aws_access_key_id     = options[:aws_access_key_id]
           @aws_secret_access_key = options[:aws_secret_access_key]
           @aws_session_token     = options[:aws_session_token]

data/lib/fog/aws/support.rb

@@ -4,7 +4,7 @@ module Fog
       extend Fog::AWS::CredentialFetcher::ServiceMethods
 
       requires :aws_access_key_id, :aws_secret_access_key
-      recognizes :host, :path, :port, :scheme, :instrumentor, :instrumentor_name, :region, :persistent, :aws_session_token
+      recognizes :host, :path, :port, :scheme, :instrumentor, :instrumentor_name, :region, :persistent, :aws_session_token, :aws_credentials_expire_at, :sts_endpoint
 
       model_path 'fog/aws/models/support'
       request_path 'fog/aws/requests/support'

data/lib/fog/aws/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module AWS
-    VERSION = "3.13.0"
+    VERSION = "3.21.0"
   end
 end

data/lib/fog/aws.rb

@@ -224,14 +224,14 @@ module Fog
         'ap-east-1',
         'ap-northeast-1', 'ap-northeast-2', 'ap-northeast-3',
         'ap-south-1',
-        'ap-southeast-1', 'ap-southeast-2',
+        'ap-southeast-1', 'ap-southeast-2', 'ap-southeast-3', 'ap-southeast-4',
         'ca-central-1',
         'cn-north-1',
         'cn-northwest-1',
         'eu-central-1',
         'eu-north-1',
-        'eu-west-1', 'eu-west-2', 'eu-west-3', 'eu-south-1',
-        'me-south-1',
+        'eu-west-1', 'eu-west-2', 'eu-west-3', 'eu-south-1', 'eu-south-2',
+        'me-south-1','me-central-1',
         'us-east-1', 'us-east-2',
         'us-west-1', 'us-west-2',
         'sa-east-1',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fog-aws
 version: !ruby/object:Gem::Version
-  version: 3.13.0
+  version: 3.21.0
 platform: ruby
 authors:
 - Josh Lane
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-13 00:00:00.000000000 Z
+date: 2023-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -1458,7 +1458,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Module for the 'fog' gem to support Amazon Web Services.