aws-sdk-s3 1.109.0 → 1.156.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b7b755424c2783bb1d1ab5ec357ce83949b1a434cca1f40d8088c4e59c982b0
-  data.tar.gz: 95b6da26a68850f8d388cc7fe67f6b152297bb8988ff6a83a915b0d800b977c7
+  metadata.gz: 444d8fbcdc8d19b75228bdec52d8e1651f4d35459866847055b0d7e16d56371a
+  data.tar.gz: 785d49ecff3a7441d4f3c196ed64d7cfbe79330627c585fc7108a4a7b11d6d2b
 SHA512:
-  metadata.gz: afbc607faa1be4484f457ffac8617350b826c8e794b75eecdc66b07dadf91a5e339328070aa1ab2c4c2a32e99a82d94e07a54d0cd5803f9a12a10db43a642414
-  data.tar.gz: 19e01fdc6064838e1af8514eccc4784a1e859dc89bc6a274f017c722154a11ad1c2f79929fa0e7aecaa70f6cd8af32db3f07e4de427b34944957d664ef2348aa
+  metadata.gz: e60cb04b18ce10ec62f4b63f28cf8ed7adf206a22910c7251ed9d1c2040a3466cb2d09a0987d6fc8d2361b4c15dfa0b764877b5c7bdedd03146383b5a11a79ae
+  data.tar.gz: 2e5654d95f462359aebeae89f2b855211b95acbe1820ad354a7a8c0e8261ff1188715e469637aea8c647124a7672f102a08e5fa78d73ed27253fcb886cfa13b8

data/CHANGELOG.md

@@ -1,6 +1,358 @@
 Unreleased Changes
 ------------------
 
+1.156.0 (2024-07-02)
+------------------
+
+* Feature - Added response overrides to Head Object requests.
+
+1.155.0 (2024-06-28)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.154.0 (2024-06-25)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.153.0 (2024-06-24)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.152.3 (2024-06-13)
+------------------
+
+* Issue - Handle 200 errors for all S3 operations that do not have streaming responses.
+
+1.152.2 (2024-06-12)
+------------------
+
+* Issue - Revert Handling of 200 errors for all S3 operations.
+
+1.152.1 (2024-06-10)
+------------------
+
+* Issue - Handle 200 errors for all S3 operations that do not have streaming responses.
+
+1.152.0 (2024-06-05)
+------------------
+
+* Feature - Added new params copySource and key to copyObject API for supporting S3 Access Grants plugin. These changes will not change any of the existing S3 API functionality.
+
+1.151.0 (2024-05-14)
+------------------
+
+* Feature - Updated a few x-id in the http uri traits
+
+1.150.0 (2024-05-13)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.149.1 (2024-05-06)
+------------------
+
+* Issue - Fix bug where destination bucket default encryption was inadvertently overridden by source object encryption.
+
+1.149.0 (2024-04-30)
+------------------
+
+* Feature - Support S3 Access Grants authentication. Access Grants can be enabled with the `access_grants` option, and custom options can be passed into the `access_grants_credentials_provider` option. This feature requires `aws-sdk-s3control` to be installed.
+
+* Feature - Add RBS signatures for customizations of S3.
+
+1.148.0 (2024-04-25)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.147.0 (2024-04-16)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+* Issue - Omit `ContentType` plugin when generating presigned url.
+
+1.146.1 (2024-03-28)
+------------------
+
+* Issue - Fix bug where thread_count option was not being respected for multipart uploads.
+
+1.146.0 (2024-03-18)
+------------------
+
+* Feature - Fix two issues with response root node names.
+
+1.145.0 (2024-03-15)
+------------------
+
+* Feature - Documentation updates for Amazon S3.
+
+1.144.0 (2024-03-13)
+------------------
+
+* Feature - This release makes the default option for S3 on Outposts request signing to use the SigV4A algorithm when using AWS Common Runtime (CRT).
+
+1.143.1 (2024-03-12)
+------------------
+
+* Issue - Include original part errors in message when aborting multipart upload fails (#2990).
+
+1.143.0 (2024-01-26)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.142.0 (2023-12-22)
+------------------
+
+* Feature - Added additional examples for some operations.
+
+1.141.0 (2023-11-28)
+------------------
+
+* Feature - Adds support for S3 Express One Zone.
+
+* Feature - Support S3 Express authentication and endpoints. Express session auth can be disabled with the `disable_s3_express_session_auth` Client option, the `AWS_S3_DISABLE_EXPRESS_SESSION_AUTH` environment variable, and the `s3_disable_express_session_auth` shared config option. A custom `express_credentials_provider` can be configured onto the Client.
+
+1.140.0 (2023-11-27)
+------------------
+
+* Feature - Adding new params - Key and Prefix, to S3 API operations for supporting S3 Access Grants. Note - These updates will not change any of the existing S3 API functionality.
+
+* Issue - Fix thread interruptions in multipart `download_file`, `file_uploader` and `stream_uploader` (#2944).
+
+1.139.0 (2023-11-22)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.138.0 (2023-11-21)
+------------------
+
+* Feature - Add support for automatic date based partitioning in S3 Server Access Logs.
+
+1.137.0 (2023-11-17)
+------------------
+
+* Feature - Removes all default 0 values for numbers and false values for booleans
+
+1.136.0 (2023-09-26)
+------------------
+
+* Feature - This release adds a new field COMPLETED to the ReplicationStatus Enum. You can now use this field to validate the replication status of S3 objects using the AWS SDK.
+
+1.135.0 (2023-09-20)
+------------------
+
+* Feature - Fix an issue where the SDK can fail to unmarshall response due to NumberFormatException
+
+1.134.0 (2023-08-24)
+------------------
+
+* Feature - Updates to endpoint ruleset tests to address Smithy validation issues.
+
+1.133.0 (2023-08-22)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+* Feature - Add support for `progress_callback` in `Object#download_file` and improve multi-threaded performance #(2901).
+
+1.132.1 (2023-08-09)
+------------------
+
+* Issue - Add support for disabling checksum validation in `Object#download_file` (#2893).
+
+1.132.0 (2023-07-24)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+* Feature - Add support for verifying checksums in FileDownloader.
+
+1.131.0 (2023-07-20)
+------------------
+
+* Feature - Improve performance of S3 clients by simplifying and optimizing endpoint resolution.
+
+1.130.0 (2023-07-13)
+------------------
+
+* Feature - S3 Inventory now supports Object Access Control List and Object Owner as available object metadata fields in inventory reports.
+
+* Feature - Allow Object multipart copy API to work when requiring a checksum algorithm.
+
+* Feature - Allow Object multipart copy API to optionally copy parts as they exist on the source object if it has parts, instead of generating new part ranges, when specifying `use_source_parts: true`.
+
+1.129.0 (2023-07-11)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.128.0 (2023-07-06)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.127.0 (2023-06-28)
+------------------
+
+* Feature - The S3 LISTObjects, ListObjectsV2 and ListObjectVersions API now supports a new optional header x-amz-optional-object-attributes. If header contains RestoreStatus as the value, then S3 will include Glacier restore status i.e. isRestoreInProgress and RestoreExpiryDate in List response.
+
+* Feature - Select minimum expiration time for presigned urls between the expiration time option and the credential expiration time.
+
+1.126.0 (2023-06-16)
+------------------
+
+* Feature - This release adds SDK support for request-payer request header and request-charged response header in the "GetBucketAccelerateConfiguration", "ListMultipartUploads", "ListObjects", "ListObjectsV2" and "ListObjectVersions" S3 APIs.
+
+1.125.0 (2023-06-15)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.124.0 (2023-06-13)
+------------------
+
+* Feature - Integrate double encryption feature to SDKs.
+
+1.123.2 (2023-06-12)
+------------------
+
+* Issue - Fix issue when decrypting noncurrent versions of objects when using client side encryption (#2866).
+
+1.123.1 (2023-06-02)
+------------------
+
+* Issue - Fix multipart `download_file` so that it does not download bytes out of range (#2859).
+
+1.123.0 (2023-05-31)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.122.0 (2023-05-04)
+------------------
+
+* Feature - Documentation updates for Amazon S3
+
+1.121.0 (2023-04-19)
+------------------
+
+* Feature - Provides support for "Snow" Storage class.
+
+1.120.1 (2023-04-05)
+------------------
+
+* Issue - Skip `#check_for_cached_region` if custom endpoint provided
+
+1.120.0 (2023-03-31)
+------------------
+
+* Feature - Documentation updates for Amazon S3
+
+1.119.2 (2023-03-22)
+------------------
+
+* Issue - Provide `endpoint` and `bucket` attributes on `Aws::S3::Errors::PermanentRedirect` error objects.
+
+1.119.1 (2023-02-13)
+------------------
+
+* Issue - Ensure object metadata is not lost on multipart copy (#2821).
+
+1.119.0 (2023-01-26)
+------------------
+
+* Feature - Allow FIPS to be used with path-style URLs.
+
+1.118.0 (2023-01-18)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+* Issue - Replace runtime endpoint resolution approach with generated ruby code.
+
+1.117.2 (2022-11-30)
+------------------
+
+* Issue - Return error messages from failures in threads in `MultipartStreamUploader` (#2793).
+
+1.117.1 (2022-10-26)
+------------------
+
+* Issue - Fix custom endpoint and port regression with `presigned_url` (#2776).
+
+1.117.0 (2022-10-25)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+* Issue - Apply checksums to MultipartStreamUploader (#2769).
+
+1.116.0 (2022-10-21)
+------------------
+
+* Feature - S3 on Outposts launches support for automatic bucket-style alias. You can use the automatic access point alias instead of an access point ARN for any object-level operation in an Outposts bucket.
+
+1.115.0 (2022-10-19)
+------------------
+
+* Feature - Updates internal logic for constructing API endpoints. We have added rule-based endpoints and internal model parameters.
+
+1.114.0 (2022-05-03)
+------------------
+
+* Feature - Documentation only update for doc bug fixes for the S3 API docs.
+
+1.113.2 (2022-04-26)
+------------------
+
+* Issue - Fix an issue where `ExpiredToken` errors were retried as if the request was from another region.
+
+1.113.1 (2022-04-25)
+------------------
+
+* Issue - Rewind the underlying file on a streaming retry that is not a truncated body (#2692).
+
+1.113.0 (2022-02-24)
+------------------
+
+* Feature - This release adds support for new integrity checking capabilities in Amazon S3. You can choose from four supported checksum algorithms for data integrity checking on your upload and download requests. In addition, AWS SDK can automatically calculate a checksum as it streams data into S3
+
+1.112.0 (2022-02-03)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+1.111.3 (2022-01-24)
+------------------
+
+* Issue - Fix starts_with fields on `PresignedPost` (#2636).
+
+1.111.2 (2022-01-20)
+------------------
+
+* Issue - Minor cleanups.
+
+1.111.1 (2022-01-06)
+------------------
+
+* Issue - Don't fail small files in `upload_file` when `:thread_count` is set. (#2628)
+
+1.111.0 (2022-01-04)
+------------------
+
+* Feature - Minor doc-based updates based on feedback bugs received.
+
+1.110.0 (2021-12-21)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
 1.109.0 (2021-11-30)
 ------------------
 

data/VERSION

@@ -1 +1 @@
-1.109.0
+1.156.0

data/lib/aws-sdk-s3/access_grants_credentials.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Aws
+  module S3
+    # @api private
+    class AccessGrantsCredentials
+      include CredentialProvider
+      include RefreshingCredentials
+
+      def initialize(options = {})
+        @client = options[:client]
+        @get_data_access_params = {}
+        options.each_pair do |key, value|
+          if self.class.get_data_access_options.include?(key)
+            @get_data_access_params[key] = value
+          end
+        end
+        @async_refresh = true
+        super
+      end
+
+      # @return [S3Control::Client]
+      attr_reader :client
+
+      # @return [String]
+      attr_reader :matched_grant_target
+
+      private
+
+      def refresh
+        c = @client.get_data_access(@get_data_access_params)
+        credentials = c.credentials
+        @matched_grant_target = c.matched_grant_target
+        @credentials = Credentials.new(
+          credentials.access_key_id,
+          credentials.secret_access_key,
+          credentials.session_token
+        )
+        @expiration = credentials.expiration
+      end
+
+      class << self
+
+        # @api private
+        def get_data_access_options
+          @gdao ||= begin
+            input = Aws::S3Control::Client.api.operation(:get_data_access).input
+            Set.new(input.shape.member_names)
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/access_grants_credentials_provider.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    # @api private
+    def self.access_grants_credentials_cache
+      @access_grants_credentials_cache ||= LRUCache.new(max_entries: 100)
+    end
+
+    # @api private
+    def self.access_grants_account_id_cache
+      @access_grants_account_id_cache ||= LRUCache.new(
+        max_entries: 100,
+        expiration: 60 * 10
+      )
+    end
+
+    # Returns Credentials class for S3 Access Grants. Accepts GetDataAccess
+    # params and other configuration as options. See
+    # {Aws::S3Control::Client#get_data_access} for details.
+    class AccessGrantsCredentialsProvider
+      # @param [Hash] options
+      # @option options [Hash] :s3_control_client_options The S3 Control
+      #  client options used to create regional S3 Control clients to
+      #  create the session. Region will be set to the region of the
+      #  bucket.
+      # @option options [Aws::STS::Client] :sts_client The STS client used for
+      #  fetching the Account ID for the credentials if credentials do not
+      #  include an Account ID.
+      # @option options [Aws::S3::Client] :s3_client The S3 client used for
+      #  fetching the location of the bucket so that a regional S3 Control
+      #  client can be created. Defaults to the S3 client from the access
+      #  grants plugin.
+      # @option options [String] :privilege ('Default') The privilege to use
+      #  when requesting credentials. (see: {Aws::S3Control::Client#get_data_access})
+      # @option options [Boolean] :fallback (false) When true, if access is
+      #  denied, the provider will fall back to the configured credentials.
+      # @option options [Boolean] :caching (true) When true, credentials and
+      #  bucket account ids will be cached.
+      # @option options [Callable] :before_refresh Proc called before
+      #  credentials are refreshed.
+      def initialize(options = {})
+        @s3_control_options = options.delete(:s3_control_client_options) || {}
+        @s3_client = options.delete(:s3_client)
+        @sts_client = options.delete(:sts_client)
+        @fallback = options.delete(:fallback) || false
+        @caching = options.delete(:caching) != false
+        @s3_control_clients = {}
+        @bucket_region_cache = Aws::S3.bucket_region_cache
+        return unless @caching
+
+        @credentials_cache = Aws::S3.access_grants_credentials_cache
+        @account_id_cache = Aws::S3.access_grants_account_id_cache
+      end
+
+      def access_grants_credentials_for(options = {})
+        target = target_prefix(
+          options[:bucket],
+          options[:key],
+          options[:prefix]
+        )
+        credentials = s3_client.config.credentials.credentials # resolves
+
+        if @caching
+          cached_credentials_for(target, options[:permission], credentials)
+        else
+          new_credentials_for(target, options[:permission], credentials)
+        end
+      rescue Aws::S3Control::Errors::AccessDenied
+        raise unless @fallback
+
+        warn 'Access denied for S3 Access Grants. Falling back to ' \
+             'configured credentials.'
+        s3_client.config.credentials
+      end
+
+      attr_accessor :s3_client
+
+      private
+
+      def s3_control_client(bucket_region)
+        @s3_control_clients[bucket_region] ||= begin
+          credentials = s3_client.config.credentials
+          config = { credentials: credentials }.merge(@s3_control_options)
+          Aws::S3Control::Client.new(config.merge(
+            region: bucket_region,
+            use_fips_endpoint: s3_client.config.use_fips_endpoint,
+            use_dualstack_endpoint: s3_client.config.use_dualstack_endpoint
+          ))
+        end
+      end
+
+      def cached_credentials_for(target, permission, credentials)
+        cached_creds = broad_search_credentials_cache_prefix(target, permission, credentials)
+        return cached_creds if cached_creds
+
+        if %w[READ WRITE].include?(permission)
+          cached_creds = broad_search_credentials_cache_prefix(target, 'READWRITE', credentials)
+          return cached_creds if cached_creds
+        end
+
+        cached_creds = broad_search_credentials_cache_characters(target, permission, credentials)
+        return cached_creds if cached_creds
+
+        if %w[READ WRITE].include?(permission)
+          cached_creds = broad_search_credentials_cache_characters(target, 'READWRITE', credentials)
+          return cached_creds if cached_creds
+        end
+
+        creds = new_credentials_for(target, permission, credentials)
+        if creds.matched_grant_target.end_with?('*')
+          # remove /* from the end of the target
+          key = credentials_cache_key(creds.matched_grant_target[0...-2], permission, credentials)
+          @credentials_cache[key] = creds
+        end
+
+        creds
+      end
+
+      def broad_search_credentials_cache_prefix(target, permission, credentials)
+        prefix = target
+        while prefix != 's3:'
+          key = credentials_cache_key(prefix, permission, credentials)
+          return @credentials_cache[key] if @credentials_cache.key?(key)
+
+          prefix = prefix.split('/', -1)[0..-2].join('/')
+        end
+        nil
+      end
+
+      def broad_search_credentials_cache_characters(target, permission, credentials)
+        prefix = target
+        while prefix != 's3://'
+          key = credentials_cache_key("#{prefix}*", permission, credentials)
+          return @credentials_cache[key] if @credentials_cache.key?(key)
+
+          prefix = prefix[0..-2]
+        end
+        nil
+      end
+
+      def new_credentials_for(target, permission, credentials)
+        bucket_region = bucket_region_for_access_grants(target)
+        client = s3_control_client(bucket_region)
+
+        AccessGrantsCredentials.new(
+          target: target,
+          account_id: account_id_for_access_grants(target, credentials),
+          permission: permission,
+          client: client
+        )
+      end
+
+      def account_id_for_access_grants(target, credentials)
+        if @caching
+          cached_account_id_for(target, credentials)
+        else
+          new_account_id_for(target, credentials)
+        end
+      end
+
+      def cached_account_id_for(target, credentials)
+        bucket = bucket_name_from(target)
+
+        if @account_id_cache.key?(bucket)
+          @account_id_cache[bucket]
+        else
+          @account_id_cache[bucket] = new_account_id_for(target, credentials)
+        end
+      end
+
+      # returns the account id associated with the access grants instance
+      def new_account_id_for(target, credentials)
+        bucket_region = bucket_region_for_access_grants(target)
+        s3_control_client = s3_control_client(bucket_region)
+        resp = s3_control_client.get_access_grants_instance_for_prefix(
+          s3_prefix: target,
+          account_id: account_id_for_credentials(bucket_region, credentials)
+        )
+        ARNParser.parse(resp.access_grants_instance_arn).account_id
+      end
+
+      def bucket_region_for_access_grants(target)
+        bucket = bucket_name_from(target)
+        # regardless of caching option, bucket region cache is always shared
+        cached_bucket_region_for(bucket)
+      end
+
+      def cached_bucket_region_for(bucket)
+        if @bucket_region_cache.key?(bucket)
+          @bucket_region_cache[bucket]
+        else
+          @bucket_region_cache[bucket] = new_bucket_region_for(bucket)
+        end
+      end
+
+      def new_bucket_region_for(bucket)
+        @s3_client.head_bucket(bucket: bucket).bucket_region
+      rescue Aws::S3::Errors::Http301Error => e
+        e.data.region
+      end
+
+      # returns the account id for the configured credentials
+      def account_id_for_credentials(region, credentials)
+        # use resolved credentials to check for account id
+        if credentials.respond_to?(:account_id) && credentials.account_id &&
+           !credentials.account_id.empty?
+          credentials.account_id
+        else
+          @sts_client ||= Aws::STS::Client.new(
+            credentials: s3_client.config.credentials,
+            region: region,
+            use_fips_endpoint: s3_client.config.use_fips_endpoint,
+            use_dualstack_endpoint: s3_client.config.use_dualstack_endpoint
+          )
+          @sts_client.get_caller_identity.account
+        end
+      end
+
+      def target_prefix(bucket, key, prefix)
+        if key && !key.empty?
+          "s3://#{bucket}/#{key}"
+        elsif prefix && !prefix.empty?
+          "s3://#{bucket}/#{prefix}"
+        else
+          "s3://#{bucket}/*"
+        end
+      end
+
+      def credentials_cache_key(target, permission, credentials)
+        "#{credentials.access_key_id}-#{credentials.secret_access_key}" \
+        "-#{permission}-#{target}"
+      end
+
+      # extracts bucket name from target prefix
+      def bucket_name_from(target)
+        URI(target).host
+      end
+    end
+  end
+end