aws-sdk-s3 1.61.2 → 1.83.1

data/lib/aws-sdk-s3/encryptionV2/utils.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'openssl'
+
+module Aws
+  module S3
+    module EncryptionV2
+      # @api private
+      module Utils
+
+        class << self
+
+          def encrypt_aes_gcm(key, data, auth_data)
+            cipher = aes_encryption_cipher(:GCM, key)
+            cipher.iv = (iv = cipher.random_iv)
+            cipher.auth_data = auth_data
+
+            iv + cipher.update(data) + cipher.final + cipher.auth_tag
+          end
+
+          def encrypt_rsa(key, data, auth_data)
+            # Plaintext must be KeyLengthInBytes (1 Byte) + DataKey + AuthData
+            buf = [data.bytesize] + data.unpack('C*') + auth_data.unpack('C*')
+            key.public_encrypt(buf.pack('C*'), OpenSSL::PKey::RSA::PKCS1_OAEP_PADDING)
+          end
+
+          def decrypt(key, data)
+            begin
+              case key
+              when OpenSSL::PKey::RSA # asymmetric decryption
+                key.private_decrypt(data)
+              when String # symmetric Decryption
+                cipher = aes_cipher(:decrypt, :ECB, key, nil)
+                cipher.update(data) + cipher.final
+              end
+            rescue OpenSSL::Cipher::CipherError
+              msg = 'decryption failed, possible incorrect key'
+              raise Errors::DecryptionError, msg
+            end
+          end
+
+          def decrypt_aes_gcm(key, data, auth_data)
+            # data is iv (12B) + key + tag (16B)
+            buf = data.unpack('C*')
+            iv = buf[0,12].pack('C*') # iv will always be 12 bytes
+            tag = buf[-16, 16].pack('C*') # tag is 16 bytes
+            enc_key = buf[12, buf.size - (12+16)].pack('C*')
+            cipher = aes_cipher(:decrypt, :GCM, key, iv)
+            cipher.auth_tag = tag
+            cipher.auth_data = auth_data
+            cipher.update(enc_key) + cipher.final
+          end
+
+          # returns the decrypted data + auth_data
+          def decrypt_rsa(key, enc_data)
+            # Plaintext must be KeyLengthInBytes (1 Byte) + DataKey + AuthData
+            buf = key.private_decrypt(enc_data, OpenSSL::PKey::RSA::PKCS1_OAEP_PADDING).unpack('C*')
+            key_length = buf[0]
+            data = buf[1, key_length].pack('C*')
+            auth_data = buf[key_length+1, buf.length - key_length].pack('C*')
+            [data, auth_data]
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_encryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:encrypt, block_mode, key, iv)
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_decryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:decrypt, block_mode, key, iv)
+          end
+
+          # @param [String] mode "encrypt" or "decrypt"
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_cipher(mode, block_mode, key, iv)
+            cipher = key ?
+              OpenSSL::Cipher.new("aes-#{cipher_size(key)}-#{block_mode.downcase}") :
+              OpenSSL::Cipher.new("aes-256-#{block_mode.downcase}")
+            cipher.send(mode) # encrypt or decrypt
+            cipher.key = key if key
+            cipher.iv = iv if iv
+            cipher
+          end
+
+          # @param [String] key
+          # @return [Integer]
+          # @raise ArgumentError
+          def cipher_size(key)
+            key.bytesize * 8
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryption_v2.rb

@@ -0,0 +1,23 @@
+require 'aws-sdk-s3/encryptionV2/client'
+require 'aws-sdk-s3/encryptionV2/decrypt_handler'
+require 'aws-sdk-s3/encryptionV2/default_cipher_provider'
+require 'aws-sdk-s3/encryptionV2/encrypt_handler'
+require 'aws-sdk-s3/encryptionV2/errors'
+require 'aws-sdk-s3/encryptionV2/io_encrypter'
+require 'aws-sdk-s3/encryptionV2/io_decrypter'
+require 'aws-sdk-s3/encryptionV2/io_auth_decrypter'
+require 'aws-sdk-s3/encryptionV2/key_provider'
+require 'aws-sdk-s3/encryptionV2/kms_cipher_provider'
+require 'aws-sdk-s3/encryptionV2/materials'
+require 'aws-sdk-s3/encryptionV2/utils'
+require 'aws-sdk-s3/encryptionV2/default_key_provider'
+
+module Aws
+  module S3
+    module EncryptionV2
+      AES_GCM_TAG_LEN_BYTES = 16
+      EC_USER_AGENT = 'S3CryptoV2'
+    end
+  end
+end
+

data/lib/aws-sdk-s3/errors.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # WARNING ABOUT GENERATED CODE
 #
 # This file is generated. See the contributing guide for more information:

data/lib/aws-sdk-s3/event_streams.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # WARNING ABOUT GENERATED CODE
 #
 # This file is generated. See the contributing guide for more information:
@@ -41,6 +43,10 @@ module Aws::S3
         @event_emitter.on(:initial_response, block) if block_given?
       end
 
+      def on_unknown_event(&block)
+        @event_emitter.on(:unknown_event, block) if block_given?
+      end
+
       def on_event(&block)
         on_records_event(&block)
         on_stats_event(&block)
@@ -49,6 +55,7 @@ module Aws::S3
         on_end_event(&block)
         on_error_event(&block)
         on_initial_response_event(&block)
+        on_unknown_event(&block)
       end
 
       # @api private

data/lib/aws-sdk-s3/file_downloader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathname'
 require 'thread'
 require 'set'
@@ -21,7 +23,7 @@ module Aws
 
       def download(destination, options = {})
         @path = destination
-        @mode = options[:mode] || "auto"
+        @mode = options[:mode] || 'auto'
         @thread_count = options[:thread_count] || THREAD_COUNT
         @chunk_size = options[:chunk_size]
         @params = {
@@ -31,19 +33,19 @@ module Aws
         @params[:version_id] = options[:version_id] if options[:version_id]
 
         case @mode
-        when "auto" then multipart_download
-        when "single_request" then single_request
-        when "get_range"
+        when 'auto' then multipart_download
+        when 'single_request' then single_request
+        when 'get_range'
           if @chunk_size
             resp = @client.head_object(@params)
             multithreaded_get_by_ranges(construct_chunks(resp.content_length))
           else
-            msg = "In :get_range mode, :chunk_size must be provided"
+            msg = 'In :get_range mode, :chunk_size must be provided'
             raise ArgumentError, msg
           end
         else
           msg = "Invalid mode #{@mode} provided, "\
-            "mode should be :single_request, :get_range or :auto"
+                'mode should be :single_request, :get_range or :auto'
           raise ArgumentError, msg
         end
       end
@@ -125,8 +127,8 @@ module Aws
       end
 
       def write(resp)
-        range, _ = resp.content_range.split(" ").last.split("/")
-        head, _ = range.split("-").map {|s| s.to_i}
+        range, _ = resp.content_range.split(' ').last.split('/')
+        head, _ = range.split('-').map {|s| s.to_i}
         IO.write(@path, resp.body.read, head)
       end
 

data/lib/aws-sdk-s3/file_part.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
 

data/lib/aws-sdk-s3/file_uploader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathname'
 
 module Aws
@@ -20,13 +22,16 @@ module Aws
       # @return [Client]
       attr_reader :client
 
-      # @return [Integer] Files larger than this in bytes are uploaded
+      # @return [Integer] Files larger than or equal to this in bytes are uploaded
       #   using a {MultipartFileUploader}.
       attr_reader :multipart_threshold
 
       # @param [String, Pathname, File, Tempfile] source The file to upload.
       # @option options [required, String] :bucket The bucket to upload to.
       # @option options [required, String] :key The key for the object.
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
       # @return [void]
       def upload(source, options = {})
         if File.size(source) >= multipart_threshold
@@ -47,11 +52,19 @@ module Aws
       end
 
       def put_object(source, options)
+        if (callback = options.delete(:progress_callback))
+          options[:on_chunk_sent] = single_part_progress(callback)
+        end
         open_file(source) do |file|
           @client.put_object(options.merge(body: file))
         end
       end
 
+      def single_part_progress(progress_callback)
+        proc do |_chunk, bytes_read, total_size|
+          progress_callback.call([bytes_read], [total_size])
+        end
+      end
     end
   end
 end

data/lib/aws-sdk-s3/legacy_signer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'set'
 require 'time'
 require 'openssl'

data/lib/aws-sdk-s3/multipart_file_uploader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'pathname'
 require 'set'
 
@@ -37,6 +39,9 @@ module Aws
       # @param [String, Pathname, File, Tempfile] source The file to upload.
       # @option options [required, String] :bucket The bucket to upload to.
       # @option options [required, String] :key The key for the object.
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the upload is sent.
+      #   It will be invoked with [bytes_read], [total_sizes]
       # @return [void]
       def upload(source, options = {})
         if File.size(source) < MIN_PART_SIZE
@@ -66,7 +71,7 @@ module Aws
       def upload_parts(upload_id, source, options)
         pending = PartList.new(compute_parts(upload_id, source, options))
         completed = PartList.new
-        errors = upload_in_threads(pending, completed)
+        errors = upload_in_threads(pending, completed, options)
         if errors.empty?
           completed.to_a.sort_by { |part| part[:part_number] }
         else
@@ -125,12 +130,21 @@ module Aws
         end
       end
 
-      def upload_in_threads(pending, completed)
+      def upload_in_threads(pending, completed, options)
         threads = []
+        if (callback = options[:progress_callback])
+          progress = MultipartProgress.new(pending, callback)
+        end
         @thread_count.times do
           thread = Thread.new do
             begin
               while part = pending.shift
+                if progress
+                  part[:on_chunk_sent] =
+                    proc do |_chunk, bytes, _total|
+                      progress.call(part[:part_number], bytes)
+                    end
+                end
                 resp = @client.upload_part(part)
                 part[:body].close
                 completed.push(etag: resp.etag, part_number: part[:part_number])
@@ -180,11 +194,34 @@ module Aws
           @mutex.synchronize { @parts.clear }
         end
 
+        def size
+          @mutex.synchronize { @parts.size }
+        end
+
+        def part_sizes
+          @mutex.synchronize { @parts.map { |p| p[:body].size } }
+        end
+
         def to_a
           @mutex.synchronize { @parts.dup }
         end
 
       end
+
+      # @api private
+      class MultipartProgress
+        def initialize(parts, progress_callback)
+          @bytes_sent = Array.new(parts.size, 0)
+          @total_sizes = parts.part_sizes
+          @progress_callback = progress_callback
+        end
+
+        def call(part_number, bytes_read)
+          # part numbers start at 1
+          @bytes_sent[part_number - 1] = bytes_read
+          @progress_callback.call(@bytes_sent, @total_sizes)
+        end
+      end
     end
   end
 end

data/lib/aws-sdk-s3/multipart_stream_uploader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'thread'
 require 'set'
 require 'tempfile'
@@ -113,7 +115,7 @@ module Aws
 
       def read_to_part_body(read_pipe)
         return if read_pipe.closed?
-        temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new
+        temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new(String.new)
         temp_io.binmode
         bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size)
         temp_io.rewind

data/lib/aws-sdk-s3/multipart_upload.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # WARNING ABOUT GENERATED CODE
 #
 # This file is generated. See the contributing guide for more information:
@@ -218,6 +220,7 @@ module Aws::S3
     #
     #   multipart_upload.abort({
     #     request_payer: "requester", # accepts requester
+    #     expected_bucket_owner: "AccountId",
     #   })
     # @param [Hash] options ({})
     # @option options [String] :request_payer
@@ -230,6 +233,10 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    # @option options [String] :expected_bucket_owner
+    #   The account id of the expected bucket owner. If the bucket is owned by
+    #   a different account, the request will fail with an HTTP `403 (Access
+    #   Denied)` error.
     # @return [Types::AbortMultipartUploadOutput]
     def abort(options = {})
       options = options.merge(
@@ -253,6 +260,7 @@ module Aws::S3
     #       ],
     #     },
     #     request_payer: "requester", # accepts requester
+    #     expected_bucket_owner: "AccountId",
     #   })
     # @param [Hash] options ({})
     # @option options [Types::CompletedMultipartUpload] :multipart_upload
@@ -267,6 +275,10 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    # @option options [String] :expected_bucket_owner
+    #   The account id of the expected bucket owner. If the bucket is owned by
+    #   a different account, the request will fail with an HTTP `403 (Access
+    #   Denied)` error.
     # @return [Object]
     def complete(options = {})
       options = options.merge(
@@ -274,7 +286,7 @@ module Aws::S3
         key: @object_key,
         upload_id: @id
       )
-      resp = @client.complete_multipart_upload(options)
+      @client.complete_multipart_upload(options)
       Object.new(
         bucket_name: @bucket_name,
         key: @object_key,
@@ -309,6 +321,7 @@ module Aws::S3
     #
     #   parts = multipart_upload.parts({
     #     request_payer: "requester", # accepts requester
+    #     expected_bucket_owner: "AccountId",
     #   })
     # @param [Hash] options ({})
     # @option options [String] :request_payer
@@ -321,6 +334,10 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    # @option options [String] :expected_bucket_owner
+    #   The account id of the expected bucket owner. If the bucket is owned by
+    #   a different account, the request will fail with an HTTP `403 (Access
+    #   Denied)` error.
     # @return [MultipartUploadPart::Collection]
     def parts(options = {})
       batches = Enumerator.new do |y|

data/lib/aws-sdk-s3/multipart_upload_error.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Aws
   module S3
     class MultipartUploadError < StandardError

data/lib/aws-sdk-s3/multipart_upload_part.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # WARNING ABOUT GENERATED CODE
 #
 # This file is generated. See the contributing guide for more information:
@@ -219,11 +221,55 @@ module Aws::S3
     #     copy_source_sse_customer_key: "CopySourceSSECustomerKey",
     #     copy_source_sse_customer_key_md5: "CopySourceSSECustomerKeyMD5",
     #     request_payer: "requester", # accepts requester
+    #     expected_bucket_owner: "AccountId",
+    #     expected_source_bucket_owner: "AccountId",
     #   })
     # @param [Hash] options ({})
     # @option options [required, String] :copy_source
-    #   The name of the source bucket and key name of the source object,
-    #   separated by a slash (/). Must be URL-encoded.
+    #   Specifies the source object for the copy operation. You specify the
+    #   value in one of two formats, depending on whether you want to access
+    #   the source object through an [access point][1]\:
+    #
+    #   * For objects not accessed through an access point, specify the name
+    #     of the source bucket and key of the source object, separated by a
+    #     slash (/). For example, to copy the object `reports/january.pdf`
+    #     from the bucket `awsexamplebucket`, use
+    #     `awsexamplebucket/reports/january.pdf`. The value must be URL
+    #     encoded.
+    #
+    #   * For objects accessed through access points, specify the Amazon
+    #     Resource Name (ARN) of the object as accessed through the access
+    #     point, in the format
+    #     `arn:aws:s3:<Region>:<account-id>:accesspoint/<access-point-name>/object/<key>`.
+    #     For example, to copy the object `reports/january.pdf` through access
+    #     point `my-access-point` owned by account `123456789012` in Region
+    #     `us-west-2`, use the URL encoding of
+    #     `arn:aws:s3:us-west-2:123456789012:accesspoint/my-access-point/object/reports/january.pdf`.
+    #     The value must be URL encoded.
+    #
+    #     <note markdown="1"> Amazon S3 supports copy operations using access points only when the
+    #     source and destination buckets are in the same AWS Region.
+    #
+    #      </note>
+    #
+    #     Alternatively, for objects accessed through Amazon S3 on Outposts,
+    #     specify the ARN of the object as accessed in the format
+    #     `arn:aws:s3-outposts:<Region>:<account-id>:outpost/<outpost-id>/object/<key>`.
+    #     For example, to copy the object `reports/january.pdf` through
+    #     outpost `my-outpost` owned by account `123456789012` in Region
+    #     `us-west-2`, use the URL encoding of
+    #     `arn:aws:s3-outposts:us-west-2:123456789012:outpost/my-outpost/object/reports/january.pdf`.
+    #     The value must be URL encoded.
+    #
+    #   To copy a specific version of an object, append
+    #   `?versionId=<version-id>` to the value (for example,
+    #   `awsexamplebucket/reports/january.pdf?versionId=QUpfdndhfd8438MNFDN93jdnJFkdmqnh893`).
+    #   If you don't specify a version ID, Amazon S3 copies the latest
+    #   version of the source object.
+    #
+    #
+    #
+    #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/access-points.html
     # @option options [String] :copy_source_if_match
     #   Copies the object if its entity tag (ETag) matches the specified tag.
     # @option options [Time,DateTime,Date,Integer,String] :copy_source_if_modified_since
@@ -248,8 +294,8 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm` header. This must
-    #   be the same encryption key specified in the initiate multipart upload
+    #   `x-amz-server-side-encryption-customer-algorithm` header. This must be
+    #   the same encryption key specified in the initiate multipart upload
     #   request.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
@@ -276,6 +322,14 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    # @option options [String] :expected_bucket_owner
+    #   The account id of the expected destination bucket owner. If the
+    #   destination bucket is owned by a different account, the request will
+    #   fail with an HTTP `403 (Access Denied)` error.
+    # @option options [String] :expected_source_bucket_owner
+    #   The account id of the expected source bucket owner. If the source
+    #   bucket is owned by a different account, the request will fail with an
+    #   HTTP `403 (Access Denied)` error.
     # @return [Types::UploadPartCopyOutput]
     def copy_from(options = {})
       options = options.merge(
@@ -298,9 +352,10 @@ module Aws::S3
     #     sse_customer_key: "SSECustomerKey",
     #     sse_customer_key_md5: "SSECustomerKeyMD5",
     #     request_payer: "requester", # accepts requester
+    #     expected_bucket_owner: "AccountId",
     #   })
     # @param [Hash] options ({})
-    # @option options [String, IO] :body
+    # @option options [String, StringIO, File] :body
     #   Object data.
     # @option options [Integer] :content_length
     #   Size of the body in bytes. This parameter is useful when the size of
@@ -317,8 +372,8 @@ module Aws::S3
     #   encrypting data. This value is used to store the object and then it is
     #   discarded; Amazon S3 does not store the encryption key. The key must
     #   be appropriate for use with the algorithm specified in the
-    #   `x-amz-server-side​-encryption​-customer-algorithm header`. This must
-    #   be the same encryption key specified in the initiate multipart upload
+    #   `x-amz-server-side-encryption-customer-algorithm header`. This must be
+    #   the same encryption key specified in the initiate multipart upload
     #   request.
     # @option options [String] :sse_customer_key_md5
     #   Specifies the 128-bit MD5 digest of the encryption key according to
@@ -334,6 +389,10 @@ module Aws::S3
     #
     #
     #   [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
+    # @option options [String] :expected_bucket_owner
+    #   The account id of the expected bucket owner. If the bucket is owned by
+    #   a different account, the request will fail with an HTTP `403 (Access
+    #   Denied)` error.
     # @return [Types::UploadPartOutput]
     def upload(options = {})
       options = options.merge(