aws-sdk-resources 2.8.4 → 2.11.632

data/lib/aws-sdk-resources/services/s3/encryptionV2/key_provider.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Aws
+  module S3
+    module EncryptionV2
+
+      # This module defines the interface required for a {Client#key_provider}.
+      # A key provider is any object that:
+      #
+      # * Responds to {#encryption_materials} with an {Materials} object.
+      #
+      # * Responds to {#key_for}, receiving a JSON document String,
+      #   returning an encryption key. The returned encryption key
+      #   must be one of:
+      #
+      #   * `OpenSSL::PKey::RSA` - for asymmetric encryption
+      #   * `String` - 32, 24, or 16 bytes long, for symmetric encryption
+      #
+      module KeyProvider
+
+        # @return [Materials]
+        def encryption_materials; end
+
+        # @param [String<JSON>] materials_description
+        # @return [OpenSSL::PKey::RSA, String] encryption_key
+        def key_for(materials_description); end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/encryptionV2/kms_cipher_provider.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Aws
+  module S3
+    module EncryptionV2
+      # @api private
+      class KmsCipherProvider
+
+        def initialize(options = {})
+          @kms_key_id = validate_kms_key(options[:kms_key_id])
+          @kms_client = options[:kms_client]
+          @key_wrap_schema = validate_key_wrap(
+            options[:key_wrap_schema]
+          )
+          @content_encryption_schema = validate_cek(
+            options[:content_encryption_schema]
+          )
+        end
+
+        # @return [Array<Hash,Cipher>] Creates and returns a new encryption
+        #   envelope and encryption cipher.
+        def encryption_cipher(options = {})
+          validate_key_for_encryption
+          encryption_context = build_encryption_context(@content_encryption_schema, options)
+          key_data = @kms_client.generate_data_key(
+            key_id: @kms_key_id,
+            encryption_context: encryption_context,
+            key_spec: 'AES_256'
+          )
+          cipher = Utils.aes_encryption_cipher(:GCM)
+          cipher.key = key_data.plaintext
+          envelope = {
+            'x-amz-key-v2' => encode64(key_data.ciphertext_blob),
+            'x-amz-iv' => encode64(cipher.iv = cipher.random_iv),
+            'x-amz-cek-alg' => @content_encryption_schema,
+            'x-amz-tag-len' => (AES_GCM_TAG_LEN_BYTES * 8).to_s,
+            'x-amz-wrap-alg' => @key_wrap_schema,
+            'x-amz-matdesc' => Json.dump(encryption_context)
+          }
+          cipher.auth_data = '' # auth_data must be set after key and iv
+          [envelope, cipher]
+        end
+
+        # @return [Cipher] Given an encryption envelope, returns a
+        #   decryption cipher.
+        def decryption_cipher(envelope, options = {})
+          encryption_context = Json.load(envelope['x-amz-matdesc'])
+          cek_alg = envelope['x-amz-cek-alg']
+
+          case envelope['x-amz-wrap-alg']
+          when 'kms'
+            unless options[:security_profile] == :v2_and_legacy
+              raise Errors::LegacyDecryptionError
+            end
+          when 'kms+context'
+            if cek_alg != encryption_context['aws:x-amz-cek-alg']
+              raise Errors::CEKAlgMismatchError
+            end
+
+            if encryption_context != build_encryption_context(cek_alg, options)
+              raise Errors::DecryptionError, 'Value of encryption context from'\
+                ' envelope does not match the provided encryption context'
+            end
+          when 'AES/GCM'
+            raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a KMS key and the x-amz-wrap-alg is AES/GCM.'
+          when 'RSA-OAEP-SHA1'
+            raise ArgumentError, 'Key mismatch - Client is configured' \
+                    ' with a KMS key and the x-amz-wrap-alg is RSA-OAEP-SHA1.'
+          else
+            raise ArgumentError, 'Unsupported wrap-alg: ' \
+                "#{envelope['x-amz-wrap-alg']}"
+          end
+
+          any_cmk_mode = false || options[:kms_allow_decrypt_with_any_cmk]
+          decrypt_options = {
+            ciphertext_blob: decode64(envelope['x-amz-key-v2']),
+            encryption_context: encryption_context
+          }
+          unless any_cmk_mode
+            decrypt_options[:key_id] = @kms_key_id
+          end
+
+          key = @kms_client.decrypt(decrypt_options).plaintext
+          iv = decode64(envelope['x-amz-iv'])
+          block_mode =
+            case cek_alg
+            when 'AES/CBC/PKCS5Padding'
+              :CBC
+            when 'AES/CBC/PKCS7Padding'
+              :CBC
+            when 'AES/GCM/NoPadding'
+              :GCM
+            else
+              type = envelope['x-amz-cek-alg'].inspect
+              msg = "unsupported content encrypting key (cek) format: #{type}"
+              raise Errors::DecryptionError, msg
+            end
+          Utils.aes_decryption_cipher(block_mode, key, iv)
+        end
+
+        private
+
+        def validate_key_wrap(key_wrap_schema)
+          case key_wrap_schema
+          when :kms_context then 'kms+context'
+          else
+            raise ArgumentError, "Unsupported key_wrap_schema: #{key_wrap_schema}"
+          end
+        end
+
+        def validate_cek(content_encryption_schema)
+          case content_encryption_schema
+          when :aes_gcm_no_padding
+            "AES/GCM/NoPadding"
+          else
+            raise ArgumentError, "Unsupported content_encryption_schema: #{content_encryption_schema}"
+          end
+        end
+
+        def validate_kms_key(kms_key_id)
+          if kms_key_id.nil? || kms_key_id.length.zero?
+            raise ArgumentError, 'KMS CMK ID was not specified. ' \
+              'Please specify a CMK ID, ' \
+              'or set kms_key_id: :kms_allow_decrypt_with_any_cmk to use ' \
+              'any valid CMK from the object.'
+          end
+
+          if kms_key_id.is_a?(Symbol) && kms_key_id != :kms_allow_decrypt_with_any_cmk
+            raise ArgumentError, 'kms_key_id must be a valid KMS CMK or be ' \
+              'set to :kms_allow_decrypt_with_any_cmk'
+          end
+          kms_key_id
+        end
+
+        def build_encryption_context(cek_alg, options = {})
+          kms_context = (options[:kms_encryption_context] || {})
+            .each_with_object({}) { |(k, v), h| h[k.to_s] = v }
+          if kms_context.include? 'aws:x-amz-cek-alg'
+            raise ArgumentError, 'Conflict in reserved KMS Encryption Context ' \
+              'key aws:x-amz-cek-alg. This value is reserved for the S3 ' \
+              'Encryption Client and cannot be set by the user.'
+          end
+          {
+            'aws:x-amz-cek-alg' => cek_alg
+          }.merge(kms_context)
+        end
+
+        def encode64(str)
+          Base64.encode64(str).split("\n") * ""
+        end
+
+        def decode64(str)
+          Base64.decode64(str)
+        end
+
+        def validate_key_for_encryption
+          if @kms_key_id == :kms_allow_decrypt_with_any_cmk
+            raise ArgumentError, 'Unable to encrypt/write objects with '\
+              'kms_key_id = :kms_allow_decrypt_with_any_cmk.  Provide ' \
+              'a valid kms_key_id on client construction.'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/encryptionV2/materials.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module Aws
+  module S3
+    module EncryptionV2
+      class Materials
+
+        # @option options [required, OpenSSL::PKey::RSA, String] :key
+        #   The master key to use for encrypting/decrypting all objects.
+        #
+        # @option options [String<JSON>] :description ('{}')
+        #   The encryption materials description. This is must be
+        #   a JSON document string.
+        #
+        def initialize(options = {})
+          @key = validate_key(options[:key])
+          @description = validate_desc(options[:description])
+        end
+
+        # @return [OpenSSL::PKey::RSA, String]
+        attr_reader :key
+
+        # @return [String<JSON>]
+        attr_reader :description
+
+        private
+
+        def validate_key(key)
+          case key
+          when OpenSSL::PKey::RSA then key
+          when String
+            if [32, 24, 16].include?(key.bytesize)
+              key
+            else
+              msg = 'invalid key, symmetric key required to be 16, 24, or '\
+                    '32 bytes in length, saw length ' + key.bytesize.to_s
+              raise ArgumentError, msg
+            end
+          else
+            msg = 'invalid encryption key, expected an OpenSSL::PKey::RSA key '\
+                  '(for asymmetric encryption) or a String (for symmetric '\
+                  'encryption).'
+            raise ArgumentError, msg
+          end
+        end
+
+        def validate_desc(description)
+          Json.load(description)
+          description
+        rescue Json::ParseError, EncodingError
+          msg = 'expected description to be a valid JSON document string'
+          raise ArgumentError, msg
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/encryptionV2/utils.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'openssl'
+
+module Aws
+  module S3
+    module EncryptionV2
+      # @api private
+      module Utils
+
+        class << self
+
+          def encrypt_aes_gcm(key, data, auth_data)
+            cipher = aes_encryption_cipher(:GCM, key)
+            cipher.iv = (iv = cipher.random_iv)
+            cipher.auth_data = auth_data
+
+            iv + cipher.update(data) + cipher.final + cipher.auth_tag
+          end
+
+          def encrypt_rsa(key, data, auth_data)
+            # Plaintext must be KeyLengthInBytes (1 Byte) + DataKey + AuthData
+            buf = [data.bytesize] + data.unpack('C*') + auth_data.unpack('C*')
+            key.public_encrypt(buf.pack('C*'), OpenSSL::PKey::RSA::PKCS1_OAEP_PADDING)
+          end
+
+          def decrypt(key, data)
+            begin
+              case key
+              when OpenSSL::PKey::RSA # asymmetric decryption
+                key.private_decrypt(data)
+              when String # symmetric Decryption
+                cipher = aes_cipher(:decrypt, :ECB, key, nil)
+                cipher.update(data) + cipher.final
+              end
+            rescue OpenSSL::Cipher::CipherError
+              msg = 'decryption failed, possible incorrect key'
+              raise Errors::DecryptionError, msg
+            end
+          end
+
+          def decrypt_aes_gcm(key, data, auth_data)
+            # data is iv (12B) + key + tag (16B)
+            buf = data.unpack('C*')
+            iv = buf[0,12].pack('C*') # iv will always be 12 bytes
+            tag = buf[-16, 16].pack('C*') # tag is 16 bytes
+            enc_key = buf[12, buf.size - (12+16)].pack('C*')
+            cipher = aes_cipher(:decrypt, :GCM, key, iv)
+            cipher.auth_tag = tag
+            cipher.auth_data = auth_data
+            cipher.update(enc_key) + cipher.final
+          end
+
+          # returns the decrypted data + auth_data
+          def decrypt_rsa(key, enc_data)
+            # Plaintext must be KeyLengthInBytes (1 Byte) + DataKey + AuthData
+            buf = key.private_decrypt(enc_data, OpenSSL::PKey::RSA::PKCS1_OAEP_PADDING).unpack('C*')
+            key_length = buf[0]
+            data = buf[1, key_length].pack('C*')
+            auth_data = buf[key_length+1, buf.length - key_length].pack('C*')
+            [data, auth_data]
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_encryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:encrypt, block_mode, key, iv)
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_decryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:decrypt, block_mode, key, iv)
+          end
+
+          # @param [String] mode "encrypt" or "decrypt"
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_cipher(mode, block_mode, key, iv)
+            cipher = key ?
+              OpenSSL::Cipher.new("aes-#{cipher_size(key)}-#{block_mode.downcase}") :
+              OpenSSL::Cipher.new("aes-256-#{block_mode.downcase}")
+            cipher.send(mode) # encrypt or decrypt
+            cipher.key = key if key
+            cipher.iv = iv if iv
+            cipher
+          end
+
+          # @param [String] key
+          # @return [Integer]
+          # @raise ArgumentError
+          def cipher_size(key)
+            key.bytesize * 8
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/encryption_v2.rb

@@ -0,0 +1,24 @@
+module Aws
+  module S3
+    module EncryptionV2
+
+      AES_GCM_TAG_LEN_BYTES = 16
+      EC_USER_AGENT = 'S3CryptoV2'
+
+      autoload :Client, 'aws-sdk-resources/services/s3/encryptionV2/client'
+      autoload :DecryptHandler, 'aws-sdk-resources/services/s3/encryptionV2/decrypt_handler'
+      autoload :DefaultCipherProvider, 'aws-sdk-resources/services/s3/encryptionV2/default_cipher_provider'
+      autoload :DefaultKeyProvider, 'aws-sdk-resources/services/s3/encryptionV2/default_key_provider'
+      autoload :EncryptHandler, 'aws-sdk-resources/services/s3/encryptionV2/encrypt_handler'
+      autoload :Errors, 'aws-sdk-resources/services/s3/encryptionV2/errors'
+      autoload :IOEncrypter, 'aws-sdk-resources/services/s3/encryptionV2/io_encrypter'
+      autoload :IOAuthDecrypter, 'aws-sdk-resources/services/s3/encryptionV2/io_auth_decrypter'
+      autoload :IODecrypter, 'aws-sdk-resources/services/s3/encryptionV2/io_decrypter'
+      autoload :KeyProvider, 'aws-sdk-resources/services/s3/encryptionV2/key_provider'
+      autoload :KmsCipherProvider, 'aws-sdk-resources/services/s3/encryptionV2/kms_cipher_provider'
+      autoload :Materials, 'aws-sdk-resources/services/s3/encryptionV2/materials'
+      autoload :Utils, 'aws-sdk-resources/services/s3/encryptionV2/utils'
+
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/file_downloader.rb

@@ -0,0 +1,169 @@
+require 'pathname'
+require 'thread'
+require 'set'
+require 'tmpdir'
+
+module Aws
+  module S3
+    # @api private
+    class FileDownloader
+
+      MIN_CHUNK_SIZE = 5 * 1024 * 1024
+      MAX_PARTS = 10_000
+      THREAD_COUNT = 10
+
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+      end
+      
+      # @return [Client]
+      attr_reader :client
+
+      def download(destination, options = {})
+        @path = destination
+        @mode = options[:mode] || "auto"
+        @thread_count = options[:thread_count] || THREAD_COUNT
+        @chunk_size = options[:chunk_size]
+        @bucket = options[:bucket]
+        @key = options[:key]
+
+        case @mode
+        when "auto" then multipart_download
+        when "single_request" then single_request
+        when "get_range"
+          if @chunk_size
+            resp = @client.head_object(bucket: @bucket, key: @key)
+            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+          else
+            msg = "In :get_range mode, :chunk_size must be provided"
+            raise ArgumentError, msg
+          end
+        else
+          msg = "Invalid mode #{@mode} provided, "\
+            "mode should be :single_request, :get_range or :auto"
+          raise ArgumentError, msg
+        end
+      end
+      
+      private
+
+      def multipart_download
+        resp = @client.head_object(bucket: @bucket, key: @key, part_number: 1)
+        count = resp.parts_count
+        if count.nil? || count <= 1
+          resp.content_length < MIN_CHUNK_SIZE ?
+            single_request :
+            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+        else
+          # partNumber is an option
+          resp = @client.head_object(bucket: @bucket, key: @key)
+          resp.content_length < MIN_CHUNK_SIZE ?
+            single_request :
+            compute_mode(resp.content_length, count)
+        end
+      end
+ 
+      def compute_mode(file_size, count)
+        chunk_size = compute_chunk(file_size)
+        part_size = (file_size.to_f / count.to_f).ceil
+        if chunk_size < part_size
+          multithreaded_get_by_ranges(construct_chunks(file_size))
+        else
+          multithreaded_get_by_parts(count)
+        end
+      end
+
+      def construct_chunks(file_size)
+        offset = 0
+        default_chunk_size = compute_chunk(file_size)
+        chunks = []
+        while offset <= file_size
+          progress = offset + default_chunk_size
+          chunks << "bytes=#{offset}-#{progress < file_size ? progress : file_size}"
+          offset = progress + 1
+        end
+        chunks
+      end
+
+      def compute_chunk(file_size)
+        if @chunk_size && @chunk_size > file_size
+          raise ArgumentError, ":chunk_size shouldn't exceed total file size."
+        else
+          default_chunk_size = @chunk_size || [(file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE].max.to_i
+        end
+      end
+
+      def sort_files(files)
+        # sort file by start range count or part number
+        files.sort do |a, b|
+          a[/([^\=]+)$/].split('-')[0].to_i <=> b[/([^\=]+)$/].split('-')[0].to_i
+        end
+      end
+
+      def concatenate_parts(fileparts)
+        File.open(@path, 'wb')do |output_path|
+          sort_files(fileparts).each {|part| IO.copy_stream(part, output_path)}
+        end
+      end
+
+      def file_batches(chunks, dir, mode)
+        batches = []
+        chunks = (1..chunks) if mode.eql? 'part_number'
+        chunks.each_slice(@thread_count) do |slice|
+          batches << map_files(slice, dir, mode)
+        end
+        batches
+      end
+
+      def map_files(slice, dir, mode)
+        case mode
+        when 'range'
+          slice.inject({}) {|h, chunk| h[chunk] = File.join(dir, chunk); h}
+        when 'part_number'
+          slice.inject({}) {|h, part| h[part] = File.join(dir, "part_number=#{part}"); h}
+        end
+      end
+
+      def multithreaded_get_by_ranges(chunks)
+        thread_batches(chunks, 'range')
+      end
+
+      def multithreaded_get_by_parts(parts)
+        thread_batches(parts, 'part_number')
+      end
+
+      def thread_batches(chunks, param)
+        # create a tmp dir under destination dir for batches
+        dir = Dir.mktmpdir(nil, File.dirname(@path))
+        batches = file_batches(chunks, dir, param)
+        parts = batches.flat_map(&:values)
+        begin
+          batches.each do |batch|
+            threads = []
+            batch.each do |chunk, file|
+              threads << Thread.new do
+                resp = @client.get_object(
+                  :bucket => @bucket,
+                  :key => @key,
+                  param.to_sym => chunk,
+                  :response_target => file
+                )
+              end
+            end
+            threads.each(&:join)
+          end
+          concatenate_parts(parts)
+        ensure
+          # clean up tmp dir
+          FileUtils.remove_entry(dir)
+        end
+      end
+
+      def single_request
+        @client.get_object(
+          bucket: @bucket, key: @key, response_target: @path
+        )
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/object.rb

@@ -252,7 +252,39 @@ module Aws
         uploader.upload(source, uploading_options.merge(bucket: bucket_name, key: key))
         true
       end
-
+     
+      # Downloads a file in S3 to a path on disk.
+      #
+      #     # small files (< 5MB) are downloaded in a single API call
+      #     obj.download_file('/path/to/file')
+      #
+      # Files larger than 5MB are downloaded using multipart method
+      #
+      #     # large files are split into parts
+      #     # and the parts are downloaded in parallel
+      #     obj.download_file('/path/to/very_large_file')
+      # 
+      # @param [String] destination Where to download the file to
+      #
+      # @option options [String] mode `auto`, `single_request`, `get_range`
+      #  `single_request` mode forces only 1 GET request is made in download,
+      #  `get_range` mode allows `chunk_size` parameter to configured in
+      #  customizing each range size in multipart_download,
+      #  By default, `auto` mode is enabled, which performs multipart_download
+      #
+      # @option options [String] chunk_size required in get_range mode
+      #
+      # @option options [String] thread_count Customize threads used in multipart
+      #   download, if not provided, 10 is default value
+      #
+      # @return [Boolean] Returns `true` when the file is downloaded
+      #   without any errors.
+      def download_file(destination, options = {})
+        downloader = FileDownloader.new(client: client)
+        downloader.download(
+          destination, options.merge(bucket: bucket_name, key: key))
+        true
+      end
     end
   end
 end

data/lib/aws-sdk-resources/services/s3/object_multipart_copier.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'cgi'
 
 module Aws
   module S3

data/lib/aws-sdk-resources/services/s3/object_summary.rb

@@ -60,6 +60,14 @@ module Aws
         object.upload_file(source, options)
       end
 
+      # @param (see Object#download_file)
+      # @options (see Object#download_file)
+      # @return (see Object#download_file)
+      # @see Object#download_file
+      def download_file(destination, options = {})
+        object.download_file(destination, options)
+      end
+
     end
   end
 end

data/lib/aws-sdk-resources/services/s3/presigned_post.rb

@@ -585,6 +585,10 @@ module Aws
         else
           url.path = '/' + @bucket_name
         end
+        if @bucket_region == 'us-east-1'
+          # keep legacy behavior by default
+          url.host = Plugins::S3IADRegionalEndpoint.legacy_host(url.host)
+        end
         url.to_s
       end
 

data/lib/aws-sdk-resources/services/s3.rb

@@ -7,8 +7,10 @@ module Aws
     require 'aws-sdk-resources/services/s3/multipart_upload'
 
     autoload :Encryption, 'aws-sdk-resources/services/s3/encryption'
+    autoload :EncryptionV2, 'aws-sdk-resources/services/s3/encryption_v2'
     autoload :FilePart, 'aws-sdk-resources/services/s3/file_part'
     autoload :FileUploader, 'aws-sdk-resources/services/s3/file_uploader'
+    autoload :FileDownloader, 'aws-sdk-resources/services/s3/file_downloader'
     autoload :MultipartFileUploader, 'aws-sdk-resources/services/s3/multipart_file_uploader'
     autoload :MultipartUploadError, 'aws-sdk-resources/services/s3/multipart_upload_error'
     autoload :ObjectCopier, 'aws-sdk-resources/services/s3/object_copier'

data/lib/aws-sdk-resources/services/sns/message_verifier.rb

@@ -59,6 +59,7 @@ module Aws
       #   verification.
       def authenticate!(message_body)
         msg = Json.load(message_body)
+        msg = convert_lambda_msg(msg) if is_from_lambda(msg)
         if public_key(msg).verify(sha1, signature(msg), canonical_string(msg))
           true
         else
@@ -69,6 +70,19 @@ module Aws
 
       private
 
+      def is_from_lambda(message)
+        message.key? 'SigningCertUrl'
+      end
+
+      def convert_lambda_msg(message)
+        cert_url = message.delete('SigningCertUrl')
+        unsubscribe_url = message.delete('UnsubscribeUrl')
+
+        message['SigningCertURL'] = cert_url
+        message['UnsubscribeURL'] = unsubscribe_url
+        message
+      end
+
       def sha1
         OpenSSL::Digest::SHA1.new
       end

data/lib/aws-sdk-resources/services/sqs/queue_poller.rb

@@ -253,7 +253,7 @@ module Aws
       #
       # @return [void]
       def before_request(&block)
-        @default_config = @default_config.with(before_request: Proc.new)
+        @default_config = @default_config.with(before_request: block) if block_given?
       end
 
       # Polls the queue, yielded a message, or an array of messages.