aws-sdk-resources 2.11.549

data/lib/aws-sdk-resources/services/s3/encryption/key_provider.rb

@@ -0,0 +1,29 @@
+module Aws
+  module S3
+    module Encryption
+
+      # This module defines the interface required for a {Client#key_provider}.
+      # A key provider is any object that:
+      #
+      # * Responds to {#encryption_materials} with an {Materials} object.
+      #
+      # * Responds to {#key_for}, receiving a JSON document String,
+      #   returning an encryption key. The returned encryption key
+      #   must be one of:
+      #
+      #   * `OpenSSL::PKey::RSA` - for asymmetric encryption
+      #   * `String` - 32, 24, or 16 bytes long, for symmetric encryption
+      #
+      module KeyProvider
+
+        # @return [Materials]
+        def encryption_materials; end
+
+        # @param [String<JSON>] materials_description
+        # @return [OpenSSL::PKey::RSA, String] encryption_key
+        def key_for(materials_description); end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/encryption/kms_cipher_provider.rb

@@ -0,0 +1,71 @@
+require 'base64'
+
+module Aws
+  module S3
+    module Encryption
+      # @api private
+      class KmsCipherProvider
+
+        def initialize(options = {})
+          @kms_key_id = options[:kms_key_id]
+          @kms_client = options[:kms_client]
+        end
+
+        # @return [Array<Hash,Cipher>] Creates an returns a new encryption
+        #   envelope and encryption cipher.
+        def encryption_cipher
+          encryption_context = { "kms_cmk_id" => @kms_key_id }
+          key_data = @kms_client.generate_data_key(
+            key_id: @kms_key_id,
+            encryption_context: encryption_context,
+            key_spec: 'AES_256',
+          )
+          cipher = Utils.aes_encryption_cipher(:CBC)
+          cipher.key = key_data.plaintext
+          envelope = {
+            'x-amz-key-v2' => encode64(key_data.ciphertext_blob),
+            'x-amz-iv' => encode64(cipher.iv = cipher.random_iv),
+            'x-amz-cek-alg' => 'AES/CBC/PKCS5Padding',
+            'x-amz-wrap-alg' => 'kms',
+            'x-amz-matdesc' => Json.dump(encryption_context)
+          }
+          [envelope, cipher]
+        end
+
+        # @return [Cipher] Given an encryption envelope, returns a
+        #   decryption cipher.
+        def decryption_cipher(envelope)
+          encryption_context = Json.load(envelope['x-amz-matdesc'])
+          key = @kms_client.decrypt(
+            ciphertext_blob: decode64(envelope['x-amz-key-v2']),
+            encryption_context: encryption_context,
+          ).plaintext
+          iv = decode64(envelope['x-amz-iv'])
+          block_mode =
+            case envelope['x-amz-cek-alg']
+            when 'AES/CBC/PKCS5Padding'
+              :CBC
+            when 'AES/GCM/NoPadding'
+              :GCM
+            else
+              type = envelope['x-amz-cek-alg'].inspect
+              msg = "unsupported content encrypting key (cek) format: #{type}"
+              raise Errors::DecryptionError, msg
+            end
+          Utils.aes_decryption_cipher(block_mode, key, iv)
+        end
+
+        private
+
+        def encode64(str)
+          Base64.encode64(str).split("\n") * ""
+        end
+
+        def decode64(str)
+          Base64.decode64(str)
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/encryption/materials.rb

@@ -0,0 +1,58 @@
+require 'base64'
+
+module Aws
+  module S3
+    module Encryption
+      class Materials
+
+        # @option options [required, OpenSSL::PKey::RSA, String] :key
+        #   The master key to use for encrypting/decrypting all objects.
+        #
+        # @option options [String<JSON>] :description ('{}')
+        #   The encryption materials description. This is must be
+        #   a JSON document string.
+        #
+        def initialize(options = {})
+          @key = validate_key(options[:key])
+          @description = validate_desc(options[:description])
+        end
+
+        # @return [OpenSSL::PKey::RSA, String]
+        attr_reader :key
+
+        # @return [String<JSON>]
+        attr_reader :description
+
+        private
+
+        def validate_key(key)
+          case key
+          when OpenSSL::PKey::RSA then key
+          when String
+            if [32, 24, 16].include?(key.bytesize)
+              key
+            else
+              msg = "invalid key, symmetric key required to be 16, 24, or "
+              msg << "32 bytes in length, saw length " + key.bytesize.to_s
+              raise ArgumentError, msg
+            end
+          else
+            msg = "invalid encryption key, expected an OpenSSL::PKey::RSA key "
+            msg << "(for asymmetric encryption) or a String (for symmetric "
+            msg << "encryption)."
+            raise ArgumentError, msg
+          end
+        end
+
+        def validate_desc(description)
+          Json.load(description)
+          description
+        rescue Json::ParseError, EncodingError
+          msg = "expected description to be a valid JSON document string"
+          raise ArgumentError, msg
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/encryption/utils.rb

@@ -0,0 +1,79 @@
+require 'openssl'
+
+module Aws
+  module S3
+    module Encryption
+      # @api private
+      module Utils
+
+        UNSAFE_MSG = "unsafe encryption, data is longer than key length"
+
+        class << self
+
+          def encrypt(key, data)
+            case key
+            when OpenSSL::PKey::RSA # asymmetric encryption
+              warn(UNSAFE_MSG) if key.public_key.n.num_bits < cipher_size(data)
+              key.public_encrypt(data)
+            when String # symmetric encryption
+              warn(UNSAFE_MSG) if cipher_size(key) < cipher_size(data)
+              cipher = aes_encryption_cipher(:ECB, key)
+              cipher.update(data) + cipher.final
+            end
+          end
+
+          def decrypt(key, data)
+            begin
+              case key
+              when OpenSSL::PKey::RSA # asymmetric decryption
+                key.private_decrypt(data)
+              when String # symmetric Decryption
+                cipher = aes_cipher(:decrypt, :ECB, key, nil)
+                cipher.update(data) + cipher.final
+              end
+            rescue OpenSSL::Cipher::CipherError
+              msg = 'decryption failed, possible incorrect key'
+              raise Errors::DecryptionError, msg
+            end
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_encryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:encrypt, block_mode, key, iv)
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_decryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:decrypt, block_mode, key, iv)
+          end
+
+          # @param [String] mode "encrypt" or "decrypt"
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_cipher(mode, block_mode, key, iv)
+            cipher = key ?
+              OpenSSL::Cipher.new("aes-#{cipher_size(key)}-#{block_mode.downcase}") :
+              OpenSSL::Cipher.new("aes-256-#{block_mode.downcase}")
+            cipher.send(mode) # encrypt or decrypt
+            cipher.key = key if key
+            cipher.iv = iv if iv
+            cipher
+          end
+
+          # @param [String] key
+          # @return [Integer]
+          # @raise ArgumentError
+          def cipher_size(key)
+            key.bytesize * 8
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/file_downloader.rb

@@ -0,0 +1,169 @@
+require 'pathname'
+require 'thread'
+require 'set'
+require 'tmpdir'
+
+module Aws
+  module S3
+    # @api private
+    class FileDownloader
+
+      MIN_CHUNK_SIZE = 5 * 1024 * 1024
+      MAX_PARTS = 10_000
+      THREAD_COUNT = 10
+
+      def initialize(options = {})
+        @client = options[:client] || Client.new
+      end
+      
+      # @return [Client]
+      attr_reader :client
+
+      def download(destination, options = {})
+        @path = destination
+        @mode = options[:mode] || "auto"
+        @thread_count = options[:thread_count] || THREAD_COUNT
+        @chunk_size = options[:chunk_size]
+        @bucket = options[:bucket]
+        @key = options[:key]
+
+        case @mode
+        when "auto" then multipart_download
+        when "single_request" then single_request
+        when "get_range"
+          if @chunk_size
+            resp = @client.head_object(bucket: @bucket, key: @key)
+            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+          else
+            msg = "In :get_range mode, :chunk_size must be provided"
+            raise ArgumentError, msg
+          end
+        else
+          msg = "Invalid mode #{@mode} provided, "\
+            "mode should be :single_request, :get_range or :auto"
+          raise ArgumentError, msg
+        end
+      end
+      
+      private
+
+      def multipart_download
+        resp = @client.head_object(bucket: @bucket, key: @key, part_number: 1)
+        count = resp.parts_count
+        if count.nil? || count <= 1
+          resp.content_length < MIN_CHUNK_SIZE ?
+            single_request :
+            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+        else
+          # partNumber is an option
+          resp = @client.head_object(bucket: @bucket, key: @key)
+          resp.content_length < MIN_CHUNK_SIZE ?
+            single_request :
+            compute_mode(resp.content_length, count)
+        end
+      end
+ 
+      def compute_mode(file_size, count)
+        chunk_size = compute_chunk(file_size)
+        part_size = (file_size.to_f / count.to_f).ceil
+        if chunk_size < part_size
+          multithreaded_get_by_ranges(construct_chunks(file_size))
+        else
+          multithreaded_get_by_parts(count)
+        end
+      end
+
+      def construct_chunks(file_size)
+        offset = 0
+        default_chunk_size = compute_chunk(file_size)
+        chunks = []
+        while offset <= file_size
+          progress = offset + default_chunk_size
+          chunks << "bytes=#{offset}-#{progress < file_size ? progress : file_size}"
+          offset = progress + 1
+        end
+        chunks
+      end
+
+      def compute_chunk(file_size)
+        if @chunk_size && @chunk_size > file_size
+          raise ArgumentError, ":chunk_size shouldn't exceed total file size."
+        else
+          default_chunk_size = @chunk_size || [(file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE].max.to_i
+        end
+      end
+
+      def sort_files(files)
+        # sort file by start range count or part number
+        files.sort do |a, b|
+          a[/([^\=]+)$/].split('-')[0].to_i <=> b[/([^\=]+)$/].split('-')[0].to_i
+        end
+      end
+
+      def concatenate_parts(fileparts)
+        File.open(@path, 'wb')do |output_path|
+          sort_files(fileparts).each {|part| IO.copy_stream(part, output_path)}
+        end
+      end
+
+      def file_batches(chunks, dir, mode)
+        batches = []
+        chunks = (1..chunks) if mode.eql? 'part_number'
+        chunks.each_slice(@thread_count) do |slice|
+          batches << map_files(slice, dir, mode)
+        end
+        batches
+      end
+
+      def map_files(slice, dir, mode)
+        case mode
+        when 'range'
+          slice.inject({}) {|h, chunk| h[chunk] = File.join(dir, chunk); h}
+        when 'part_number'
+          slice.inject({}) {|h, part| h[part] = File.join(dir, "part_number=#{part}"); h}
+        end
+      end
+
+      def multithreaded_get_by_ranges(chunks)
+        thread_batches(chunks, 'range')
+      end
+
+      def multithreaded_get_by_parts(parts)
+        thread_batches(parts, 'part_number')
+      end
+
+      def thread_batches(chunks, param)
+        # create a tmp dir under destination dir for batches
+        dir = Dir.mktmpdir(nil, File.dirname(@path))
+        batches = file_batches(chunks, dir, param)
+        parts = batches.flat_map(&:values)
+        begin
+          batches.each do |batch|
+            threads = []
+            batch.each do |chunk, file|
+              threads << Thread.new do
+                resp = @client.get_object(
+                  :bucket => @bucket,
+                  :key => @key,
+                  param.to_sym => chunk,
+                  :response_target => file
+                )
+              end
+            end
+            threads.each(&:join)
+          end
+          concatenate_parts(parts)
+        ensure
+          # clean up tmp dir
+          FileUtils.remove_entry(dir)
+        end
+      end
+
+      def single_request
+        @client.get_object(
+          bucket: @bucket, key: @key, response_target: @path
+        )
+      end
+    end
+  end
+end

data/lib/aws-sdk-resources/services/s3/file_part.rb

@@ -0,0 +1,75 @@
+module Aws
+  module S3
+
+    # A utility class that provides an IO-like interface to a portion of
+    # a file on disk.
+    # @api private
+    class FilePart
+
+      # @option options [required,String,Pathname,File,Tempfile] :source
+      # @option options [required,Integer] :offset The file part will read
+      #   starting at this byte offset.
+      # @option options [required,Integer] :size The maximum number of bytes to
+      #   read from the `:offset`.
+      def initialize(options = {})
+        @source = options[:source]
+        @first_byte = options[:offset]
+        @last_byte = @first_byte + options[:size]
+        @size = options[:size]
+        @file = nil
+      end
+
+      # @return [String,Pathname,File,Tempfile]
+      attr_reader :source
+
+      # @return [Integer]
+      attr_reader :first_byte
+
+      # @return [Integer]
+      attr_reader :last_byte
+
+      # @return [Integer]
+      attr_reader :size
+
+      def read(bytes = nil, output_buffer = nil)
+        open_file unless @file
+        read_from_file(bytes, output_buffer)
+      end
+
+      def rewind
+        if @file
+          @file.seek(@first_byte)
+          @position = @first_byte
+        end
+        0
+      end
+
+      def close
+        @file.close if @file
+      end
+
+      private
+
+      def open_file
+        @file = File.open(@source, 'rb')
+        rewind
+      end
+
+      def read_from_file(bytes, output_buffer)
+        if bytes
+          data = @file.read([remaining_bytes, bytes].min)
+          data = nil if data == ''
+        else
+          data = @file.read(remaining_bytes)
+        end
+        @position += data ? data.bytesize : 0
+        output_buffer ? output_buffer.replace(data || '') : data
+      end
+
+      def remaining_bytes
+        @last_byte - @position
+      end
+
+    end
+  end
+end