aws-sdk-resources 2.11.632 → 3.0.0.rc1

data/lib/aws-sdk-resources/services/s3/encryptionV2/utils.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-require 'openssl'
-
-module Aws
-  module S3
-    module EncryptionV2
-      # @api private
-      module Utils
-
-        class << self
-
-          def encrypt_aes_gcm(key, data, auth_data)
-            cipher = aes_encryption_cipher(:GCM, key)
-            cipher.iv = (iv = cipher.random_iv)
-            cipher.auth_data = auth_data
-
-            iv + cipher.update(data) + cipher.final + cipher.auth_tag
-          end
-
-          def encrypt_rsa(key, data, auth_data)
-            # Plaintext must be KeyLengthInBytes (1 Byte) + DataKey + AuthData
-            buf = [data.bytesize] + data.unpack('C*') + auth_data.unpack('C*')
-            key.public_encrypt(buf.pack('C*'), OpenSSL::PKey::RSA::PKCS1_OAEP_PADDING)
-          end
-
-          def decrypt(key, data)
-            begin
-              case key
-              when OpenSSL::PKey::RSA # asymmetric decryption
-                key.private_decrypt(data)
-              when String # symmetric Decryption
-                cipher = aes_cipher(:decrypt, :ECB, key, nil)
-                cipher.update(data) + cipher.final
-              end
-            rescue OpenSSL::Cipher::CipherError
-              msg = 'decryption failed, possible incorrect key'
-              raise Errors::DecryptionError, msg
-            end
-          end
-
-          def decrypt_aes_gcm(key, data, auth_data)
-            # data is iv (12B) + key + tag (16B)
-            buf = data.unpack('C*')
-            iv = buf[0,12].pack('C*') # iv will always be 12 bytes
-            tag = buf[-16, 16].pack('C*') # tag is 16 bytes
-            enc_key = buf[12, buf.size - (12+16)].pack('C*')
-            cipher = aes_cipher(:decrypt, :GCM, key, iv)
-            cipher.auth_tag = tag
-            cipher.auth_data = auth_data
-            cipher.update(enc_key) + cipher.final
-          end
-
-          # returns the decrypted data + auth_data
-          def decrypt_rsa(key, enc_data)
-            # Plaintext must be KeyLengthInBytes (1 Byte) + DataKey + AuthData
-            buf = key.private_decrypt(enc_data, OpenSSL::PKey::RSA::PKCS1_OAEP_PADDING).unpack('C*')
-            key_length = buf[0]
-            data = buf[1, key_length].pack('C*')
-            auth_data = buf[key_length+1, buf.length - key_length].pack('C*')
-            [data, auth_data]
-          end
-
-          # @param [String] block_mode "CBC" or "ECB"
-          # @param [OpenSSL::PKey::RSA, String, nil] key
-          # @param [String, nil] iv The initialization vector
-          def aes_encryption_cipher(block_mode, key = nil, iv = nil)
-            aes_cipher(:encrypt, block_mode, key, iv)
-          end
-
-          # @param [String] block_mode "CBC" or "ECB"
-          # @param [OpenSSL::PKey::RSA, String, nil] key
-          # @param [String, nil] iv The initialization vector
-          def aes_decryption_cipher(block_mode, key = nil, iv = nil)
-            aes_cipher(:decrypt, block_mode, key, iv)
-          end
-
-          # @param [String] mode "encrypt" or "decrypt"
-          # @param [String] block_mode "CBC" or "ECB"
-          # @param [OpenSSL::PKey::RSA, String, nil] key
-          # @param [String, nil] iv The initialization vector
-          def aes_cipher(mode, block_mode, key, iv)
-            cipher = key ?
-              OpenSSL::Cipher.new("aes-#{cipher_size(key)}-#{block_mode.downcase}") :
-              OpenSSL::Cipher.new("aes-256-#{block_mode.downcase}")
-            cipher.send(mode) # encrypt or decrypt
-            cipher.key = key if key
-            cipher.iv = iv if iv
-            cipher
-          end
-
-          # @param [String] key
-          # @return [Integer]
-          # @raise ArgumentError
-          def cipher_size(key)
-            key.bytesize * 8
-          end
-
-        end
-      end
-    end
-  end
-end

data/lib/aws-sdk-resources/services/s3/encryption_v2.rb

@@ -1,24 +0,0 @@
-module Aws
-  module S3
-    module EncryptionV2
-
-      AES_GCM_TAG_LEN_BYTES = 16
-      EC_USER_AGENT = 'S3CryptoV2'
-
-      autoload :Client, 'aws-sdk-resources/services/s3/encryptionV2/client'
-      autoload :DecryptHandler, 'aws-sdk-resources/services/s3/encryptionV2/decrypt_handler'
-      autoload :DefaultCipherProvider, 'aws-sdk-resources/services/s3/encryptionV2/default_cipher_provider'
-      autoload :DefaultKeyProvider, 'aws-sdk-resources/services/s3/encryptionV2/default_key_provider'
-      autoload :EncryptHandler, 'aws-sdk-resources/services/s3/encryptionV2/encrypt_handler'
-      autoload :Errors, 'aws-sdk-resources/services/s3/encryptionV2/errors'
-      autoload :IOEncrypter, 'aws-sdk-resources/services/s3/encryptionV2/io_encrypter'
-      autoload :IOAuthDecrypter, 'aws-sdk-resources/services/s3/encryptionV2/io_auth_decrypter'
-      autoload :IODecrypter, 'aws-sdk-resources/services/s3/encryptionV2/io_decrypter'
-      autoload :KeyProvider, 'aws-sdk-resources/services/s3/encryptionV2/key_provider'
-      autoload :KmsCipherProvider, 'aws-sdk-resources/services/s3/encryptionV2/kms_cipher_provider'
-      autoload :Materials, 'aws-sdk-resources/services/s3/encryptionV2/materials'
-      autoload :Utils, 'aws-sdk-resources/services/s3/encryptionV2/utils'
-
-    end
-  end
-end

data/lib/aws-sdk-resources/services/s3/file_downloader.rb

@@ -1,169 +0,0 @@
-require 'pathname'
-require 'thread'
-require 'set'
-require 'tmpdir'
-
-module Aws
-  module S3
-    # @api private
-    class FileDownloader
-
-      MIN_CHUNK_SIZE = 5 * 1024 * 1024
-      MAX_PARTS = 10_000
-      THREAD_COUNT = 10
-
-      def initialize(options = {})
-        @client = options[:client] || Client.new
-      end
-      
-      # @return [Client]
-      attr_reader :client
-
-      def download(destination, options = {})
-        @path = destination
-        @mode = options[:mode] || "auto"
-        @thread_count = options[:thread_count] || THREAD_COUNT
-        @chunk_size = options[:chunk_size]
-        @bucket = options[:bucket]
-        @key = options[:key]
-
-        case @mode
-        when "auto" then multipart_download
-        when "single_request" then single_request
-        when "get_range"
-          if @chunk_size
-            resp = @client.head_object(bucket: @bucket, key: @key)
-            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
-          else
-            msg = "In :get_range mode, :chunk_size must be provided"
-            raise ArgumentError, msg
-          end
-        else
-          msg = "Invalid mode #{@mode} provided, "\
-            "mode should be :single_request, :get_range or :auto"
-          raise ArgumentError, msg
-        end
-      end
-      
-      private
-
-      def multipart_download
-        resp = @client.head_object(bucket: @bucket, key: @key, part_number: 1)
-        count = resp.parts_count
-        if count.nil? || count <= 1
-          resp.content_length < MIN_CHUNK_SIZE ?
-            single_request :
-            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
-        else
-          # partNumber is an option
-          resp = @client.head_object(bucket: @bucket, key: @key)
-          resp.content_length < MIN_CHUNK_SIZE ?
-            single_request :
-            compute_mode(resp.content_length, count)
-        end
-      end
- 
-      def compute_mode(file_size, count)
-        chunk_size = compute_chunk(file_size)
-        part_size = (file_size.to_f / count.to_f).ceil
-        if chunk_size < part_size
-          multithreaded_get_by_ranges(construct_chunks(file_size))
-        else
-          multithreaded_get_by_parts(count)
-        end
-      end
-
-      def construct_chunks(file_size)
-        offset = 0
-        default_chunk_size = compute_chunk(file_size)
-        chunks = []
-        while offset <= file_size
-          progress = offset + default_chunk_size
-          chunks << "bytes=#{offset}-#{progress < file_size ? progress : file_size}"
-          offset = progress + 1
-        end
-        chunks
-      end
-
-      def compute_chunk(file_size)
-        if @chunk_size && @chunk_size > file_size
-          raise ArgumentError, ":chunk_size shouldn't exceed total file size."
-        else
-          default_chunk_size = @chunk_size || [(file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE].max.to_i
-        end
-      end
-
-      def sort_files(files)
-        # sort file by start range count or part number
-        files.sort do |a, b|
-          a[/([^\=]+)$/].split('-')[0].to_i <=> b[/([^\=]+)$/].split('-')[0].to_i
-        end
-      end
-
-      def concatenate_parts(fileparts)
-        File.open(@path, 'wb')do |output_path|
-          sort_files(fileparts).each {|part| IO.copy_stream(part, output_path)}
-        end
-      end
-
-      def file_batches(chunks, dir, mode)
-        batches = []
-        chunks = (1..chunks) if mode.eql? 'part_number'
-        chunks.each_slice(@thread_count) do |slice|
-          batches << map_files(slice, dir, mode)
-        end
-        batches
-      end
-
-      def map_files(slice, dir, mode)
-        case mode
-        when 'range'
-          slice.inject({}) {|h, chunk| h[chunk] = File.join(dir, chunk); h}
-        when 'part_number'
-          slice.inject({}) {|h, part| h[part] = File.join(dir, "part_number=#{part}"); h}
-        end
-      end
-
-      def multithreaded_get_by_ranges(chunks)
-        thread_batches(chunks, 'range')
-      end
-
-      def multithreaded_get_by_parts(parts)
-        thread_batches(parts, 'part_number')
-      end
-
-      def thread_batches(chunks, param)
-        # create a tmp dir under destination dir for batches
-        dir = Dir.mktmpdir(nil, File.dirname(@path))
-        batches = file_batches(chunks, dir, param)
-        parts = batches.flat_map(&:values)
-        begin
-          batches.each do |batch|
-            threads = []
-            batch.each do |chunk, file|
-              threads << Thread.new do
-                resp = @client.get_object(
-                  :bucket => @bucket,
-                  :key => @key,
-                  param.to_sym => chunk,
-                  :response_target => file
-                )
-              end
-            end
-            threads.each(&:join)
-          end
-          concatenate_parts(parts)
-        ensure
-          # clean up tmp dir
-          FileUtils.remove_entry(dir)
-        end
-      end
-
-      def single_request
-        @client.get_object(
-          bucket: @bucket, key: @key, response_target: @path
-        )
-      end
-    end
-  end
-end

data/lib/aws-sdk-resources/services/s3/file_part.rb

@@ -1,75 +0,0 @@
-module Aws
-  module S3
-
-    # A utility class that provides an IO-like interface to a portion of
-    # a file on disk.
-    # @api private
-    class FilePart
-
-      # @option options [required,String,Pathname,File,Tempfile] :source
-      # @option options [required,Integer] :offset The file part will read
-      #   starting at this byte offset.
-      # @option options [required,Integer] :size The maximum number of bytes to
-      #   read from the `:offset`.
-      def initialize(options = {})
-        @source = options[:source]
-        @first_byte = options[:offset]
-        @last_byte = @first_byte + options[:size]
-        @size = options[:size]
-        @file = nil
-      end
-
-      # @return [String,Pathname,File,Tempfile]
-      attr_reader :source
-
-      # @return [Integer]
-      attr_reader :first_byte
-
-      # @return [Integer]
-      attr_reader :last_byte
-
-      # @return [Integer]
-      attr_reader :size
-
-      def read(bytes = nil, output_buffer = nil)
-        open_file unless @file
-        read_from_file(bytes, output_buffer)
-      end
-
-      def rewind
-        if @file
-          @file.seek(@first_byte)
-          @position = @first_byte
-        end
-        0
-      end
-
-      def close
-        @file.close if @file
-      end
-
-      private
-
-      def open_file
-        @file = File.open(@source, 'rb')
-        rewind
-      end
-
-      def read_from_file(bytes, output_buffer)
-        if bytes
-          data = @file.read([remaining_bytes, bytes].min)
-          data = nil if data == ''
-        else
-          data = @file.read(remaining_bytes)
-        end
-        @position += data ? data.bytesize : 0
-        output_buffer ? output_buffer.replace(data || '') : data
-      end
-
-      def remaining_bytes
-        @last_byte - @position
-      end
-
-    end
-  end
-end

data/lib/aws-sdk-resources/services/s3/file_uploader.rb

@@ -1,58 +0,0 @@
-require 'pathname'
-
-module Aws
-  module S3
-    # @api private
-    class FileUploader
-
-      FIFTEEN_MEGABYTES = 15 * 1024 * 1024
-
-      # @option options [Client] :client
-      # @option options [Integer] :multipart_threshold Files greater than
-      #   `:multipart_threshold` bytes are uploaded using S3 multipart APIs.
-      def initialize(options = {})
-        @options = options
-        @client = options[:client] || Client.new
-        @multipart_threshold = options[:multipart_threshold] || FIFTEEN_MEGABYTES
-      end
-
-      # @return [Client]
-      attr_reader :client
-
-      # @return [Integer] Files larger than this in bytes are uploaded
-      #   using a {MultipartFileUploader}.
-      attr_reader :multipart_threshold
-
-      # @param [String,Pathname,File,Tempfile] source
-      # @option options [required,String] :bucket
-      # @option options [required,String] :key
-      # @return [void]
-      def upload(source, options = {})
-        if File.size(source) >= multipart_threshold
-          MultipartFileUploader.new(@options).upload(source, options)
-        else
-          put_object(source, options)
-        end
-      end
-
-      private
-
-      def put_object(source, options)
-        open_file(source) do |file|
-          @client.put_object(options.merge(body: file))
-        end
-      end
-
-      def open_file(source)
-        if String === source || Pathname === source
-          file = File.open(source, 'rb')
-          yield(file)
-          file.close
-        else
-          yield(source)
-        end
-      end
-
-    end
-  end
-end

data/lib/aws-sdk-resources/services/s3/multipart_file_uploader.rb

@@ -1,187 +0,0 @@
-require 'pathname'
-require 'thread'
-require 'set'
-
-module Aws
-  module S3
-    # @api private
-    class MultipartFileUploader
-
-      MIN_PART_SIZE = 5 * 1024 * 1024 # 5MB
-
-      FILE_TOO_SMALL = "unable to multipart upload files smaller than 5MB"
-
-      MAX_PARTS = 10_000
-
-      THREAD_COUNT = 10
-
-      # @api private
-      CREATE_OPTIONS =
-        Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names)
-
-      # @api private
-      UPLOAD_PART_OPTIONS =
-        Set.new(Client.api.operation(:upload_part).input.shape.member_names)
-
-      # @option options [Client] :client
-      def initialize(options = {})
-        @client = options[:client] || Client.new
-        @thread_count = options[:thread_count] || THREAD_COUNT
-      end
-
-      # @return [Client]
-      attr_reader :client
-
-      # @param [String,Pathname,File,Tempfile] source
-      # @option options [required,String] :bucket
-      # @option options [required,String] :key
-      # @return [void]
-      def upload(source, options = {})
-        if File.size(source) < MIN_PART_SIZE
-          raise ArgumentError, FILE_TOO_SMALL
-        else
-          upload_id = initiate_upload(options)
-          parts = upload_parts(upload_id, source, options)
-          complete_upload(upload_id, parts, options)
-        end
-      end
-
-      private
-
-      def initiate_upload(options)
-        @client.create_multipart_upload(create_opts(options)).upload_id
-      end
-
-      def complete_upload(upload_id, parts, options)
-        @client.complete_multipart_upload(
-          bucket: options[:bucket],
-          key: options[:key],
-          upload_id: upload_id,
-          multipart_upload: { parts: parts })
-      end
-
-      def upload_parts(upload_id, source, options)
-        pending = PartList.new(compute_parts(upload_id, source, options))
-        completed = PartList.new
-        errors = upload_in_threads(pending, completed)
-        if errors.empty?
-          completed.to_a.sort_by { |part| part[:part_number] }
-        else
-          abort_upload(upload_id, options, errors)
-        end
-      end
-
-      def abort_upload(upload_id, options, errors)
-        @client.abort_multipart_upload(
-          bucket: options[:bucket],
-          key: options[:key],
-          upload_id: upload_id
-        )
-        msg = "multipart upload failed: #{errors.map(&:message).join("; ")}"
-        raise MultipartUploadError.new(msg, errors)
-      rescue MultipartUploadError => error
-        raise error
-      rescue => error
-        msg = "failed to abort multipart upload: #{error.message}"
-        raise MultipartUploadError.new(msg, errors + [error])
-      end
-
-      def compute_parts(upload_id, source, options)
-        size = File.size(source)
-        default_part_size = compute_default_part_size(size)
-        offset = 0
-        part_number = 1
-        parts = []
-        while offset < size
-          parts << upload_part_opts(options).merge({
-            upload_id: upload_id,
-            part_number: part_number,
-            body: FilePart.new(
-              source: source,
-              offset: offset,
-              size: part_size(size, default_part_size, offset)
-            )
-          })
-          part_number += 1
-          offset += default_part_size
-        end
-        parts
-      end
-
-      def create_opts(options)
-        CREATE_OPTIONS.inject({}) do |hash, key|
-          hash[key] = options[key] if options.key?(key)
-          hash
-        end
-      end
-
-      def upload_part_opts(options)
-        UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
-          hash[key] = options[key] if options.key?(key)
-          hash
-        end
-      end
-
-      def upload_in_threads(pending, completed)
-        threads = []
-        @thread_count.times do
-          thread = Thread.new do
-            begin
-              while part = pending.shift
-                resp = @client.upload_part(part)
-                part[:body].close
-                completed.push(etag: resp.etag, part_number: part[:part_number])
-              end
-              nil
-            rescue => error
-              # keep other threads from uploading other parts
-              pending.clear!
-              error
-            end
-          end
-          thread.abort_on_exception = true
-          threads << thread
-        end
-        threads.map(&:value).compact
-      end
-
-      def compute_default_part_size(source_size)
-        [(source_size.to_f / MAX_PARTS).ceil, MIN_PART_SIZE].max.to_i
-      end
-
-      def part_size(total_size, part_size, offset)
-        if offset + part_size > total_size
-          total_size - offset
-        else
-          part_size
-        end
-      end
-
-      # @api private
-      class PartList
-
-        def initialize(parts = [])
-          @parts = parts
-          @mutex = Mutex.new
-        end
-
-        def push(part)
-          @mutex.synchronize { @parts.push(part) }
-        end
-
-        def shift
-          @mutex.synchronize { @parts.shift }
-        end
-
-        def clear!
-          @mutex.synchronize { @parts.clear }
-        end
-
-        def to_a
-          @mutex.synchronize { @parts.dup }
-        end
-
-      end
-    end
-  end
-end