aws-sdk-s3 1.0.0.rc1

data/lib/aws-sdk-s3/encryption/key_provider.rb

@@ -0,0 +1,29 @@
+module Aws
+  module S3
+    module Encryption
+
+      # This module defines the interface required for a {Client#key_provider}.
+      # A key provider is any object that:
+      #
+      # * Responds to {#encryption_materials} with an {Materials} object.
+      #
+      # * Responds to {#key_for}, receiving a JSON document String,
+      #   returning an encryption key. The returned encryption key
+      #   must be one of:
+      #
+      #   * `OpenSSL::PKey::RSA` - for asymmetric encryption
+      #   * `String` - 32, 24, or 16 bytes long, for symmetric encryption
+      #
+      module KeyProvider
+
+        # @return [Materials]
+        def encryption_materials; end
+
+        # @param [String<JSON>] materials_description
+        # @return [OpenSSL::PKey::RSA, String] encryption_key
+        def key_for(materials_description); end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb

@@ -0,0 +1,71 @@
+require 'base64'
+
+module Aws
+  module S3
+    module Encryption
+      # @api private
+      class KmsCipherProvider
+
+        def initialize(options = {})
+          @kms_key_id = options[:kms_key_id]
+          @kms_client = options[:kms_client]
+        end
+
+        # @return [Array<Hash,Cipher>] Creates an returns a new encryption
+        #   envelope and encryption cipher.
+        def encryption_cipher
+          encryption_context = { "kms_cmk_id" => @kms_key_id }
+          key_data = @kms_client.generate_data_key(
+            key_id: @kms_key_id,
+            encryption_context: encryption_context,
+            key_spec: 'AES_256',
+          )
+          cipher = Utils.aes_encryption_cipher(:CBC)
+          cipher.key = key_data.plaintext
+          envelope = {
+            'x-amz-key-v2' => encode64(key_data.ciphertext_blob),
+            'x-amz-iv' => encode64(cipher.iv = cipher.random_iv),
+            'x-amz-cek-alg' => 'AES/CBC/PKCS5Padding',
+            'x-amz-wrap-alg' => 'kms',
+            'x-amz-matdesc' => Json.dump(encryption_context)
+          }
+          [envelope, cipher]
+        end
+
+        # @return [Cipher] Given an encryption envelope, returns a
+        #   decryption cipher.
+        def decryption_cipher(envelope)
+          encryption_context = Json.load(envelope['x-amz-matdesc'])
+          key = @kms_client.decrypt(
+            ciphertext_blob: decode64(envelope['x-amz-key-v2']),
+            encryption_context: encryption_context,
+          ).plaintext
+          iv = decode64(envelope['x-amz-iv'])
+          block_mode =
+            case envelope['x-amz-cek-alg']
+            when 'AES/CBC/PKCS5Padding'
+              :CBC
+            when 'AES/GCM/NoPadding'
+              :GCM
+            else
+              type = envelope['x-amz-cek-alg'].inspect
+              msg = "unsupported content encrypting key (cek) format: #{type}"
+              raise Errors::DecryptionError, msg
+            end
+          Utils.aes_decryption_cipher(block_mode, key, iv)
+        end
+
+        private
+
+        def encode64(str)
+          Base64.encode64(str).split("\n") * ""
+        end
+
+        def decode64(str)
+          Base64.decode64(str)
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryption/materials.rb

@@ -0,0 +1,58 @@
+require 'base64'
+
+module Aws
+  module S3
+    module Encryption
+      class Materials
+
+        # @option options [required, OpenSSL::PKey::RSA, String] :key
+        #   The master key to use for encrypting/decrypting all objects.
+        #
+        # @option options [String<JSON>] :description ('{}')
+        #   The encryption materials description. This is must be
+        #   a JSON document string.
+        #
+        def initialize(options = {})
+          @key = validate_key(options[:key])
+          @description = validate_desc(options[:description])
+        end
+
+        # @return [OpenSSL::PKey::RSA, String]
+        attr_reader :key
+
+        # @return [String<JSON>]
+        attr_reader :description
+
+        private
+
+        def validate_key(key)
+          case key
+          when OpenSSL::PKey::RSA then key
+          when String
+            if [32, 24, 16].include?(key.bytesize)
+              key
+            else
+              msg = "invalid key, symmetric key required to be 16, 24, or "
+              msg << "32 bytes in length, saw length " + key.bytesize.to_s
+              raise ArgumentError, msg
+            end
+          else
+            msg = "invalid encryption key, expected an OpenSSL::PKey::RSA key "
+            msg << "(for asymmetric encryption) or a String (for symmetric "
+            msg << "encryption)."
+            raise ArgumentError, msg
+          end
+        end
+
+        def validate_desc(description)
+          Json.load(description)
+          description
+        rescue Json::ParseError
+          msg = "expected description to be a valid JSON document string"
+          raise ArgumentError, msg
+        end
+
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/encryption/utils.rb

@@ -0,0 +1,79 @@
+require 'openssl'
+
+module Aws
+  module S3
+    module Encryption
+      # @api private
+      module Utils
+
+        UNSAFE_MSG = "unsafe encryption, data is longer than key length"
+
+        class << self
+
+          def encrypt(key, data)
+            case key
+            when OpenSSL::PKey::RSA # asymmetric encryption
+              warn(UNSAFE_MSG) if key.public_key.n.num_bits < cipher_size(data)
+              key.public_encrypt(data)
+            when String # symmetric encryption
+              warn(UNSAFE_MSG) if cipher_size(key) < cipher_size(data)
+              cipher = aes_encryption_cipher(:ECB, key)
+              cipher.update(data) + cipher.final
+            end
+          end
+
+          def decrypt(key, data)
+            begin
+              case key
+              when OpenSSL::PKey::RSA # asymmetric decryption
+                key.private_decrypt(data)
+              when String # symmetric Decryption
+                cipher = aes_cipher(:decrypt, :ECB, key, nil)
+                cipher.update(data) + cipher.final
+              end
+            rescue OpenSSL::Cipher::CipherError
+              msg = 'decryption failed, possible incorrect key'
+              raise Errors::DecryptionError, msg
+            end
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_encryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:encrypt, block_mode, key, iv)
+          end
+
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_decryption_cipher(block_mode, key = nil, iv = nil)
+            aes_cipher(:decrypt, block_mode, key, iv)
+          end
+
+          # @param [String] mode "encrypt" or "decrypt"
+          # @param [String] block_mode "CBC" or "ECB"
+          # @param [OpenSSL::PKey::RSA, String, nil] key
+          # @param [String, nil] iv The initialization vector
+          def aes_cipher(mode, block_mode, key, iv)
+            cipher = key ?
+              OpenSSL::Cipher.new("aes-#{cipher_size(key)}-#{block_mode.downcase}") :
+              OpenSSL::Cipher.new("aes-256-#{block_mode.downcase}")
+            cipher.send(mode) # encrypt or decrypt
+            cipher.key = key if key
+            cipher.iv = iv if iv
+            cipher
+          end
+
+          # @param [String] key
+          # @return [Integer]
+          # @raise ArgumentError
+          def cipher_size(key)
+            key.bytesize * 8
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-s3/errors.rb

@@ -0,0 +1,23 @@
+# WARNING ABOUT GENERATED CODE
+#
+# This file is generated. See the contributing for info on making contributions:
+# https://github.com/aws/aws-sdk-ruby/blob/master/CONTRIBUTING.md
+#
+# WARNING ABOUT GENERATED CODE
+
+module Aws
+  module S3
+    module Errors
+
+      extend Aws::Errors::DynamicErrors
+
+      # Raised when calling #load or #data on a resource class that can not be
+      # loaded.  This can happen when:
+      #
+      # * A resource class has identifiers, but no data attributes.
+      # * Resource data is only available when making an API call that
+      # enumerates all resources of that type.
+      class ResourceNotLoadable < RuntimeError; end
+    end
+  end
+end

data/lib/aws-sdk-s3/file_part.rb

@@ -0,0 +1,75 @@
+module Aws
+  module S3
+
+    # A utility class that provides an IO-like interface to a portion of
+    # a file on disk.
+    # @api private
+    class FilePart
+
+      # @option options [required,String,Pathname,File,Tempfile] :source
+      # @option options [required,Integer] :offset The file part will read
+      #   starting at this byte offset.
+      # @option options [required,Integer] :size The maximum number of bytes to
+      #   read from the `:offset`.
+      def initialize(options = {})
+        @source = options[:source]
+        @first_byte = options[:offset]
+        @last_byte = @first_byte + options[:size]
+        @size = options[:size]
+        @file = nil
+      end
+
+      # @return [String,Pathname,File,Tempfile]
+      attr_reader :source
+
+      # @return [Integer]
+      attr_reader :first_byte
+
+      # @return [Integer]
+      attr_reader :last_byte
+
+      # @return [Integer]
+      attr_reader :size
+
+      def read(bytes = nil, output_buffer = nil)
+        open_file unless @file
+        read_from_file(bytes, output_buffer)
+      end
+
+      def rewind
+        if @file
+          @file.seek(@first_byte)
+          @position = @first_byte
+        end
+        0
+      end
+
+      def close
+        @file.close if @file
+      end
+
+      private
+
+      def open_file
+        @file = File.open(@source, 'rb')
+        rewind
+      end
+
+      def read_from_file(bytes, output_buffer)
+        if bytes
+          data = @file.read([remaining_bytes, bytes].min)
+          data = nil if data == ''
+        else
+          data = @file.read(remaining_bytes)
+        end
+        @position += data ? data.bytesize : 0
+        output_buffer ? output_buffer.replace(data || '') : data
+      end
+
+      def remaining_bytes
+        @last_byte - @position
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-s3/file_uploader.rb

@@ -0,0 +1,58 @@
+require 'pathname'
+
+module Aws
+  module S3
+    # @api private
+    class FileUploader
+
+      FIFTEEN_MEGABYTES = 15 * 1024 * 1024
+
+      # @option options [Client] :client
+      # @option options [Integer] :multipart_threshold Files greater than
+      #   `:multipart_threshold` bytes are uploaded using S3 multipart APIs.
+      def initialize(options = {})
+        @options = options
+        @client = options[:client] || Client.new
+        @multipart_threshold = options[:multipart_threshold] || FIFTEEN_MEGABYTES
+      end
+
+      # @return [Client]
+      attr_reader :client
+
+      # @return [Integer] Files larger than this in bytes are uploaded
+      #   using a {MultipartFileUploader}.
+      attr_reader :multipart_threshold
+
+      # @param [String,Pathname,File,Tempfile] source
+      # @option options [required,String] :bucket
+      # @option options [required,String] :key
+      # @return [void]
+      def upload(source, options = {})
+        if File.size(source) >= multipart_threshold
+          MultipartFileUploader.new(@options).upload(source, options)
+        else
+          put_object(source, options)
+        end
+      end
+
+      private
+
+      def put_object(source, options)
+        open_file(source) do |file|
+          @client.put_object(options.merge(body: file))
+        end
+      end
+
+      def open_file(source)
+        if String === source || Pathname === source
+          file = File.open(source, 'rb')
+          yield(file)
+          file.close
+        else
+          yield(source)
+        end
+      end
+
+    end
+  end
+end

data/lib/aws-sdk-s3/legacy_signer.rb

@@ -0,0 +1,186 @@
+require 'set'
+require 'time'
+require 'openssl'
+require 'webrick/httputils'
+require 'aws-sdk-core/query'
+
+module Aws
+  module S3
+    # @api private
+    class LegacySigner
+
+      SIGNED_QUERYSTRING_PARAMS = Set.new(%w(
+
+        acl delete cors lifecycle location logging notification partNumber
+        policy requestPayment restore tagging torrent uploadId uploads
+        versionId versioning versions website replication requestPayment
+        accelerate
+
+        response-content-type response-content-language
+        response-expires response-cache-control
+        response-content-disposition response-content-encoding
+
+      ))
+
+      def self.sign(context)
+        new(
+          context.config.credentials,
+          context.params,
+          context.config.force_path_style
+        ).sign(context.http_request)
+      end
+
+      # @param [CredentialProvider] credentials
+      def initialize(credentials, params, force_path_style)
+        @credentials = credentials.credentials
+        @params = Query::ParamList.new
+        params.each_pair do |param_name, param_value|
+          @params.set(param_name, param_value)
+        end
+        @force_path_style = force_path_style
+      end
+
+      attr_reader :credentials, :params
+
+      def sign(request)
+        if token = credentials.session_token
+          request.headers["X-Amz-Security-Token"] = token
+        end
+        request.headers['Authorization'] = authorization(request)
+      end
+
+      def authorization(request)
+        "AWS #{credentials.access_key_id}:#{signature(request)}"
+      end
+
+      def signature(request)
+        string_to_sign = string_to_sign(request)
+        signature = digest(credentials.secret_access_key, string_to_sign)
+        uri_escape(signature)
+      end
+
+      def digest(secret, string_to_sign)
+        Base64.encode64(hmac(secret, string_to_sign)).strip
+      end
+
+      def hmac(key, value)
+        OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), key, value)
+      end
+
+      # From the S3 developer guide:
+      #
+      #     StringToSign =
+      #       HTTP-Verb ` "\n" `
+      #       content-md5 ` "\n" `
+      #       content-type ` "\n" `
+      #       date ` "\n" `
+      #       CanonicalizedAmzHeaders + CanonicalizedResource;
+      #
+      def string_to_sign(request)
+        [
+          request.http_method,
+          request.headers.values_at('Content-Md5', 'Content-Type').join("\n"),
+          signing_string_date(request),
+          canonicalized_headers(request),
+          canonicalized_resource(request.endpoint),
+        ].flatten.compact.join("\n")
+      end
+
+      def signing_string_date(request)
+        # if a date is provided via x-amz-date then we should omit the
+        # Date header from the signing string (should appear as a blank line)
+        if request.headers.detect{|k,v| k.to_s =~ /^x-amz-date$/i }
+          ''
+        else
+          request.headers['Date'] = Time.now.httpdate
+        end
+      end
+
+      # CanonicalizedAmzHeaders
+      #
+      # See the developer guide for more information on how this element
+      # is generated.
+      #
+      def canonicalized_headers(request)
+        x_amz = request.headers.select{|k, v| k =~ /^x-amz-/i }
+        x_amz = x_amz.collect{|k, v| [k.downcase, v] }
+        x_amz = x_amz.sort_by{|k, v| k }
+        x_amz = x_amz.collect{|k, v| "#{k}:#{v.to_s.strip}" }.join("\n")
+        x_amz == '' ? nil : x_amz
+      end
+
+      # From the S3 developer guide
+      #
+      #     CanonicalizedResource =
+      #       [ "/" ` Bucket ] `
+      #       <HTTP-Request-URI, protocol name up to the querystring> +
+      #       [ sub-resource, if present. e.g. "?acl", "?location",
+      #       "?logging", or "?torrent"];
+      #
+      # @api private
+      def canonicalized_resource(endpoint)
+
+        parts = []
+
+        # virtual hosted-style requests require the hostname to appear
+        # in the canonicalized resource prefixed by a forward slash.
+        if bucket = params[:bucket]
+          bucket = bucket.value
+          ssl = endpoint.scheme == 'https'
+          if Plugins::BucketDns.dns_compatible?(bucket, ssl) && !@force_path_style
+            parts << "/#{bucket}"
+          end
+        end
+
+        # append the path name (no querystring)
+        parts << endpoint.path
+
+        # lastly any sub resource querystring params need to be appened
+        # in lexigraphical ordered joined by '&' and prefixed by '?'
+        params = signed_querystring_params(endpoint)
+
+        unless params.empty?
+          parts << '?'
+          parts << params.sort.collect{|p| p.to_s }.join('&')
+        end
+
+        parts.join
+      end
+
+      def signed_querystring_params(endpoint)
+        endpoint.query.to_s.split('&').select do |p|
+          SIGNED_QUERYSTRING_PARAMS.include?(p.split('=')[0])
+        end.map { |p| CGI.unescape(p) }
+      end
+
+      def uri_escape(s)
+
+        #URI.escape(s)
+
+        # URI.escape is deprecated, replacing it with escape from webrick
+        # to squelch the massive number of warnings generated from Ruby.
+        # The following script was used to determine the differences
+        # between the various escape methods available. The webrick
+        # escape only had two differences and it is available in the
+        # standard lib.
+        #
+        #     (0..255).each {|c|
+        #       s = [c].pack("C")
+        #       e = [
+        #         CGI.escape(s),
+        #         ERB::Util.url_encode(s),
+        #         URI.encode_www_form_component(s),
+        #         WEBrick::HTTPUtils.escape_form(s),
+        #         WEBrick::HTTPUtils.escape(s),
+        #         URI.escape(s),
+        #       ]
+        #       next if e.uniq.length == 1
+        #       puts("%5s %5s %5s %5s %5s %5s %5s" % ([s.inspect] + e))
+        #     }
+        #
+        WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']')
+      end
+
+    end
+  end
+end