smithy-client 1.0.0.pre0 → 1.0.0.pre1

data/lib/smithy-client/plugins/request_compression.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+module Smithy
+  module Client
+    module Plugins
+      # @api private
+      class RequestCompression < Plugin
+        DEFAULT_MIN_COMPRESSION_SIZE = 10_240
+        MIN_COMPRESSION_SIZE_LIMIT = 10_485_760
+        SUPPORTED_ENCODINGS = %w[gzip].freeze
+        CHUNK_SIZE = 1 * 1024 * 1024 # one MB
+
+        option(
+          :disable_request_compression,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: 'When `true`, the request body will not be compressed for supported operations.'
+        ) do |_config|
+          value = ENV['DISABLE_REQUEST_COMPRESSION'] || 'false'
+          Util.str_to_bool(value)
+        end
+
+        option(
+          :request_min_compression_size_bytes,
+          default: DEFAULT_MIN_COMPRESSION_SIZE,
+          doc_type: Integer,
+          docstring: <<~DOCS) do |_config|
+            The minimum size in bytes that triggers compression for request bodies.
+            The value must be non-negative integer value between 0 and 10,485,780 bytes inclusive.
+          DOCS
+          value = ENV['REQUEST_MIN_COMPRESSION_SIZE_BYTES'] || DEFAULT_MIN_COMPRESSION_SIZE
+          Integer(value)
+        end
+
+        def after_initialize(client)
+          validate_disable_request_compression(client.config)
+          validate_request_min_compression_size_bytes(client.config)
+        end
+
+        def validate_disable_request_compression(config)
+          return if [true, false].include?(config.disable_request_compression)
+
+          raise ArgumentError,
+                ':disable_request_compression must be either `true` or `false`'
+        end
+
+        def validate_request_min_compression_size_bytes(config)
+          begin
+            value = Integer(config.request_min_compression_size_bytes)
+            return if value.between?(0, MIN_COMPRESSION_SIZE_LIMIT)
+          rescue ArgumentError
+            # handled below
+          end
+
+          raise ArgumentError,
+                ':request_min_compression_size_bytes must be a non-negative integer ' \
+                'value between `0` and `10,485,760` bytes inclusive'
+        end
+
+        def add_handlers(handlers, config)
+          # Ensure compression is performed BEFORE calculating a checksum
+          handlers.add(Handler, priority: 25) unless config.disable_request_compression
+        end
+
+        # @api private
+        class Handler < Client::Handler
+          def call(context)
+            if request_compression_trait?(context)
+              selected_encoding = request_encoding_selection(context)
+              if selected_encoding
+                if streaming?(context.operation.input)
+                  process_streaming_compression(selected_encoding, context)
+                elsif context.http_request.body.size >= context.config.request_min_compression_size_bytes
+                  process_compression(selected_encoding, context)
+                end
+              end
+            end
+            @handler.call(context)
+          end
+
+          private
+
+          def request_compression_trait?(context)
+            context.operation.traits.key?('smithy.api#requestCompression')
+          end
+
+          def request_encoding_selection(context)
+            encodings = context.operation.traits['smithy.api#requestCompression']['encodings']
+            encodings.find { |encoding| RequestCompression::SUPPORTED_ENCODINGS.include?(encoding) }
+          end
+
+          def streaming?(input)
+            input.shape.members.any? do |_, member_ref|
+              member_ref.shape.traits.key?('smithy.api#streaming') &&
+                !member_ref.shape.traits.key?('smithy.api#requiresLength')
+            end
+          end
+
+          def process_streaming_compression(encoding, context)
+            case encoding
+            when 'gzip'
+              context.http_request.body = GzipIO.new(context.http_request.body)
+            else
+              raise StandardError, "Encoding #{encoding} is not supported"
+            end
+            update_content_encoding(encoding, context)
+          end
+
+          def process_compression(encoding, context)
+            case encoding
+            when 'gzip'
+              gzip_compress(context)
+            else
+              raise StandardError, "We currently do not support #{encoding} encoding"
+            end
+            update_content_encoding(encoding, context)
+          end
+
+          def gzip_compress(context) # rubocop:disable Metrics/AbcSize
+            compressed = StringIO.new
+            compressed.binmode
+            gzip_writer = Zlib::GzipWriter.new(compressed)
+            if context.http_request.body.respond_to?(:read)
+              update_in_chunks(gzip_writer, context.http_request.body)
+            else
+              gzip_writer.write(context.http_request.body)
+            end
+            gzip_writer.close
+            context.http_request.body = StringIO.new(compressed.string)
+          end
+
+          def update_in_chunks(compressor, io)
+            loop do
+              chunk = io.read(CHUNK_SIZE)
+              break unless chunk
+
+              compressor.write(chunk)
+            end
+          end
+
+          def update_content_encoding(encoding, context)
+            headers = context.http_request.headers
+            if headers['Content-Encoding']
+              headers['Content-Encoding'] += ", #{encoding}"
+            else
+              headers['Content-Encoding'] = encoding
+            end
+          end
+
+          # @api private
+          class GzipIO
+            def initialize(body)
+              @body = body
+              @buffer = ChunkBuffer.new
+              @gzip_writer = Zlib::GzipWriter.new(@buffer)
+            end
+
+            def read(length, buff = nil)
+              if @gzip_writer.closed?
+                # an empty string to signify an end as
+                # there will be nothing remaining to be read
+                StringIO.new('').read(length, buff)
+                return
+              end
+
+              chunk = @body.read(length)
+              if !chunk || chunk.empty?
+                # closing the writer will write one last chunk
+                # with a trailer (to be read from the @buffer)
+                @gzip_writer.close
+              else
+                # flush happens first to ensure that header fields
+                # are being sent over since write will override
+                @gzip_writer.flush
+                @gzip_writer.write(chunk)
+              end
+
+              StringIO.new(@buffer.last_chunk).read(length, buff)
+            end
+          end
+
+          # @api private
+          class ChunkBuffer
+            def initialize
+              @last_chunk = nil
+            end
+
+            attr_reader :last_chunk
+
+            def write(data)
+              @last_chunk = data
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/smithy-client/plugins/response_target.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module Smithy
+  module Client
+    module Plugins
+      # @api private
+      class ResponseTarget < Plugin
+        # This handler is responsible for replacing the response body IO
+        # object with custom targets, such as a block, or a file. It is important
+        # to not write data to the custom target in the case of a non-success
+        # response. We do not want to write an XML error message to someone's
+        # file.
+        class Handler < Client::Handler
+          def call(context)
+            target = context[:response_target]
+            add_event_listeners(context.http_response, target) if target
+            @handler.call(context)
+          end
+
+          private
+
+          def add_event_listeners(response, target)
+            add_header_listener(response, target)
+            add_success_listener(response)
+            add_error_listener(response)
+          end
+
+          def add_header_listener(response, target)
+            response.on_headers(200..299) do
+              # In a fresh response, the body will be a StringIO. However, when a request
+              # is retried we may have an existing ManagedFile or BlockIO,
+              # and those should be reused.
+              response.body = io(target, response.headers) if response.body.is_a?(StringIO)
+            end
+          end
+
+          def add_success_listener(response)
+            response.on_success(200..299) do
+              body = response.body
+              body.close if body.is_a?(ManagedFile) && body.open?
+            end
+          end
+
+          def add_error_listener(response)
+            response.on_error do
+              body = response.body
+              # When using a File response_target, we do not want to write
+              # error messages to the file. So set the body to a new StringIO
+              if body.is_a?(ManagedFile)
+                File.unlink(body)
+                response.body = StringIO.new
+              end
+            end
+          end
+
+          def io(target, headers)
+            case target
+            when Proc then BlockIO.new(headers, &target)
+            when String, Pathname then ManagedFile.new(target, 'w+b')
+            else target
+            end
+          end
+        end
+
+        handler(Handler, step: :initialize, priority: 90)
+      end
+    end
+  end
+end

data/lib/smithy-client/plugins/retry_errors.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Smithy
+  module Client
+    module Plugins
+      # @api private
+      class RetryErrors < Plugin
+        option(
+          :retry_strategy,
+          default: 'standard',
+          doc_type: 'String, Class',
+          docstring: <<~DOCS)
+            The retry strategy to use when retrying errors. This can be one of the following:
+            * `standard` - A standardized retry strategy used by the AWS SDKs. This includes support
+              for retry quotas, which limit the number of unsuccessful retries a client can make.
+            * `adaptive` - An experimental retry strategy that includes all the functionality of the
+              `standard` strategy along with automatic client side throttling. This is a provisional
+              strategy that may change behavior in the future.
+            * Any instance of a class that implements the following methods:
+              - `acquire_initial_retry_token(token_scope)`
+              - `refresh_retry_token(retry_token, error_info)`
+              - `record_success(retry_token)`
+          DOCS
+
+        option(
+          :retry_max_attempts,
+          default: 3,
+          doc_type: Integer,
+          docstring: <<~DOCS)
+            The maximum number attempts that will be made for a single request, including
+            the initial attempt. Used in the `standard` and `adaptive` retry strategies.
+          DOCS
+
+        option(
+          :retry_backoff,
+          default: Retry::EXPONENTIAL_BACKOFF,
+          doc_default: 'Smithy::Client::Retry::EXPONENTIAL_BACKOFF',
+          doc_type: 'lambda',
+          docstring: <<~DOCS)
+            A callable object that calculates a backoff delay for a retry attempt. The callable
+            should accept a single argument, `attempts`, that represents the number of attempts
+            that have been made. Used in the `standard` and `adaptive` retry strategies.
+          DOCS
+
+        option(
+          :adaptive_retry_wait_to_fill,
+          default: true,
+          doc_type: 'Boolean',
+          docstring: <<~DOCS)
+            When true, the request will sleep until there is sufficient client side capacity to retry
+            the request. When false, the request will raise a `CapacityNotAvailableError` and will
+            not retry instead of sleeping.
+          DOCS
+
+        def after_initialize(client)
+          config = client.config
+          config.retry_strategy =
+            case config.retry_strategy
+            when 'standard'
+              Retry::Standard.new(
+                max_attempts: config.retry_max_attempts,
+                backoff: config.retry_backoff
+              )
+            when 'adaptive'
+              Retry::Adaptive.new(
+                max_attempts: config.retry_max_attempts,
+                backoff: config.retry_backoff,
+                wait_to_fill: config.adaptive_retry_wait_to_fill
+              )
+            else
+              config.retry_strategy
+            end
+        end
+
+        # @api private
+        class Handler < Client::Handler
+          def call(context)
+            retry_strategy = context.config.retry_strategy
+            token = retry_strategy.acquire_initial_retry_token(nil)
+            handle(context, retry_strategy, token)
+          end
+
+          private
+
+          def handle(context, retry_strategy, token)
+            response = @handler.call(context)
+            if (error = response.error)
+              return response unless retryable?(context.http_request)
+
+              error_info = HTTP::ErrorInspector.new(error, context.http_response)
+              token = retry_strategy.refresh_retry_token(token, error_info)
+              return response unless token
+
+              Kernel.sleep(token.retry_delay)
+            else
+              retry_strategy.record_success(token)
+              return response
+            end
+
+            reset_request(context)
+            reset_response(context, response)
+            context.retries += 1
+            handle(context, retry_strategy, token)
+          end
+
+          def retryable?(request)
+            # IO responds to #rewind however it returns an illegal seek error
+            request.body.respond_to?(:rewind) && !request.body.instance_of?(IO)
+          end
+
+          def reset_request(context)
+            context.http_request.body.rewind
+          end
+
+          def reset_response(context, response)
+            context.http_response.reset
+            response.error = nil
+          end
+        end
+
+        handler(Handler, step: :retry)
+      end
+    end
+  end
+end

data/lib/smithy-client/plugins/sign_requests.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Smithy
+  module Client
+    module Plugins
+      # @api private
+      class SignRequests < Plugin
+        # @api private
+        class Handler < Client::Handler
+          def call(context)
+            context[:auth].signer.sign(
+              request: context.http_request,
+              identity: context[:auth].identity,
+              properties: context[:auth].signer_properties
+            )
+            @handler.call(context)
+          end
+        end
+
+        handler(Handler, step: :sign)
+      end
+    end
+  end
+end

data/lib/smithy-client/plugins/stub_responses.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Smithy
+  module Client
+    module Plugins
+      # @api private
+      class StubResponses < Plugin
+        option(
+          :stub_responses,
+          default: false,
+          doc_type: 'Boolean',
+          docstring: <<~DOCS)
+            When `true`, the client will return stubbed responses instead of networking requests.
+            By default fake responses are generated and returned. You can specify the response data
+            to return or errors to raise by calling {Stubs#stub_responses}.
+            @see Stubs
+          DOCS
+
+        option(:stubs) { {} }
+        # @api private
+        option(:stubs_mutex) { Mutex.new }
+        # @api private
+        option(:api_requests) { [] }
+        # @api private
+        option(:api_requests_mutex) { Mutex.new }
+
+        def add_handlers(handlers, config)
+          return unless config.stub_responses
+
+          handlers.add(APIRequestsHandler)
+          handlers.add(StubHandler, step: :send)
+        end
+
+        def before_initialize(_client_class, options)
+          return unless options[:stub_responses]
+
+          options[:endpoint_provider] ||= Stubbing::EndpointProvider.new
+        end
+
+        def after_initialize(client)
+          return unless client.config.stub_responses
+
+          client.handlers.remove(RetryErrors::Handler)
+        end
+
+        # Returns a registered stubbed response instead of a real response.
+        # @api private
+        class StubHandler < Client::Handler
+          def call(context)
+            response = Smithy::Client::Response.new(context: context)
+            stub = context.client.next_stub(context)
+            stub[:mutex].synchronize { apply_stub(stub, response) }
+            response
+          end
+
+          private
+
+          def apply_stub(stub, response)
+            http_response = response.context.http_response
+            if stub[:error]
+              signal_error(stub[:error], http_response)
+            elsif stub[:http]
+              signal_http(stub[:http], http_response)
+            end
+          end
+
+          def signal_error(error, http_response)
+            if error.is_a?(Exception)
+              http_response.signal_error(error)
+            else
+              http_response.signal_error(error.new)
+            end
+          end
+
+          def signal_http(stub, http_response)
+            http_response.signal_headers(stub.status_code, stub.headers)
+            signal_data(stub, http_response)
+            http_response.signal_done
+          end
+
+          def signal_data(stub, http_response)
+            while (chunk = stub.body.read(1024 * 1024))
+              http_response.signal_data(chunk)
+            end
+            stub.body.rewind
+          end
+        end
+
+        # Tracks API requests made by the client.
+        # @api private
+        class APIRequestsHandler < Client::Handler
+          def call(context)
+            context.config.api_requests_mutex.synchronize do
+              context.config.api_requests << context
+            end
+            @handler.call(context)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/smithy-client/protocol_spec_matcher.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'rspec/expectations'
+
+# Provides an RSpec matcher for protocol specs.
+# rubocop:disable Metrics/BlockLength
+RSpec::Matchers.define :match_data do |expected|
+  match do |actual|
+    # identical values don't need more comparison
+    return true if actual == expected
+
+    expect(actual.class).to eq(expected.class)
+
+    def match_hash(actual, expected)
+      expected.each do |key, value|
+        expect(actual).to include(key)
+        match_data(actual[key], value)
+      end
+
+      actual.each_key do |key|
+        expect(expected).to include(key)
+      end
+    end
+
+    def match_array(actual, expected)
+      actual.each_with_index do |value, index|
+        match_data(value, expected[index])
+      end
+    end
+
+    def match_float(actual, expected)
+      return if actual.nan? && expected.nan?
+      return if actual.infinite? && expected.infinite?
+
+      expect(actual).to be_within(0.0001).of(expected)
+    end
+
+    def match_data(actual, expected) # rubocop:disable Metrics/AbcSize
+      case actual
+      when Hash
+        match_hash(actual, expected)
+      when Array
+        match_array(actual, expected)
+      when Float
+        match_float(actual, expected)
+      when Time
+        expect(actual.utc.iso8601).to eq(expected.utc.iso8601)
+      when StringIO
+        expect(actual.string).to eq(expected)
+      else
+        expect(actual).to eq(expected)
+      end
+    end
+
+    match_data(actual, expected)
+  end
+
+  diffable
+end
+# rubocop:enable Metrics/BlockLength

data/lib/smithy-client/refreshing_identity_provider.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Smithy
+  module Client
+    # A module that can be included in a class to provide a #identity method.
+    # The class must implement #refresh(properties) that sets @identity. The
+    # refresh method will be called when #identity is called and the identity
+    # is nil or near expiration.
+    module RefreshingIdentityProvider
+      SYNC_EXPIRATION_LENGTH = 300 # 5 minutes
+      ASYNC_EXPIRATION_LENGTH = 600 # 10 minutes
+
+      def initialize
+        @mutex = Mutex.new
+      end
+
+      # @return [Identities::Base]
+      def identity(properties = {})
+        if @identity
+          refresh_if_near_expiration!(properties)
+        else # initialization
+          @mutex.synchronize { refresh(properties) }
+        end
+        @identity
+      end
+
+      private
+
+      def sync_expiration_length
+        self.class::SYNC_EXPIRATION_LENGTH
+      end
+
+      def async_expiration_length
+        self.class::ASYNC_EXPIRATION_LENGTH
+      end
+
+      # Refreshes identity asynchronously and synchronously.
+      # If we are near to expiration, block while refreshing the identity.
+      # Otherwise, if we're approaching expiration, use the existing identity
+      # but attempt a refresh in the background.
+      def refresh_if_near_expiration!(properties)
+        # NOTE: This check is an optimization. Rather than acquire the mutex on
+        # every #refresh_if_near_expiration call, we check before doing so, and
+        # then we check within the mutex to avoid a race condition.
+        if near_expiration?(sync_expiration_length)
+          @mutex.synchronize do
+            refresh(properties) if near_expiration?(sync_expiration_length)
+          end
+        elsif @async_refresh && near_expiration?(async_expiration_length)
+          unless @mutex.locked?
+            Thread.new do
+              @mutex.synchronize do
+                refresh(properties) if near_expiration?(async_expiration_length)
+              end
+            end
+          end
+        end
+      end
+
+      def near_expiration?(expiration_length)
+        (Time.now.to_i + expiration_length) > @identity.expiration.to_i
+      end
+    end
+  end
+end