httpx 0.17.0 → 0.18.0

data/lib/httpx/plugins/aws_sdk_authentication.rb

@@ -8,6 +8,23 @@ module HTTPX
     # It requires the "aws-sdk-core" gem.
     #
     module AwsSdkAuthentication
+      # Mock configuration, to be used only when resolving credentials
+      class Configuration
+        attr_reader :profile
+
+        def initialize(profile)
+          @profile = profile
+        end
+
+        def respond_to_missing?(*)
+          true
+        end
+
+        def method_missing(*)
+          nil
+        end
+      end
+
       #
       # encapsulates access to an AWS SDK credentials store.
       #
@@ -30,23 +47,8 @@ module HTTPX
       end
 
       class << self
-        attr_reader :credentials, :region
-
         def load_dependencies(_klass)
           require "aws-sdk-core"
-
-          client = Class.new(Seahorse::Client::Base) do
-            @identifier = :httpx
-            set_api(Aws::S3::ClientApi::API)
-            add_plugin(Aws::Plugins::CredentialsConfiguration)
-            add_plugin(Aws::Plugins::RegionalEndpoint)
-            class << self
-              attr_reader :identifier
-            end
-          end.new
-
-          @credentials = Credentials.new(client.config[:credentials])
-          @region = client.config[:region]
         end
 
         def configure(klass)
@@ -56,6 +58,26 @@ module HTTPX
         def extra_options(options)
           options.merge(max_concurrent_requests: 1)
         end
+
+        def credentials(profile)
+          mock_configuration = Configuration.new(profile)
+          Credentials.new(Aws::CredentialProviderChain.new(mock_configuration).resolve)
+        end
+
+        def region(profile)
+          # https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb#L62
+          keys = %w[AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION]
+          env_region = ENV.values_at(*keys).compact.first
+          env_region = nil if env_region == ""
+          cfg_region = Aws.shared_config.region(profile: profile)
+          env_region || cfg_region
+        end
+      end
+
+      module OptionsMethods
+        def option_aws_profile(value)
+          String(value)
+        end
       end
 
       module InstanceMethods
@@ -64,9 +86,11 @@ module HTTPX
         # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
         # aws_authentication()
         #
-        def aws_sdk_authentication(**options)
-          credentials = AwsSdkAuthentication.credentials
-          region = AwsSdkAuthentication.region
+        def aws_sdk_authentication(
+          credentials: AwsSdkAuthentication.credentials(@options.aws_profile),
+          region: AwsSdkAuthentication.region(@options.aws_profile),
+          **options
+        )
 
           aws_sigv4_authentication(
             credentials: credentials,

data/lib/httpx/plugins/aws_sigv4.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-require "aws-sdk-s3"
-
 module HTTPX
   module Plugins
     #
@@ -75,16 +72,16 @@ module HTTPX
 
           # canonical request
           creq = "#{request.verb.to_s.upcase}" \
-            "\n#{request.canonical_path}" \
-            "\n#{request.canonical_query}" \
-            "\n#{canonical_headers}" \
-            "\n#{signed_headers}" \
-            "\n#{content_hashed}"
+                 "\n#{request.canonical_path}" \
+                 "\n#{request.canonical_query}" \
+                 "\n#{canonical_headers}" \
+                 "\n#{signed_headers}" \
+                 "\n#{content_hashed}"
 
           credential_scope = "#{date}" \
-            "/#{@region}" \
-            "/#{@service}" \
-            "/#{lower_provider_prefix}_request"
+                             "/#{@region}" \
+                             "/#{@service}" \
+                             "/#{lower_provider_prefix}_request"
 
           algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
           # string to sign
@@ -142,6 +139,7 @@ module HTTPX
 
       class << self
         def load_dependencies(*)
+          require "set"
           require "digest/sha2"
           require "openssl"
         end

data/lib/httpx/plugins/compression.rb

@@ -72,6 +72,8 @@ module HTTPX
       end
 
       module ResponseBodyMethods
+        using ArrayExtensions
+
         attr_reader :encodings
 
         def initialize(*)
@@ -90,7 +92,7 @@ module HTTPX
             Float::INFINITY
           end
 
-          @_inflaters = @headers.get("content-encoding").map do |encoding|
+          @_inflaters = @headers.get("content-encoding").filter_map do |encoding|
             next if encoding == "identity"
 
             inflater = @options.encodings.registry(encoding).inflater(compressed_length)
@@ -100,7 +102,7 @@ module HTTPX
 
             @encodings << encoding
             inflater
-          end.compact
+          end
 
           # this can happen if the only declared encoding is "identity"
           remove_instance_variable(:@_inflaters) if @_inflaters.empty?
@@ -134,7 +136,7 @@ module HTTPX
         end
 
         def each(&blk)
-          return enum_for(__method__) unless block_given?
+          return enum_for(__method__) unless blk
 
           return deflate(&blk) if @buffer.size.zero?
 

data/lib/httpx/plugins/cookies/jar.rb

@@ -55,7 +55,7 @@ module HTTPX
       end
 
       def each(uri = nil, &blk)
-        return enum_for(__method__, uri) unless block_given?
+        return enum_for(__method__, uri) unless blk
 
         return @cookies.each(&blk) unless uri
 

data/lib/httpx/plugins/expect.rb

@@ -69,9 +69,14 @@ module HTTPX
       end
 
       module ConnectionMethods
-        def send(request)
+        def send_request_to_parser(request)
+          super
+
+          return unless request.headers["expect"] == "100-continue"
+
           request.once(:expect) do
-            @timers.after(@options.expect_timeout) do
+            @timers.after(request.options.expect_timeout) do
+              # expect timeout expired
               if request.state == :expect && !request.expects?
                 Expect.no_expect_store << request.origin
                 request.headers.delete("expect")
@@ -79,7 +84,6 @@ module HTTPX
               end
             end
           end
-          super
         end
       end
 

data/lib/httpx/plugins/grpc/message.rb

@@ -17,7 +17,7 @@ module HTTPX
 
         # lazy decodes a grpc stream response
         def stream(response, &block)
-          return enum_for(__method__, response) unless block_given?
+          return enum_for(__method__, response) unless block
 
           response.each do |frame|
             decode(frame, encodings: response.headers.get("grpc-encoding"), encoders: response.encoders, &block)
@@ -57,7 +57,7 @@ module HTTPX
 
             yield data
 
-            message = message.byteslice(5 + size..-1)
+            message = message.byteslice((5 + size)..-1)
           end
         end
 

data/lib/httpx/plugins/grpc.rb

@@ -143,9 +143,9 @@ module HTTPX
 
           session_class = Class.new(self.class) do
             class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-              def #{rpc_name}(input, **opts)
-                rpc_execute("#{rpc_name}", input, **opts)
-              end
+              def #{rpc_name}(input, **opts)              # def grpc_action(input, **opts)
+                rpc_execute("#{rpc_name}", input, **opts) #   rpc_execute("grpc_action", input, **opts)
+              end                                         # end
             OUT
           end
 

data/lib/httpx/plugins/internal_telemetry.rb

@@ -44,6 +44,11 @@ module HTTPX
           meter_elapsed_time("Session: initialized!!!")
         end
 
+        def close(*)
+          super
+          meter_elapsed_time("Session -> close")
+        end
+
         private
 
         def build_requests(*)
@@ -55,11 +60,6 @@ module HTTPX
           meter_elapsed_time("Session -> response") if response
           response
         end
-
-        def close(*)
-          super
-          meter_elapsed_time("Session -> close")
-        end
       end
 
       module RequestMethods
@@ -69,9 +69,9 @@ module HTTPX
         end
 
         def transition(nextstate)
-          state = @state
+          prev_state = @state
           super
-          meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
+          meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{prev_state}] -> #{@state}") if prev_state != @state
         end
       end
 
@@ -84,7 +84,7 @@ module HTTPX
         def transition(nextstate)
           state = @state
           super
-          meter_elapsed_time("Connection[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
+          meter_elapsed_time("Connection##{object_id}[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
         end
       end
     end

data/lib/httpx/plugins/multipart.rb

@@ -29,8 +29,8 @@ module HTTPX
               # in order not to break legacy code, we'll keep loading http/form_data for them.
               require "http/form_data"
               warn "httpx: http/form_data is no longer a requirement to use HTTPX :multipart plugin. See migration instructions under" \
-                "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
-                "If you'd like to stop seeing this message, require 'http/form_data' yourself."
+                   "https://honeyryderchuck.gitlab.io/httpx/wiki/Multipart-Uploads.html#notes. \n\n" \
+                   "If you'd like to stop seeing this message, require 'http/form_data' yourself."
             end
           rescue LoadError
           end

data/lib/httpx/plugins/response_cache/store.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module HTTPX::Plugins
+  module ResponseCache
+    class Store
+      extend Forwardable
+
+      def_delegator :@store, :clear
+
+      def initialize
+        @store = {}
+      end
+
+      def lookup(uri)
+        @store[uri]
+      end
+
+      def cached?(uri)
+        @store.key?(uri)
+      end
+
+      def cache(uri, response)
+        @store[uri] = response
+      end
+
+      def prepare(request)
+        cached_response = @store[request.uri]
+
+        return unless cached_response
+
+        original_request = cached_response.instance_variable_get(:@request)
+
+        if (vary = cached_response.headers["vary"])
+          if vary == "*"
+            return unless request.headers.same_headers?(original_request.headers)
+          else
+            return unless vary.split(/ *, */).all? do |cache_field|
+              !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
+            end
+          end
+        end
+
+        if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
+          request.headers.add("if-modified-since", last_modified)
+        end
+
+        if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
+          request.headers.add("if-none-match", etag)
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/response_cache.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when certain errors happen.
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/Response-Cache
+    #
+    module ResponseCache
+      CACHEABLE_VERBS = %i[get head].freeze
+      private_constant :CACHEABLE_VERBS
+
+      class << self
+        def load_dependencies(*)
+          require_relative "response_cache/store"
+        end
+
+        def cacheable_request?(request)
+          CACHEABLE_VERBS.include?(request.verb)
+        end
+
+        def cacheable_response?(response)
+          response.is_a?(Response) &&
+            # partial responses shall not be cached, only full ones.
+            response.status != 206 && (
+            response.headers.key?("etag") || response.headers.key?("last-modified-at")
+          )
+        end
+
+        def cached_response?(response)
+          response.is_a?(Response) && response.status == 304
+        end
+
+        def extra_options(options)
+          options.merge(response_cache_store: Store.new)
+        end
+      end
+
+      module OptionsMethods
+        def option_response_cache_store(value)
+          raise TypeError, "must be an instance of #{Store}" unless value.is_a?(Store)
+
+          value
+        end
+      end
+
+      module InstanceMethods
+        def clear_response_cache
+          @options.response_cache_store.clear
+        end
+
+        def build_request(*)
+          request = super
+          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request.uri)
+
+          @options.response_cache_store.prepare(request)
+
+          request
+        end
+
+        def fetch_response(request, *)
+          response = super
+
+          if response && ResponseCache.cached_response?(response)
+            log { "returning cached response for #{request.uri}" }
+            cached_response = @options.response_cache_store.lookup(request.uri)
+
+            response.copy_from_cached(cached_response)
+          end
+
+          @options.response_cache_store.cache(request.uri, response) if response && ResponseCache.cacheable_response?(response)
+
+          response
+        end
+      end
+
+      module ResponseMethods
+        def copy_from_cached(other)
+          @body = other.body
+
+          @body.__send__(:rewind)
+        end
+      end
+    end
+    register_plugin :response_cache, ResponseCache
+  end
+end

data/lib/httpx/plugins/retries.rb

@@ -22,9 +22,16 @@ module HTTPX
                           Parser::Error,
                           Errno::EINVAL,
                           Errno::ETIMEDOUT].freeze
+      DEFAULT_JITTER = ->(interval) { interval * (0.5 * (1 + rand)) }
 
-      def self.extra_options(options)
-        options.merge(max_retries: MAX_RETRIES)
+      if ENV.key?("HTTPX_NO_JITTER")
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES)
+        end
+      else
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+        end
       end
 
       module OptionsMethods
@@ -38,6 +45,13 @@ module HTTPX
           value
         end
 
+        def option_retry_jitter(value)
+          # return early if callable
+          raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
+
+          value
+        end
+
         def option_max_retries(value)
           num = Integer(value)
           raise TypeError, ":max_retries must be positive" unless num.positive?
@@ -87,10 +101,15 @@ module HTTPX
             retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
 
             if retry_after
+              # apply jitter
+              if (jitter = request.options.retry_jitter)
+                retry_after = jitter.call(retry_after)
+              end
 
+              retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
               pool.after(retry_after) do
-                log { "retrying!!" }
+                log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                 connection = find_connection(request, connections, options)
                 connection.send(request)
               end

data/lib/httpx/plugins/stream.rb

@@ -9,7 +9,7 @@ module HTTPX
     end
 
     def each(&block)
-      return enum_for(__method__) unless block_given?
+      return enum_for(__method__) unless block
 
       raise Error, "response already streamed" if @response
 

data/lib/httpx/pool.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
 require "forwardable"
-require "timers"
 require "httpx/selector"
 require "httpx/connection"
 require "httpx/resolver"
 
 module HTTPX
   class Pool
+    using ArrayExtensions
     extend Forwardable
 
     def_delegator :@timers, :after
@@ -15,7 +15,7 @@ module HTTPX
     def initialize
       @resolvers = {}
       @_resolver_ios = {}
-      @timers = Timers::Group.new
+      @timers = Timers.new
       @selector = Selector.new
       @connections = []
       @connected_connections = 0
@@ -27,15 +27,18 @@ module HTTPX
 
     def next_tick
       catch(:jump_tick) do
-        timeout = [next_timeout, @timers.wait_interval].compact.min
+        timeout = next_timeout
         if timeout && timeout.negative?
           @timers.fire
           throw(:jump_tick)
         end
 
-        @selector.select(timeout, &:call)
-
-        @timers.fire
+        begin
+          @selector.select(timeout, &:call)
+          @timers.fire
+        rescue TimeoutError => e
+          @timers.fire(e)
+        end
       end
     rescue StandardError => e
       @connections.each do |connection|
@@ -64,6 +67,16 @@ module HTTPX
       connection.on(:open) do
         @connected_connections += 1
       end
+      connection.on(:activate) do
+        select_connection(connection)
+      end
+    end
+
+    def deactivate(connections)
+      connections.each do |connection|
+        connection.deactivate
+        deselect_connection(connection) if connection.state == :inactive
+      end
     end
 
     # opens a connection to the IP reachable through +uri+.
@@ -81,7 +94,7 @@ module HTTPX
     def resolve_connection(connection)
       @connections << connection unless @connections.include?(connection)
 
-      if connection.addresses || connection.state == :open
+      if connection.addresses || connection.open?
         #
         # there are two cases in which we want to activate initialization of
         # connection immediately:
@@ -98,7 +111,7 @@ module HTTPX
       resolver << connection
       return if resolver.empty?
 
-      @_resolver_ios[resolver] ||= @selector.register(resolver)
+      @_resolver_ios[resolver] ||= select_connection(resolver)
     end
 
     def on_resolver_connection(connection)
@@ -107,7 +120,7 @@ module HTTPX
       end
       return register_connection(connection) unless found_connection
 
-      if found_connection.state == :open
+      if found_connection.open?
         coalesce_connections(found_connection, connection)
         throw(:coalesced, found_connection)
       else
@@ -129,7 +142,7 @@ module HTTPX
 
       @resolvers.delete(resolver_type)
 
-      @selector.deregister(resolver)
+      deselect_connection(resolver)
       @_resolver_ios.delete(resolver)
       resolver.close unless resolver.closed?
     end
@@ -140,7 +153,7 @@ module HTTPX
         # consider it connected already.
         @connected_connections += 1
       end
-      @selector.register(connection)
+      select_connection(connection)
       connection.on(:close) do
         unregister_connection(connection)
       end
@@ -148,10 +161,18 @@ module HTTPX
 
     def unregister_connection(connection)
       @connections.delete(connection)
-      @selector.deregister(connection)
+      deselect_connection(connection)
       @connected_connections -= 1
     end
 
+    def select_connection(connection)
+      @selector.register(connection)
+    end
+
+    def deselect_connection(connection)
+      @selector.deregister(connection)
+    end
+
     def coalesce_connections(conn1, conn2)
       if conn1.coalescable?(conn2)
         conn1.merge(conn2)
@@ -162,7 +183,11 @@ module HTTPX
     end
 
     def next_timeout
-      @resolvers.values.reject(&:closed?).map(&:timeout).compact.min || @connections.map(&:timeout).compact.min
+      [
+        @timers.wait_interval,
+        *@resolvers.values.reject(&:closed?).filter_map(&:timeout),
+        *@connections.filter_map(&:timeout),
+      ].compact.min
     end
 
     def find_resolver_for(connection)
@@ -172,6 +197,7 @@ module HTTPX
 
       @resolvers[resolver_type] ||= begin
         resolver = resolver_type.new(connection_options)
+        resolver.pool = self if resolver.respond_to?(:pool=)
         resolver.on(:resolve, &method(:on_resolver_connection))
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }

data/lib/httpx/request.rb

@@ -148,10 +148,10 @@ module HTTPX
     # :nocov:
     def inspect
       "#<HTTPX::Request:#{object_id} " \
-      "#{@verb.to_s.upcase} " \
-      "#{uri} " \
-      "@headers=#{@headers} " \
-      "@body=#{@body}>"
+        "#{@verb.to_s.upcase} " \
+        "#{uri} " \
+        "@headers=#{@headers} " \
+        "@body=#{@body}>"
     end
     # :nocov:
 
@@ -181,7 +181,7 @@ module HTTPX
       end
 
       def each(&block)
-        return enum_for(__method__) unless block_given?
+        return enum_for(__method__) unless block
         return if @body.nil?
 
         body = stream(@body)
@@ -236,7 +236,7 @@ module HTTPX
       # :nocov:
       def inspect
         "#<HTTPX::Request::Body:#{object_id} " \
-        "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
+          "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
       end
       # :nocov:
     end