httpx 0.16.0 → 0.18.1

data/lib/httpx/plugins/response_cache.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when certain errors happen.
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/Response-Cache
+    #
+    module ResponseCache
+      CACHEABLE_VERBS = %i[get head].freeze
+      private_constant :CACHEABLE_VERBS
+
+      class << self
+        def load_dependencies(*)
+          require_relative "response_cache/store"
+        end
+
+        def cacheable_request?(request)
+          CACHEABLE_VERBS.include?(request.verb)
+        end
+
+        def cacheable_response?(response)
+          response.is_a?(Response) &&
+            # partial responses shall not be cached, only full ones.
+            response.status != 206 && (
+            response.headers.key?("etag") || response.headers.key?("last-modified-at")
+          )
+        end
+
+        def cached_response?(response)
+          response.is_a?(Response) && response.status == 304
+        end
+
+        def extra_options(options)
+          options.merge(response_cache_store: Store.new)
+        end
+      end
+
+      module OptionsMethods
+        def option_response_cache_store(value)
+          raise TypeError, "must be an instance of #{Store}" unless value.is_a?(Store)
+
+          value
+        end
+      end
+
+      module InstanceMethods
+        def clear_response_cache
+          @options.response_cache_store.clear
+        end
+
+        def build_request(*)
+          request = super
+          return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request.uri)
+
+          @options.response_cache_store.prepare(request)
+
+          request
+        end
+
+        def fetch_response(request, *)
+          response = super
+
+          if response && ResponseCache.cached_response?(response)
+            log { "returning cached response for #{request.uri}" }
+            cached_response = @options.response_cache_store.lookup(request.uri)
+
+            response.copy_from_cached(cached_response)
+          end
+
+          @options.response_cache_store.cache(request.uri, response) if response && ResponseCache.cacheable_response?(response)
+
+          response
+        end
+      end
+
+      module ResponseMethods
+        def copy_from_cached(other)
+          @body = other.body
+
+          @body.__send__(:rewind)
+        end
+      end
+    end
+    register_plugin :response_cache, ResponseCache
+  end
+end

data/lib/httpx/plugins/retries.rb

@@ -12,19 +12,29 @@ module HTTPX
       # TODO: pass max_retries in a configure/load block
 
       IDEMPOTENT_METHODS = %i[get options head put delete].freeze
-      RETRYABLE_ERRORS = [IOError,
-                          EOFError,
-                          Errno::ECONNRESET,
-                          Errno::ECONNABORTED,
-                          Errno::EPIPE,
-                          TLSError,
-                          TimeoutError,
-                          Parser::Error,
-                          Errno::EINVAL,
-                          Errno::ETIMEDOUT].freeze
-
-      def self.extra_options(options)
-        options.merge(max_retries: MAX_RETRIES)
+      RETRYABLE_ERRORS = [
+        IOError,
+        EOFError,
+        Errno::ECONNRESET,
+        Errno::ECONNABORTED,
+        Errno::EPIPE,
+        Errno::EINVAL,
+        Errno::ETIMEDOUT,
+        Parser::Error,
+        TLSError,
+        TimeoutError,
+        Connection::HTTP2::GoawayError,
+      ].freeze
+      DEFAULT_JITTER = ->(interval) { interval * (0.5 * (1 + rand)) }
+
+      if ENV.key?("HTTPX_NO_JITTER")
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES)
+        end
+      else
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+        end
       end
 
       module OptionsMethods
@@ -38,6 +48,13 @@ module HTTPX
           value
         end
 
+        def option_retry_jitter(value)
+          # return early if callable
+          raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
+
+          value
+        end
+
         def option_max_retries(value)
           num = Integer(value)
           raise TypeError, ":max_retries must be positive" unless num.positive?
@@ -87,10 +104,15 @@ module HTTPX
             retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
 
             if retry_after
+              # apply jitter
+              if (jitter = request.options.retry_jitter)
+                retry_after = jitter.call(retry_after)
+              end
 
+              retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
               pool.after(retry_after) do
-                log { "retrying!!" }
+                log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                 connection = find_connection(request, connections, options)
                 connection.send(request)
               end

data/lib/httpx/plugins/stream.rb

@@ -6,11 +6,10 @@ module HTTPX
       @request = request
       @session = session
       @connections = connections
-      @options = @request.options
     end
 
     def each(&block)
-      return enum_for(__method__) unless block_given?
+      return enum_for(__method__) unless block
 
       raise Error, "response already streamed" if @response
 
@@ -72,7 +71,7 @@ module HTTPX
     private
 
     def response
-      @session.__send__(:receive_requests, [@request], @connections, @options) until @request.response
+      @session.__send__(:receive_requests, [@request], @connections) until @request.response
 
       @request.response
     end
@@ -106,7 +105,7 @@ module HTTPX
 
           request = requests.first
 
-          connections = _send_requests(requests, request.options)
+          connections = _send_requests(requests)
 
           StreamResponse.new(request, self, connections)
         end

data/lib/httpx/pool.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
 require "forwardable"
-require "timers"
 require "httpx/selector"
 require "httpx/connection"
 require "httpx/resolver"
 
 module HTTPX
   class Pool
+    using ArrayExtensions
     extend Forwardable
 
     def_delegator :@timers, :after
@@ -15,7 +15,7 @@ module HTTPX
     def initialize
       @resolvers = {}
       @_resolver_ios = {}
-      @timers = Timers::Group.new
+      @timers = Timers.new
       @selector = Selector.new
       @connections = []
       @connected_connections = 0
@@ -27,15 +27,18 @@ module HTTPX
 
     def next_tick
       catch(:jump_tick) do
-        timeout = [next_timeout, @timers.wait_interval].compact.min
+        timeout = next_timeout
         if timeout && timeout.negative?
           @timers.fire
           throw(:jump_tick)
         end
 
-        @selector.select(timeout, &:call)
-
-        @timers.fire
+        begin
+          @selector.select(timeout, &:call)
+          @timers.fire
+        rescue TimeoutError => e
+          @timers.fire(e)
+        end
       end
     rescue StandardError => e
       @connections.each do |connection|
@@ -64,6 +67,16 @@ module HTTPX
       connection.on(:open) do
         @connected_connections += 1
       end
+      connection.on(:activate) do
+        select_connection(connection)
+      end
+    end
+
+    def deactivate(connections)
+      connections.each do |connection|
+        connection.deactivate
+        deselect_connection(connection) if connection.state == :inactive
+      end
     end
 
     # opens a connection to the IP reachable through +uri+.
@@ -81,7 +94,7 @@ module HTTPX
     def resolve_connection(connection)
       @connections << connection unless @connections.include?(connection)
 
-      if connection.addresses || connection.state == :open
+      if connection.addresses || connection.open?
         #
         # there are two cases in which we want to activate initialization of
         # connection immediately:
@@ -98,7 +111,7 @@ module HTTPX
       resolver << connection
       return if resolver.empty?
 
-      @_resolver_ios[resolver] ||= @selector.register(resolver)
+      @_resolver_ios[resolver] ||= select_connection(resolver)
     end
 
     def on_resolver_connection(connection)
@@ -107,7 +120,7 @@ module HTTPX
       end
       return register_connection(connection) unless found_connection
 
-      if found_connection.state == :open
+      if found_connection.open?
         coalesce_connections(found_connection, connection)
         throw(:coalesced, found_connection)
       else
@@ -129,7 +142,7 @@ module HTTPX
 
       @resolvers.delete(resolver_type)
 
-      @selector.deregister(resolver)
+      deselect_connection(resolver)
       @_resolver_ios.delete(resolver)
       resolver.close unless resolver.closed?
     end
@@ -140,7 +153,7 @@ module HTTPX
         # consider it connected already.
         @connected_connections += 1
       end
-      @selector.register(connection)
+      select_connection(connection)
       connection.on(:close) do
         unregister_connection(connection)
       end
@@ -148,10 +161,18 @@ module HTTPX
 
     def unregister_connection(connection)
       @connections.delete(connection)
-      @selector.deregister(connection)
+      deselect_connection(connection)
       @connected_connections -= 1
     end
 
+    def select_connection(connection)
+      @selector.register(connection)
+    end
+
+    def deselect_connection(connection)
+      @selector.deregister(connection)
+    end
+
     def coalesce_connections(conn1, conn2)
       if conn1.coalescable?(conn2)
         conn1.merge(conn2)
@@ -162,7 +183,11 @@ module HTTPX
     end
 
     def next_timeout
-      @resolvers.values.reject(&:closed?).map(&:timeout).compact.min || @connections.map(&:timeout).compact.min
+      [
+        @timers.wait_interval,
+        *@resolvers.values.reject(&:closed?).filter_map(&:timeout),
+        *@connections.filter_map(&:timeout),
+      ].compact.min
     end
 
     def find_resolver_for(connection)
@@ -172,6 +197,7 @@ module HTTPX
 
       @resolvers[resolver_type] ||= begin
         resolver = resolver_type.new(connection_options)
+        resolver.pool = self if resolver.respond_to?(:pool=)
         resolver.on(:resolve, &method(:on_resolver_connection))
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }

data/lib/httpx/registry.rb

@@ -59,7 +59,7 @@ module HTTPX
         @registry ||= {}
         return @registry if tag.nil?
 
-        handler = @registry.fetch(tag)
+        handler = @registry[tag]
         raise(Error, "#{tag} is not registered in #{self}") unless handler
 
         handler

data/lib/httpx/request.rb

@@ -41,16 +41,15 @@ module HTTPX
 
     def_delegator :@body, :empty?
 
-    def_delegator :@body, :chunk!
-
     def initialize(verb, uri, options = {})
       @verb    = verb.to_s.downcase.to_sym
       @options = Options.new(options)
       @uri     = Utils.to_uri(uri)
       if @uri.relative?
-        raise(Error, "invalid URI: #{@uri}") unless @options.origin
+        origin = @options.origin
+        raise(Error, "invalid URI: #{@uri}") unless origin
 
-        @uri = @options.origin.merge(@uri)
+        @uri = origin.merge(@uri)
       end
 
       raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
@@ -98,7 +97,7 @@ module HTTPX
     def response=(response)
       return unless response
 
-      if response.status == 100
+      if response.is_a?(Response) && response.status == 100
         @informational_status = response.status
         return
       end
@@ -149,16 +148,16 @@ module HTTPX
     # :nocov:
     def inspect
       "#<HTTPX::Request:#{object_id} " \
-      "#{@verb.to_s.upcase} " \
-      "#{uri} " \
-      "@headers=#{@headers} " \
-      "@body=#{@body}>"
+        "#{@verb.to_s.upcase} " \
+        "#{uri} " \
+        "@headers=#{@headers} " \
+        "@body=#{@body}>"
     end
     # :nocov:
 
     class Body < SimpleDelegator
       class << self
-        def new(*, options)
+        def new(_, options)
           return options.body if options.body.is_a?(self)
 
           super
@@ -182,7 +181,7 @@ module HTTPX
       end
 
       def each(&block)
-        return enum_for(__method__) unless block_given?
+        return enum_for(__method__) unless block
         return if @body.nil?
 
         body = stream(@body)
@@ -223,7 +222,7 @@ module HTTPX
       def unbounded_body?
         return @unbounded_body if defined?(@unbounded_body)
 
-        @unbounded_body = (chunked? || @body.bytesize == Float::INFINITY)
+        @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
       end
 
       def chunked?
@@ -237,7 +236,7 @@ module HTTPX
       # :nocov:
       def inspect
         "#<HTTPX::Request::Body:#{object_id} " \
-        "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
+          "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
       end
       # :nocov:
     end

data/lib/httpx/resolver/https.rb

@@ -24,7 +24,9 @@ module HTTPX
       record_types: RECORD_TYPES.keys,
     }.freeze
 
-    def_delegators :@resolver_connection, :connecting?, :to_io, :call, :close
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close
+
+    attr_writer :pool
 
     def initialize(options)
       @options = Options.new(options)
@@ -63,15 +65,11 @@ module HTTPX
 
     private
 
-    def pool
-      Thread.current[:httpx_connection_pool] ||= Pool.new
-    end
-
     def resolver_connection
-      @resolver_connection ||= pool.find_connection(@uri, @options) || begin
+      @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
         @building_connection = true
         connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
-        pool.init_connection(connection, @options)
+        @pool.init_connection(connection, @options)
         emit_addresses(connection, @uri_addresses)
         @building_connection = false
         connection

data/lib/httpx/resolver/native.rb

@@ -47,6 +47,8 @@ module HTTPX
 
     def_delegator :@connections, :empty?
 
+    attr_reader :state
+
     def initialize(options)
       @options = Options.new(options)
       @ns_index = 0
@@ -120,7 +122,7 @@ module HTTPX
     def timeout
       return if @connections.empty?
 
-      @start_timeout = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @start_timeout = Utils.now
       hosts = @queries.keys
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
@@ -140,7 +142,7 @@ module HTTPX
     def do_retry
       return if @queries.empty?
 
-      loop_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_timeout
+      loop_time = Utils.elapsed_time(@start_timeout)
       connections = []
       queries = {}
       while (query = @queries.shift)
@@ -217,15 +219,27 @@ module HTTPX
         end
       else
         address = addresses.first
-        connection = @queries.delete(address["name"])
-        return unless connection # probably a retried query for which there's an answer
+        name = address["name"]
+
+        connection = @queries.delete(name)
+
+        unless connection
+          # absolute name
+          name_labels = Resolv::DNS::Name.create(name).to_a
+          name = @queries.keys.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
+
+          # probably a retried query for which there's an answer
+          return unless name
+
+          address["name"] = name
+          connection = @queries.delete(name)
+        end
 
         if address.key?("alias") # CNAME
           if early_resolve(connection, hostname: address["alias"])
             @connections.delete(connection)
           else
             resolve(connection, address["alias"])
-            @queries.delete(address["name"])
             return
           end
         else

data/lib/httpx/resolver/resolver_mixin.rb

@@ -9,7 +9,7 @@ module HTTPX
       include Callbacks
       include Loggable
 
-      CHECK_IF_IP = proc do |name|
+      CHECK_IF_IP = lambda do |name|
         begin
           IPAddr.new(name)
           true
@@ -55,6 +55,7 @@ module HTTPX
         return if ips.empty?
 
         ips.map { |ip| IPAddr.new(ip) }
+      rescue IOError
       end
 
       def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)

data/lib/httpx/resolver/system.rb

@@ -12,6 +12,8 @@ module HTTPX
                      Resolv::DNS::EncodeError,
                      Resolv::DNS::DecodeError].freeze
 
+    attr_reader :state
+
     def initialize(options)
       @options = Options.new(options)
       @resolver_options = @options.resolver_options

data/lib/httpx/resolver.rb

@@ -26,14 +26,14 @@ module HTTPX
     module_function
 
     def cached_lookup(hostname)
-      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      now = Utils.now
       @lookup_mutex.synchronize do
         lookup(hostname, now)
       end
     end
 
     def cached_lookup_set(hostname, entries)
-      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      now = Utils.now
       entries.each do |entry|
         entry["TTL"] += now
       end