httpx 0.17.0 → 0.18.3

data/lib/httpx/plugins/retries.rb

@@ -12,19 +12,29 @@ module HTTPX
       # TODO: pass max_retries in a configure/load block
 
       IDEMPOTENT_METHODS = %i[get options head put delete].freeze
-      RETRYABLE_ERRORS = [IOError,
-                          EOFError,
-                          Errno::ECONNRESET,
-                          Errno::ECONNABORTED,
-                          Errno::EPIPE,
-                          TLSError,
-                          TimeoutError,
-                          Parser::Error,
-                          Errno::EINVAL,
-                          Errno::ETIMEDOUT].freeze
-
-      def self.extra_options(options)
-        options.merge(max_retries: MAX_RETRIES)
+      RETRYABLE_ERRORS = [
+        IOError,
+        EOFError,
+        Errno::ECONNRESET,
+        Errno::ECONNABORTED,
+        Errno::EPIPE,
+        Errno::EINVAL,
+        Errno::ETIMEDOUT,
+        Parser::Error,
+        TLSError,
+        TimeoutError,
+        Connection::HTTP2::GoawayError,
+      ].freeze
+      DEFAULT_JITTER = ->(interval) { interval * (0.5 * (1 + rand)) }
+
+      if ENV.key?("HTTPX_NO_JITTER")
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES)
+        end
+      else
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+        end
       end
 
       module OptionsMethods
@@ -38,6 +48,13 @@ module HTTPX
           value
         end
 
+        def option_retry_jitter(value)
+          # return early if callable
+          raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
+
+          value
+        end
+
         def option_max_retries(value)
           num = Integer(value)
           raise TypeError, ":max_retries must be positive" unless num.positive?
@@ -87,10 +104,15 @@ module HTTPX
             retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
 
             if retry_after
+              # apply jitter
+              if (jitter = request.options.retry_jitter)
+                retry_after = jitter.call(retry_after)
+              end
 
+              retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
               pool.after(retry_after) do
-                log { "retrying!!" }
+                log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                 connection = find_connection(request, connections, options)
                 connection.send(request)
               end

data/lib/httpx/plugins/stream.rb

@@ -9,7 +9,7 @@ module HTTPX
     end
 
     def each(&block)
-      return enum_for(__method__) unless block_given?
+      return enum_for(__method__) unless block
 
       raise Error, "response already streamed" if @response
 

data/lib/httpx/pool.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
 require "forwardable"
-require "timers"
 require "httpx/selector"
 require "httpx/connection"
 require "httpx/resolver"
 
 module HTTPX
   class Pool
+    using ArrayExtensions
     extend Forwardable
 
     def_delegator :@timers, :after
@@ -15,7 +15,7 @@ module HTTPX
     def initialize
       @resolvers = {}
       @_resolver_ios = {}
-      @timers = Timers::Group.new
+      @timers = Timers.new
       @selector = Selector.new
       @connections = []
       @connected_connections = 0
@@ -27,15 +27,18 @@ module HTTPX
 
     def next_tick
       catch(:jump_tick) do
-        timeout = [next_timeout, @timers.wait_interval].compact.min
+        timeout = next_timeout
         if timeout && timeout.negative?
           @timers.fire
           throw(:jump_tick)
         end
 
-        @selector.select(timeout, &:call)
-
-        @timers.fire
+        begin
+          @selector.select(timeout, &:call)
+          @timers.fire
+        rescue TimeoutError => e
+          @timers.fire(e)
+        end
       end
     rescue StandardError => e
       @connections.each do |connection|
@@ -64,6 +67,16 @@ module HTTPX
       connection.on(:open) do
         @connected_connections += 1
       end
+      connection.on(:activate) do
+        select_connection(connection)
+      end
+    end
+
+    def deactivate(connections)
+      connections.each do |connection|
+        connection.deactivate
+        deselect_connection(connection) if connection.state == :inactive
+      end
     end
 
     # opens a connection to the IP reachable through +uri+.
@@ -81,7 +94,7 @@ module HTTPX
     def resolve_connection(connection)
       @connections << connection unless @connections.include?(connection)
 
-      if connection.addresses || connection.state == :open
+      if connection.addresses || connection.open?
         #
         # there are two cases in which we want to activate initialization of
         # connection immediately:
@@ -98,7 +111,7 @@ module HTTPX
       resolver << connection
       return if resolver.empty?
 
-      @_resolver_ios[resolver] ||= @selector.register(resolver)
+      @_resolver_ios[resolver] ||= select_connection(resolver)
     end
 
     def on_resolver_connection(connection)
@@ -107,7 +120,7 @@ module HTTPX
       end
       return register_connection(connection) unless found_connection
 
-      if found_connection.state == :open
+      if found_connection.open?
         coalesce_connections(found_connection, connection)
         throw(:coalesced, found_connection)
       else
@@ -129,7 +142,7 @@ module HTTPX
 
       @resolvers.delete(resolver_type)
 
-      @selector.deregister(resolver)
+      deselect_connection(resolver)
       @_resolver_ios.delete(resolver)
       resolver.close unless resolver.closed?
     end
@@ -140,7 +153,7 @@ module HTTPX
         # consider it connected already.
         @connected_connections += 1
       end
-      @selector.register(connection)
+      select_connection(connection)
       connection.on(:close) do
         unregister_connection(connection)
       end
@@ -148,10 +161,18 @@ module HTTPX
 
     def unregister_connection(connection)
       @connections.delete(connection)
-      @selector.deregister(connection)
+      deselect_connection(connection)
       @connected_connections -= 1
     end
 
+    def select_connection(connection)
+      @selector.register(connection)
+    end
+
+    def deselect_connection(connection)
+      @selector.deregister(connection)
+    end
+
     def coalesce_connections(conn1, conn2)
       if conn1.coalescable?(conn2)
         conn1.merge(conn2)
@@ -162,7 +183,11 @@ module HTTPX
     end
 
     def next_timeout
-      @resolvers.values.reject(&:closed?).map(&:timeout).compact.min || @connections.map(&:timeout).compact.min
+      [
+        @timers.wait_interval,
+        *@resolvers.values.reject(&:closed?).filter_map(&:timeout),
+        *@connections.filter_map(&:timeout),
+      ].compact.min
     end
 
     def find_resolver_for(connection)
@@ -172,6 +197,7 @@ module HTTPX
 
       @resolvers[resolver_type] ||= begin
         resolver = resolver_type.new(connection_options)
+        resolver.pool = self if resolver.respond_to?(:pool=)
         resolver.on(:resolve, &method(:on_resolver_connection))
         resolver.on(:error, &method(:on_resolver_error))
         resolver.on(:close) { on_resolver_close(resolver) }

data/lib/httpx/request.rb

@@ -148,10 +148,10 @@ module HTTPX
     # :nocov:
     def inspect
       "#<HTTPX::Request:#{object_id} " \
-      "#{@verb.to_s.upcase} " \
-      "#{uri} " \
-      "@headers=#{@headers} " \
-      "@body=#{@body}>"
+        "#{@verb.to_s.upcase} " \
+        "#{uri} " \
+        "@headers=#{@headers} " \
+        "@body=#{@body}>"
     end
     # :nocov:
 
@@ -181,7 +181,7 @@ module HTTPX
       end
 
       def each(&block)
-        return enum_for(__method__) unless block_given?
+        return enum_for(__method__) unless block
         return if @body.nil?
 
         body = stream(@body)
@@ -236,7 +236,7 @@ module HTTPX
       # :nocov:
       def inspect
         "#<HTTPX::Request::Body:#{object_id} " \
-        "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
+          "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
       end
       # :nocov:
     end
@@ -285,7 +285,7 @@ module HTTPX
       end
 
       def write(data)
-        @block.call(data)
+        @block.call(data.dup)
         data.bytesize
       end
     end

data/lib/httpx/resolver/https.rb

@@ -24,7 +24,9 @@ module HTTPX
       record_types: RECORD_TYPES.keys,
     }.freeze
 
-    def_delegators :@resolver_connection, :connecting?, :to_io, :call, :close
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close
+
+    attr_writer :pool
 
     def initialize(options)
       @options = Options.new(options)
@@ -63,15 +65,11 @@ module HTTPX
 
     private
 
-    def pool
-      Thread.current[:httpx_connection_pool] ||= Pool.new
-    end
-
     def resolver_connection
-      @resolver_connection ||= pool.find_connection(@uri, @options) || begin
+      @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
         @building_connection = true
         connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
-        pool.init_connection(connection, @options)
+        @pool.init_connection(connection, @options)
         emit_addresses(connection, @uri_addresses)
         @building_connection = false
         connection

data/lib/httpx/resolver/native.rb

@@ -47,6 +47,8 @@ module HTTPX
 
     def_delegator :@connections, :empty?
 
+    attr_reader :state
+
     def initialize(options)
       @options = Options.new(options)
       @ns_index = 0
@@ -120,7 +122,7 @@ module HTTPX
     def timeout
       return if @connections.empty?
 
-      @start_timeout = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @start_timeout = Utils.now
       hosts = @queries.keys
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
@@ -140,7 +142,7 @@ module HTTPX
     def do_retry
       return if @queries.empty?
 
-      loop_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_timeout
+      loop_time = Utils.elapsed_time(@start_timeout)
       connections = []
       queries = {}
       while (query = @queries.shift)

data/lib/httpx/resolver/system.rb

@@ -12,6 +12,8 @@ module HTTPX
                      Resolv::DNS::EncodeError,
                      Resolv::DNS::DecodeError].freeze
 
+    attr_reader :state
+
     def initialize(options)
       @options = Options.new(options)
       @resolver_options = @options.resolver_options

data/lib/httpx/resolver.rb

@@ -26,14 +26,14 @@ module HTTPX
     module_function
 
     def cached_lookup(hostname)
-      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      now = Utils.now
       @lookup_mutex.synchronize do
         lookup(hostname, now)
       end
     end
 
     def cached_lookup_set(hostname, entries)
-      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      now = Utils.now
       entries.each do |entry|
         entry["TTL"] += now
       end

data/lib/httpx/response.rb

@@ -57,17 +57,23 @@ module HTTPX
     # :nocov:
     def inspect
       "#<Response:#{object_id} "\
-      "HTTP/#{version} " \
-      "@status=#{@status} " \
-      "@headers=#{@headers} " \
-      "@body=#{@body.bytesize}>"
+        "HTTP/#{version} " \
+        "@status=#{@status} " \
+        "@headers=#{@headers} " \
+        "@body=#{@body.bytesize}>"
     end
     # :nocov:
 
-    def raise_for_status
+    def error
       return if @status < 400
 
-      raise HTTPError, self
+      HTTPError.new(self)
+    end
+
+    def raise_for_status
+      return self unless (err = error)
+
+      raise err
     end
 
     def json(options = nil)
@@ -211,18 +217,20 @@ module HTTPX
       end
 
       def ==(other)
-        if other.respond_to?(:read)
-          _with_same_buffer_pos { FileUtils.compare_stream(@buffer, other) }
-        else
-          to_s == other.to_s
+        object_id == other.object_id || begin
+          if other.respond_to?(:read)
+            _with_same_buffer_pos { FileUtils.compare_stream(@buffer, other) }
+          else
+            to_s == other.to_s
+          end
         end
       end
 
       # :nocov:
       def inspect
         "#<HTTPX::Response::Body:#{object_id} " \
-        "@state=#{@state} " \
-        "@length=#{@length}>"
+          "@state=#{@state} " \
+          "@length=#{@length}>"
       end
       # :nocov:
 
@@ -311,17 +319,18 @@ module HTTPX
     end
 
     def status
+      warn ":#{__method__} is deprecated, use :error.message instead"
       @error.message
     end
 
     if Exception.method_defined?(:full_message)
       def to_s
-        @error.full_message
+        @error.full_message(highlight: false)
       end
     else
       def to_s
         "#{@error.message} (#{@error.class})\n" \
-        "#{@error.backtrace.join("\n") if @error.backtrace}"
+          "#{@error.backtrace.join("\n") if @error.backtrace}"
       end
     end
 

data/lib/httpx/selector.rb

@@ -2,20 +2,6 @@
 
 require "io/wait"
 
-module IOExtensions
-  refine IO do
-    # provides a fallback for rubies where IO#wait isn't implemented,
-    # but IO#wait_readable and IO#wait_writable are.
-    def wait(timeout = nil, _mode = :read_write)
-      r, w = IO.select([self], [self], nil, timeout)
-
-      return unless r || w
-
-      self
-    end
-  end
-end
-
 class HTTPX::Selector
   READABLE = %i[rw r].freeze
   WRITABLE = %i[rw w].freeze
@@ -23,7 +9,7 @@ class HTTPX::Selector
   private_constant :READABLE
   private_constant :WRITABLE
 
-  using IOExtensions unless IO.method_defined?(:wait) && IO.instance_method(:wait).arity == 2
+  using HTTPX::IOExtensions
 
   def initialize
     @selectables = []
@@ -58,11 +44,13 @@ class HTTPX::Selector
           selectables = @selectables
           @selectables = []
 
-          selectables.each do |io|
+          selectables.delete_if do |io|
             interests = io.interests
 
             (r ||= []) << io if READABLE.include?(interests)
             (w ||= []) << io if WRITABLE.include?(interests)
+
+            io.state == :closed
           end
 
           if @selectables.empty?
@@ -70,7 +58,7 @@ class HTTPX::Selector
 
             # do not run event loop if there's nothing to wait on.
             # this might happen if connect failed and connection was unregistered.
-            return if (!r || r.empty?) && (!w || w.empty?)
+            return if (!r || r.empty?) && (!w || w.empty?) && !selectables.empty?
 
             break
           else
@@ -129,6 +117,13 @@ class HTTPX::Selector
   end
 
   def select(interval, &block)
+    # do not cause an infinite loop here.
+    #
+    # this may happen if timeout calculation actually triggered an error which causes
+    # the connections to be reaped (such as the total timeout error) before #select
+    # gets called.
+    return if interval.nil? && @selectables.empty?
+
     return select_one(interval, &block) if @selectables.size == 1
 
     select_many(interval, &block)

data/lib/httpx/session.rb

@@ -11,7 +11,7 @@ module HTTPX
       @options = self.class.default_options.merge(options)
       @responses = {}
       @persistent = @options.persistent
-      wrap(&blk) if block_given?
+      wrap(&blk) if blk
     end
 
     def wrap
@@ -21,6 +21,7 @@ module HTTPX
         yield self
       ensure
         @persistent = prev_persistent
+        close unless @persistent
       end
     end
 
@@ -226,7 +227,11 @@ module HTTPX
         end
         responses
       ensure
-        close(connections) unless @persistent
+        if @persistent
+          pool.deactivate(connections)
+        else
+          close(connections)
+        end
       end
     end
 

data/lib/httpx/session2.rb

@@ -7,7 +7,7 @@ module HTTPX
       @options = self.class.default_options.merge(options)
       @responses = {}
       @persistent = @options.persistent
-      wrap(&blk) if block_given?
+      wrap(&blk) if blk
     end
 
     def wrap

data/lib/httpx/timers.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class Timers
+    def initialize
+      @intervals = []
+    end
+
+    def after(interval_in_secs, &blk)
+      return unless interval_in_secs
+
+      # I'm assuming here that most requests will have the same
+      # request timeout, as in most cases they share common set of
+      # options. A user setting different request timeouts for 100s of
+      # requests will already have a hard time dealing with that.
+      unless (interval = @intervals.find { |t| t == interval_in_secs })
+        interval = Interval.new(interval_in_secs)
+        @intervals << interval
+        @intervals.sort!
+      end
+
+      interval << blk
+    end
+
+    def wait_interval
+      return if @intervals.empty?
+
+      @next_interval_at = Utils.now
+
+      @intervals.first.interval
+    end
+
+    def fire(error = nil)
+      raise error if error && error.timeout != @intervals.first
+      return if @intervals.empty? || !@next_interval_at
+
+      elapsed_time = Utils.elapsed_time(@next_interval_at)
+
+      @intervals.delete_if { |interval| interval.elapse(elapsed_time) <= 0 }
+    end
+
+    def cancel
+      @intervals.clear
+    end
+
+    class Interval
+      include Comparable
+
+      attr_reader :interval
+
+      def initialize(interval)
+        @interval = interval
+        @callbacks = []
+      end
+
+      def <=>(other)
+        @interval <=> other.interval
+      end
+
+      def ==(other)
+        return @interval == other if other.is_a?(Numeric)
+
+        @interval == other.to_f # rubocop:disable Lint/FloatComparison
+      end
+
+      def to_f
+        @interval
+      end
+
+      def <<(callback)
+        @callbacks << callback
+      end
+
+      def elapse(elapsed)
+        @interval -= elapsed
+
+        @callbacks.each(&:call) if @interval <= 0
+
+        @interval
+      end
+    end
+    private_constant :Interval
+  end
+end

data/lib/httpx/transcoder/body.rb

@@ -9,6 +9,7 @@ module HTTPX::Transcoder
     module_function
 
     class Encoder
+      using HTTPX::ArrayExtensions
       extend Forwardable
 
       def_delegator :@raw, :to_s
@@ -21,7 +22,7 @@ module HTTPX::Transcoder
         if @raw.respond_to?(:bytesize)
           @raw.bytesize
         elsif @raw.respond_to?(:to_ary)
-          @raw.map(&:bytesize).reduce(0, :+)
+          @raw.sum(&:bytesize)
         elsif @raw.respond_to?(:size)
           @raw.size || Float::INFINITY
         elsif @raw.respond_to?(:length)

data/lib/httpx/transcoder/form.rb

@@ -50,7 +50,7 @@ module HTTPX::Transcoder
     def decode(response)
       content_type = response.content_type.mime_type
 
-      raise Error, "invalid form mime type (#{content_type})" unless content_type == "application/x-www-form-urlencoded"
+      raise HTTPX::Error, "invalid form mime type (#{content_type})" unless content_type == "application/x-www-form-urlencoded"
 
       Decoder
     end

data/lib/httpx/transcoder/json.rb

@@ -35,7 +35,7 @@ module HTTPX::Transcoder
     def decode(response)
       content_type = response.content_type.mime_type
 
-      raise Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
+      raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type)
 
       ::JSON.method(:parse)
     end

data/lib/httpx/utils.rb

@@ -6,6 +6,14 @@ module HTTPX
 
     module_function
 
+    def now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def elapsed_time(monotonic_timestamp)
+      Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_timestamp
+    end
+
     # The value of this field can be either an HTTP-date or a number of
     # seconds to delay after the response is received.
     def parse_retry_after(retry_after)

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.17.0"
+  VERSION = "0.18.3"
 end