httpx 1.4.3 → 1.5.0

data/lib/httpx/plugins/retries.rb

@@ -17,7 +17,9 @@ module HTTPX
       # TODO: pass max_retries in a configure/load block
 
       IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
-      RETRYABLE_ERRORS = [
+
+      # subset of retryable errors which are safe to retry when reconnecting
+      RECONNECTABLE_ERRORS = [
         IOError,
         EOFError,
         Errno::ECONNRESET,
@@ -25,12 +27,15 @@ module HTTPX
         Errno::EPIPE,
         Errno::EINVAL,
         Errno::ETIMEDOUT,
-        Parser::Error,
-        TLSError,
-        TimeoutError,
         ConnectionError,
-        Connection::HTTP2::GoawayError,
+        TLSError,
+        Connection::HTTP2::Error,
       ].freeze
+
+      RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
+        Parser::Error,
+        TimeoutError,
+      ]).freeze
       DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
 
       if ENV.key?("HTTPX_NO_JITTER")
@@ -88,6 +93,7 @@ module HTTPX
       end
 
       module InstanceMethods
+        # returns a `:retries` plugin enabled session with +n+ maximum retries per request setting.
         def max_retries(n)
           with(max_retries: n)
         end
@@ -99,18 +105,18 @@ module HTTPX
 
           if response &&
              request.retries.positive? &&
-             __repeatable_request?(request, options) &&
+             repeatable_request?(request, options) &&
              (
                (
-                 response.is_a?(ErrorResponse) && __retryable_error?(response.error)
+                 response.is_a?(ErrorResponse) && retryable_error?(response.error)
                ) ||
                (
                  options.retry_on && options.retry_on.call(response)
                )
              )
-            __try_partial_retry(request, response)
+            try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
-            request.retries -= 1
+            request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
             request.transition(:idle)
 
             retry_after = options.retry_after
@@ -125,9 +131,10 @@ module HTTPX
               retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
               selector.after(retry_after) do
-                if request.response
+                if (response = request.response)
+                  response.finish!
                   # request has terminated abruptly meanwhile
-                  request.emit(:response, request.response)
+                  request.emit(:response, response)
                 else
                   log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                   send_request(request, selector, options)
@@ -142,11 +149,13 @@ module HTTPX
           response
         end
 
-        def __repeatable_request?(request, options)
+        # returns whether +request+ can be retried.
+        def repeatable_request?(request, options)
           IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
         end
 
-        def __retryable_error?(ex)
+        # returns whether the +ex+ exception happend for a retriable request.
+        def retryable_error?(ex)
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
 
@@ -155,11 +164,11 @@ module HTTPX
         end
 
         #
-        # Atttempt to set the request to perform a partial range request.
+        # Attempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and
         # the last response contains some body payload.
         #
-        def __try_partial_retry(request, response)
+        def try_partial_retry(request, response)
           response = response.response if response.is_a?(ErrorResponse)
 
           return unless response
@@ -180,10 +189,13 @@ module HTTPX
       end
 
       module RequestMethods
+        # number of retries left.
         attr_accessor :retries
 
+        # a response partially received before.
         attr_writer :partial_response
 
+        # initializes the request instance, sets the number of retries for the request.
         def initialize(*args)
           super
           @retries = @options.max_retries

data/lib/httpx/plugins/stream.rb

@@ -2,29 +2,43 @@
 
 module HTTPX
   class StreamResponse
+    attr_reader :request
+
     def initialize(request, session)
       @request = request
+      @options = @request.options
       @session = session
-      @response = nil
+      @response_enum = nil
+      @buffered_chunks = []
     end
 
     def each(&block)
       return enum_for(__method__) unless block
 
+      if (response_enum = @response_enum)
+        @response_enum = nil
+        # streaming already started, let's finish it
+
+        while (chunk = @buffered_chunks.shift)
+          block.call(chunk)
+        end
+
+        # consume enum til the end
+        begin
+          while (chunk = response_enum.next)
+            block.call(chunk)
+          end
+        rescue StopIteration
+          return
+        end
+      end
+
       @request.stream = self
 
       begin
         @on_chunk = block
 
-        if @request.response
-          # if we've already started collecting the payload, yield it first
-          # before proceeding.
-          body = @request.response.body
-
-          body.each do |chunk|
-            on_chunk(chunk)
-          end
-        end
+        response = @session.request(@request)
 
         response.raise_for_status
       ensure
@@ -59,38 +73,50 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<StreamResponse:#{object_id}>"
+      "#<#{self.class}:#{object_id}>"
     end
     # :nocov:
 
     def to_s
-      response.to_s
+      if @request.response
+        @request.response.to_s
+      else
+        @buffered_chunks.join
+      end
     end
 
     private
 
     def response
-      return @response if @response
-
       @request.response || begin
-        @response = @session.request(@request)
+        response_enum = each
+        while (chunk = response_enum.next)
+          @buffered_chunks << chunk
+          break if @request.response
+        end
+        @response_enum = response_enum
+        @request.response
       end
     end
 
-    def respond_to_missing?(meth, *args)
-      response.respond_to?(meth, *args) || super
+    def respond_to_missing?(meth, include_private)
+      if (response = @request.response)
+        response.respond_to_missing?(meth, include_private)
+      else
+        @options.response_class.method_defined?(meth) || (include_private && @options.response_class.private_method_defined?(meth))
+      end || super
     end
 
-    def method_missing(meth, *args, &block)
+    def method_missing(meth, *args, **kwargs, &block)
       return super unless response.respond_to?(meth)
 
-      response.__send__(meth, *args, &block)
+      response.__send__(meth, *args, **kwargs, &block)
     end
   end
 
   module Plugins
     #
-    # This plugin adds support for stream response (text/event-stream).
+    # This plugin adds support for streaming a response (useful for i.e. "text/event-stream" payloads).
     #
     # https://gitlab.com/os85/httpx/wikis/Stream
     #

data/lib/httpx/plugins/stream_bidi.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for bidirectional HTTP/2 streams.
+    #
+    # https://gitlab.com/os85/httpx/wikis/StreamBidi
+    #
+    # It is required that the request body allows chunk to be buffered, (i.e., responds to +#<<(chunk)+).
+    module StreamBidi
+      # Extension of the Connection::HTTP2 class, which adds functionality to
+      # deal with a request that can't be drained and must be interleaved with
+      # the response streams.
+      #
+      # The streams keeps send DATA frames while there's data; when they're ain't,
+      # the stream is kept open; it must be explicitly closed by the end user.
+      #
+      class HTTP2Bidi < Connection::HTTP2
+        def initialize(*)
+          super
+          @lock = Thread::Mutex.new
+        end
+
+        %i[close empty? exhausted? send <<].each do |lock_meth|
+          class_eval(<<-METH, __FILE__, __LINE__ + 1)
+            # lock.aware version of +#{lock_meth}+
+            def #{lock_meth}(*)                # def close(*)
+              return super if @lock.owned?
+
+              # small race condition between
+              # checking for ownership and
+              # acquiring lock.
+              # TODO: fix this at the parser.
+              @lock.synchronize { super }
+            end
+          METH
+        end
+
+        private
+
+        %i[join_headers join_trailers join_body].each do |lock_meth|
+          class_eval(<<-METH, __FILE__, __LINE__ + 1)
+            # lock.aware version of +#{lock_meth}+
+            def #{lock_meth}(*)                # def join_headers(*)
+              return super if @lock.owned?
+
+              # small race condition between
+              # checking for ownership and
+              # acquiring lock.
+              # TODO: fix this at the parser.
+              @lock.synchronize { super }
+            end
+          METH
+        end
+
+        def handle_stream(stream, request)
+          request.on(:body) do
+            next unless request.headers_sent
+
+            handle(request, stream)
+
+            emit(:flush_buffer)
+          end
+          super
+        end
+
+        # when there ain't more chunks, it makes the buffer as full.
+        def send_chunk(request, stream, chunk, next_chunk)
+          super
+
+          return if next_chunk
+
+          request.transition(:waiting_for_chunk)
+          throw(:buffer_full)
+        end
+
+        # sets end-stream flag when the request is closed.
+        def end_stream?(request, next_chunk)
+          request.closed? && next_chunk.nil?
+        end
+      end
+
+      # BidiBuffer is a Buffer which can be receive data from threads othr
+      # than the thread of the corresponding Connection/Session.
+      #
+      # It synchronizes access to a secondary internal +@oob_buffer+, which periodically
+      # is reconciled to the main internal +@buffer+.
+      class BidiBuffer < Buffer
+        def initialize(*)
+          super
+          @parent_thread = Thread.current
+          @oob_mutex = Thread::Mutex.new
+          @oob_buffer = "".b
+        end
+
+        # buffers the +chunk+ to be sent
+        def <<(chunk)
+          return super if Thread.current == @parent_thread
+
+          @oob_mutex.synchronize { @oob_buffer << chunk }
+        end
+
+        # reconciles the main and secondary buffer (which receives data from other threads).
+        def rebuffer
+          raise Error, "can only rebuffer while waiting on a response" unless Thread.current == @parent_thread
+
+          @oob_mutex.synchronize do
+            @buffer << @oob_buffer
+            @oob_buffer.clear
+          end
+        end
+      end
+
+      # Proxy to wake up the session main loop when one
+      # of the connections has buffered data to write. It abides by the HTTPX::_Selectable API,
+      # which allows it to be registered in the selector alongside actual HTTP-based
+      # HTTPX::Connection objects.
+      class Signal
+        def initialize
+          @closed = false
+          @pipe_read, @pipe_write = ::IO.pipe
+        end
+
+        def state
+          @closed ? :closed : :open
+        end
+
+        # noop
+        def log(**); end
+
+        def to_io
+          @pipe_read.to_io
+        end
+
+        def wakeup
+          return if @closed
+
+          @pipe_write.write("\0")
+        end
+
+        def call
+          return if @closed
+
+          @pipe_read.readpartial(1)
+        end
+
+        def interests
+          return if @closed
+
+          :r
+        end
+
+        def timeout; end
+
+        def terminate
+          @pipe_write.close
+          @pipe_read.close
+          @closed = true
+        end
+
+        # noop (the owner connection will take of it)
+        def handle_socket_timeout(interval); end
+      end
+
+      class << self
+        def load_dependencies(klass)
+          klass.plugin(:stream)
+        end
+
+        def extra_options(options)
+          options.merge(fallback_protocol: "h2")
+        end
+      end
+
+      module InstanceMethods
+        def initialize(*)
+          @signal = Signal.new
+          super
+        end
+
+        def close(selector = Selector.new)
+          @signal.terminate
+          selector.deregister(@signal)
+          super(selector)
+        end
+
+        def select_connection(connection, selector)
+          super
+          selector.register(@signal)
+          connection.signal = @signal
+        end
+
+        def deselect_connection(connection, *)
+          super
+          connection.signal = nil
+        end
+      end
+
+      # Adds synchronization to request operations which may buffer payloads from different
+      # threads.
+      module RequestMethods
+        attr_accessor :headers_sent
+
+        def initialize(*)
+          super
+          @headers_sent = false
+          @closed = false
+          @mutex = Thread::Mutex.new
+        end
+
+        def closed?
+          @closed
+        end
+
+        def can_buffer?
+          super && @state != :waiting_for_chunk
+        end
+
+        # overrides state management transitions to introduce an intermediate
+        # +:waiting_for_chunk+ state, which the request transitions to once payload
+        # is buffered.
+        def transition(nextstate)
+          headers_sent = @headers_sent
+
+          case nextstate
+          when :waiting_for_chunk
+            return unless @state == :body
+          when :body
+            case @state
+            when :headers
+              headers_sent = true
+            when :waiting_for_chunk
+              # HACK: to allow super to pass through
+              @state = :headers
+            end
+          end
+
+          super.tap do
+            # delay setting this up until after the first transition to :body
+            @headers_sent = headers_sent
+          end
+        end
+
+        def <<(chunk)
+          @mutex.synchronize do
+            if @drainer
+              @body.clear if @body.respond_to?(:clear)
+              @drainer = nil
+            end
+            @body << chunk
+
+            transition(:body)
+          end
+        end
+
+        def close
+          @mutex.synchronize do
+            return if @closed
+
+            @closed = true
+          end
+
+          # last chunk to send which ends the stream
+          self << ""
+        end
+      end
+
+      module RequestBodyMethods
+        def initialize(*, **)
+          super
+          @headers.delete("content-length")
+        end
+
+        def empty?
+          false
+        end
+      end
+
+      # overrides the declaration of +@write_buffer+, which is now a thread-safe buffer
+      # responding to the same API.
+      module ConnectionMethods
+        attr_writer :signal
+
+        def initialize(*)
+          super
+          @write_buffer = BidiBuffer.new(@options.buffer_size)
+        end
+
+        # rebuffers the +@write_buffer+ before calculating interests.
+        def interests
+          @write_buffer.rebuffer
+
+          super
+        end
+
+        private
+
+        def parser_type(protocol)
+          return HTTP2Bidi if protocol == "h2"
+
+          super
+        end
+
+        def set_parser_callbacks(parser)
+          super
+          parser.on(:flush_buffer) do
+            @signal.wakeup if @signal
+          end
+        end
+      end
+    end
+    register_plugin :stream_bidi, StreamBidi
+  end
+end

data/lib/httpx/pool.rb

@@ -13,25 +13,28 @@ module HTTPX
 
     # Sets up the connection pool with the given +options+, which can be the following:
     #
+    # :max_connections:: the maximum number of connections held in the pool.
     # :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
     # :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
     #
     def initialize(options)
+      @max_connections = options.fetch(:max_connections, Float::INFINITY)
       @max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
       @pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
       @resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
       @resolver_mtx = Thread::Mutex.new
       @connections = []
       @connection_mtx = Thread::Mutex.new
+      @connections_counter = 0
+      @max_connections_cond = ConditionVariable.new
       @origin_counters = Hash.new(0)
       @origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
     end
 
+    # connections returned by this function are not expected to return to the connection pool.
     def pop_connection
       @connection_mtx.synchronize do
-        conn = @connections.shift
-        @origin_conds.delete(conn.origin) if conn && (@origin_counters[conn.origin.to_s] -= 1).zero?
-        conn
+        drop_connection
       end
     end
 
@@ -44,13 +47,34 @@ module HTTPX
 
       @connection_mtx.synchronize do
         acquire_connection(uri, options) || begin
+          if @connections_counter == @max_connections
+            # this takes precedence over per-origin
+            @max_connections_cond.wait(@connection_mtx, @pool_timeout)
+
+            acquire_connection(uri, options) || begin
+              if @connections_counter == @max_connections
+                # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
+                # this means that all of them are persistent or being used, so raise a timeout error.
+                conn = @connections.find { |c| c.state == :closed }
+
+                raise PoolTimeoutError.new(@pool_timeout,
+                                           "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
+
+                drop_connection(conn)
+              end
+            end
+          end
+
           if @origin_counters[uri.origin] == @max_connections_per_origin
 
             @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
 
-            return acquire_connection(uri, options) || raise(PoolTimeoutError.new(uri.origin, @pool_timeout))
+            return acquire_connection(uri, options) ||
+                   raise(PoolTimeoutError.new(@pool_timeout,
+                                              "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
           end
 
+          @connections_counter += 1
           @origin_counters[uri.origin] += 1
 
           checkout_new_connection(uri, options)
@@ -64,6 +88,7 @@ module HTTPX
       @connection_mtx.synchronize do
         @connections << connection
 
+        @max_connections_cond.signal
         @origin_conds[connection.origin.to_s].signal
       end
     end
@@ -107,6 +132,15 @@ module HTTPX
       end
     end
 
+    # :nocov:
+    def inspect
+      "#<#{self.class}:#{object_id} " \
+        "@max_connections_per_origin=#{@max_connections_per_origin} " \
+        "@pool_timeout=#{@pool_timeout} " \
+        "@connections=#{@connections.size}>"
+    end
+    # :nocov:
+
     private
 
     def acquire_connection(uri, options)
@@ -114,7 +148,9 @@ module HTTPX
         connection.match?(uri, options)
       end
 
-      @connections.delete_at(idx) if idx
+      return unless idx
+
+      @connections.delete_at(idx)
     end
 
     def checkout_new_connection(uri, options)
@@ -128,5 +164,22 @@ module HTTPX
         resolver_type.new(options)
       end
     end
+
+    # drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),
+    # the first available connection from the pool will be dropped.
+    def drop_connection(connection = nil)
+      if connection
+        @connections.delete(connection)
+      else
+        connection = @connections.shift
+
+        return unless connection
+      end
+
+      @connections_counter -= 1
+      @origin_conds.delete(connection.origin) if (@origin_counters[connection.origin.to_s] -= 1).zero?
+
+      connection
+    end
   end
 end

data/lib/httpx/request/body.rb

@@ -116,7 +116,7 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<HTTPX::Request::Body:#{object_id} " \
+      "#<#{self.class}:#{object_id} " \
         "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
     end
     # :nocov: