httpx 1.4.4 → 1.5.0

data/lib/httpx/plugins/stream.rb

@@ -2,6 +2,8 @@
 
 module HTTPX
   class StreamResponse
+    attr_reader :request
+
     def initialize(request, session)
       @request = request
       @options = @request.options
@@ -71,7 +73,7 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<StreamResponse:#{object_id}>"
+      "#<#{self.class}:#{object_id}>"
     end
     # :nocov:
 
@@ -114,7 +116,7 @@ module HTTPX
 
   module Plugins
     #
-    # This plugin adds support for stream response (text/event-stream).
+    # This plugin adds support for streaming a response (useful for i.e. "text/event-stream" payloads).
     #
     # https://gitlab.com/os85/httpx/wikis/Stream
     #

data/lib/httpx/plugins/stream_bidi.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for bidirectional HTTP/2 streams.
+    #
+    # https://gitlab.com/os85/httpx/wikis/StreamBidi
+    #
+    # It is required that the request body allows chunk to be buffered, (i.e., responds to +#<<(chunk)+).
+    module StreamBidi
+      # Extension of the Connection::HTTP2 class, which adds functionality to
+      # deal with a request that can't be drained and must be interleaved with
+      # the response streams.
+      #
+      # The streams keeps send DATA frames while there's data; when they're ain't,
+      # the stream is kept open; it must be explicitly closed by the end user.
+      #
+      class HTTP2Bidi < Connection::HTTP2
+        def initialize(*)
+          super
+          @lock = Thread::Mutex.new
+        end
+
+        %i[close empty? exhausted? send <<].each do |lock_meth|
+          class_eval(<<-METH, __FILE__, __LINE__ + 1)
+            # lock.aware version of +#{lock_meth}+
+            def #{lock_meth}(*)                # def close(*)
+              return super if @lock.owned?
+
+              # small race condition between
+              # checking for ownership and
+              # acquiring lock.
+              # TODO: fix this at the parser.
+              @lock.synchronize { super }
+            end
+          METH
+        end
+
+        private
+
+        %i[join_headers join_trailers join_body].each do |lock_meth|
+          class_eval(<<-METH, __FILE__, __LINE__ + 1)
+            # lock.aware version of +#{lock_meth}+
+            def #{lock_meth}(*)                # def join_headers(*)
+              return super if @lock.owned?
+
+              # small race condition between
+              # checking for ownership and
+              # acquiring lock.
+              # TODO: fix this at the parser.
+              @lock.synchronize { super }
+            end
+          METH
+        end
+
+        def handle_stream(stream, request)
+          request.on(:body) do
+            next unless request.headers_sent
+
+            handle(request, stream)
+
+            emit(:flush_buffer)
+          end
+          super
+        end
+
+        # when there ain't more chunks, it makes the buffer as full.
+        def send_chunk(request, stream, chunk, next_chunk)
+          super
+
+          return if next_chunk
+
+          request.transition(:waiting_for_chunk)
+          throw(:buffer_full)
+        end
+
+        # sets end-stream flag when the request is closed.
+        def end_stream?(request, next_chunk)
+          request.closed? && next_chunk.nil?
+        end
+      end
+
+      # BidiBuffer is a Buffer which can be receive data from threads othr
+      # than the thread of the corresponding Connection/Session.
+      #
+      # It synchronizes access to a secondary internal +@oob_buffer+, which periodically
+      # is reconciled to the main internal +@buffer+.
+      class BidiBuffer < Buffer
+        def initialize(*)
+          super
+          @parent_thread = Thread.current
+          @oob_mutex = Thread::Mutex.new
+          @oob_buffer = "".b
+        end
+
+        # buffers the +chunk+ to be sent
+        def <<(chunk)
+          return super if Thread.current == @parent_thread
+
+          @oob_mutex.synchronize { @oob_buffer << chunk }
+        end
+
+        # reconciles the main and secondary buffer (which receives data from other threads).
+        def rebuffer
+          raise Error, "can only rebuffer while waiting on a response" unless Thread.current == @parent_thread
+
+          @oob_mutex.synchronize do
+            @buffer << @oob_buffer
+            @oob_buffer.clear
+          end
+        end
+      end
+
+      # Proxy to wake up the session main loop when one
+      # of the connections has buffered data to write. It abides by the HTTPX::_Selectable API,
+      # which allows it to be registered in the selector alongside actual HTTP-based
+      # HTTPX::Connection objects.
+      class Signal
+        def initialize
+          @closed = false
+          @pipe_read, @pipe_write = ::IO.pipe
+        end
+
+        def state
+          @closed ? :closed : :open
+        end
+
+        # noop
+        def log(**); end
+
+        def to_io
+          @pipe_read.to_io
+        end
+
+        def wakeup
+          return if @closed
+
+          @pipe_write.write("\0")
+        end
+
+        def call
+          return if @closed
+
+          @pipe_read.readpartial(1)
+        end
+
+        def interests
+          return if @closed
+
+          :r
+        end
+
+        def timeout; end
+
+        def terminate
+          @pipe_write.close
+          @pipe_read.close
+          @closed = true
+        end
+
+        # noop (the owner connection will take of it)
+        def handle_socket_timeout(interval); end
+      end
+
+      class << self
+        def load_dependencies(klass)
+          klass.plugin(:stream)
+        end
+
+        def extra_options(options)
+          options.merge(fallback_protocol: "h2")
+        end
+      end
+
+      module InstanceMethods
+        def initialize(*)
+          @signal = Signal.new
+          super
+        end
+
+        def close(selector = Selector.new)
+          @signal.terminate
+          selector.deregister(@signal)
+          super(selector)
+        end
+
+        def select_connection(connection, selector)
+          super
+          selector.register(@signal)
+          connection.signal = @signal
+        end
+
+        def deselect_connection(connection, *)
+          super
+          connection.signal = nil
+        end
+      end
+
+      # Adds synchronization to request operations which may buffer payloads from different
+      # threads.
+      module RequestMethods
+        attr_accessor :headers_sent
+
+        def initialize(*)
+          super
+          @headers_sent = false
+          @closed = false
+          @mutex = Thread::Mutex.new
+        end
+
+        def closed?
+          @closed
+        end
+
+        def can_buffer?
+          super && @state != :waiting_for_chunk
+        end
+
+        # overrides state management transitions to introduce an intermediate
+        # +:waiting_for_chunk+ state, which the request transitions to once payload
+        # is buffered.
+        def transition(nextstate)
+          headers_sent = @headers_sent
+
+          case nextstate
+          when :waiting_for_chunk
+            return unless @state == :body
+          when :body
+            case @state
+            when :headers
+              headers_sent = true
+            when :waiting_for_chunk
+              # HACK: to allow super to pass through
+              @state = :headers
+            end
+          end
+
+          super.tap do
+            # delay setting this up until after the first transition to :body
+            @headers_sent = headers_sent
+          end
+        end
+
+        def <<(chunk)
+          @mutex.synchronize do
+            if @drainer
+              @body.clear if @body.respond_to?(:clear)
+              @drainer = nil
+            end
+            @body << chunk
+
+            transition(:body)
+          end
+        end
+
+        def close
+          @mutex.synchronize do
+            return if @closed
+
+            @closed = true
+          end
+
+          # last chunk to send which ends the stream
+          self << ""
+        end
+      end
+
+      module RequestBodyMethods
+        def initialize(*, **)
+          super
+          @headers.delete("content-length")
+        end
+
+        def empty?
+          false
+        end
+      end
+
+      # overrides the declaration of +@write_buffer+, which is now a thread-safe buffer
+      # responding to the same API.
+      module ConnectionMethods
+        attr_writer :signal
+
+        def initialize(*)
+          super
+          @write_buffer = BidiBuffer.new(@options.buffer_size)
+        end
+
+        # rebuffers the +@write_buffer+ before calculating interests.
+        def interests
+          @write_buffer.rebuffer
+
+          super
+        end
+
+        private
+
+        def parser_type(protocol)
+          return HTTP2Bidi if protocol == "h2"
+
+          super
+        end
+
+        def set_parser_callbacks(parser)
+          super
+          parser.on(:flush_buffer) do
+            @signal.wakeup if @signal
+          end
+        end
+      end
+    end
+    register_plugin :stream_bidi, StreamBidi
+  end
+end

data/lib/httpx/pool.rb

@@ -13,25 +13,28 @@ module HTTPX
 
     # Sets up the connection pool with the given +options+, which can be the following:
     #
+    # :max_connections:: the maximum number of connections held in the pool.
     # :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
     # :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
     #
     def initialize(options)
+      @max_connections = options.fetch(:max_connections, Float::INFINITY)
       @max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
       @pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
       @resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
       @resolver_mtx = Thread::Mutex.new
       @connections = []
       @connection_mtx = Thread::Mutex.new
+      @connections_counter = 0
+      @max_connections_cond = ConditionVariable.new
       @origin_counters = Hash.new(0)
       @origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
     end
 
+    # connections returned by this function are not expected to return to the connection pool.
     def pop_connection
       @connection_mtx.synchronize do
-        conn = @connections.shift
-        @origin_conds.delete(conn.origin) if conn && (@origin_counters[conn.origin.to_s] -= 1).zero?
-        conn
+        drop_connection
       end
     end
 
@@ -44,13 +47,34 @@ module HTTPX
 
       @connection_mtx.synchronize do
         acquire_connection(uri, options) || begin
+          if @connections_counter == @max_connections
+            # this takes precedence over per-origin
+            @max_connections_cond.wait(@connection_mtx, @pool_timeout)
+
+            acquire_connection(uri, options) || begin
+              if @connections_counter == @max_connections
+                # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
+                # this means that all of them are persistent or being used, so raise a timeout error.
+                conn = @connections.find { |c| c.state == :closed }
+
+                raise PoolTimeoutError.new(@pool_timeout,
+                                           "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
+
+                drop_connection(conn)
+              end
+            end
+          end
+
           if @origin_counters[uri.origin] == @max_connections_per_origin
 
             @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
 
-            return acquire_connection(uri, options) || raise(PoolTimeoutError.new(uri.origin, @pool_timeout))
+            return acquire_connection(uri, options) ||
+                   raise(PoolTimeoutError.new(@pool_timeout,
+                                              "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
           end
 
+          @connections_counter += 1
           @origin_counters[uri.origin] += 1
 
           checkout_new_connection(uri, options)
@@ -64,6 +88,7 @@ module HTTPX
       @connection_mtx.synchronize do
         @connections << connection
 
+        @max_connections_cond.signal
         @origin_conds[connection.origin.to_s].signal
       end
     end
@@ -107,6 +132,15 @@ module HTTPX
       end
     end
 
+    # :nocov:
+    def inspect
+      "#<#{self.class}:#{object_id} " \
+        "@max_connections_per_origin=#{@max_connections_per_origin} " \
+        "@pool_timeout=#{@pool_timeout} " \
+        "@connections=#{@connections.size}>"
+    end
+    # :nocov:
+
     private
 
     def acquire_connection(uri, options)
@@ -114,7 +148,9 @@ module HTTPX
         connection.match?(uri, options)
       end
 
-      @connections.delete_at(idx) if idx
+      return unless idx
+
+      @connections.delete_at(idx)
     end
 
     def checkout_new_connection(uri, options)
@@ -128,5 +164,22 @@ module HTTPX
         resolver_type.new(options)
       end
     end
+
+    # drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),
+    # the first available connection from the pool will be dropped.
+    def drop_connection(connection = nil)
+      if connection
+        @connections.delete(connection)
+      else
+        connection = @connections.shift
+
+        return unless connection
+      end
+
+      @connections_counter -= 1
+      @origin_conds.delete(connection.origin) if (@origin_counters[connection.origin.to_s] -= 1).zero?
+
+      connection
+    end
   end
 end

data/lib/httpx/request/body.rb

@@ -116,7 +116,7 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<HTTPX::Request::Body:#{object_id} " \
+      "#<#{self.class}:#{object_id} " \
         "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
     end
     # :nocov:

data/lib/httpx/request.rb

@@ -155,6 +155,10 @@ module HTTPX
       :w
     end
 
+    def can_buffer?
+      @state != :done
+    end
+
     # merges +h+ into the instance of HTTPX::Headers of the request.
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -222,7 +226,7 @@ module HTTPX
       return @query if defined?(@query)
 
       query = []
-      if (q = @query_params)
+      if (q = @query_params) && !q.empty?
         query << Transcoder::Form.encode(q)
       end
       query << @uri.query if @uri.query
@@ -247,7 +251,7 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<HTTPX::Request:#{object_id} " \
+      "#<#{self.class}:#{object_id} " \
         "#{@verb} " \
         "#{uri} " \
         "@headers=#{@headers} " \

data/lib/httpx/resolver/https.rb

@@ -2,11 +2,14 @@
 
 require "resolv"
 require "uri"
-require "cgi"
 require "forwardable"
 require "httpx/base64"
 
 module HTTPX
+  # Implementation of a DoH name resolver (https://www.youtube.com/watch?v=unMXvnY2FNM).
+  # It wraps an HTTPX::Connection object which integrates with the main session in the
+  # same manner as other performed HTTP requests.
+  #
   class Resolver::HTTPS < Resolver::Resolver
     extend Forwardable
     using URIExtensions
@@ -27,14 +30,13 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?, :handle_socket_timeout
 
     def initialize(_, options)
       super
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
       @queries = {}
       @requests = {}
-      @connections = []
       @uri = URI(@resolver_options[:uri])
       @uri_addresses = nil
       @resolver = Resolv::DNS.new
@@ -75,7 +77,11 @@ module HTTPX
 
     private
 
-    def resolve(connection = @connections.first, hostname = nil)
+    def resolve(connection = nil, hostname = nil)
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      connection ||= @connections.first
+
       return unless connection
 
       hostname ||= @queries.key(connection)

data/lib/httpx/resolver/native.rb

@@ -4,6 +4,9 @@ require "forwardable"
 require "resolv"
 
 module HTTPX
+  # Implements a pure ruby name resolver, which abides by the Selectable API.
+  # It delegates DNS payload encoding/decoding to the +resolv+ stlid gem.
+  #
   class Resolver::Native < Resolver::Resolver
     extend Forwardable
     using URIExtensions
@@ -34,7 +37,6 @@ module HTTPX
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-      @connections = []
       @name = nil
       @queries = {}
       @read_buffer = "".b
@@ -46,6 +48,10 @@ module HTTPX
       transition(:closed)
     end
 
+    def terminate
+      emit(:close, self)
+    end
+
     def closed?
       @state == :closed
     end
@@ -120,10 +126,7 @@ module HTTPX
       @ns_index += 1
       nameserver = @nameserver
       if nameserver && @ns_index < nameserver.size
-        log do
-          "resolver #{FAMILY_TYPES[@record_type]}: " \
-            "failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
-        end
+        log { "resolver #{FAMILY_TYPES[@record_type]}: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
         transition(:idle)
         @timeouts.clear
         retry
@@ -158,9 +161,7 @@ module HTTPX
       timeouts = @timeouts[h]
 
       if !timeouts.empty?
-        log do
-          "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..."
-        end
+        log { "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..." }
         # must downgrade to tcp AND retry on same host as last
         downgrade_socket
         resolve(connection, h)
@@ -388,10 +389,9 @@ module HTTPX
 
       if hostname.nil?
         hostname = connection.peer.host
-        log do
-          "resolver #{FAMILY_TYPES[@record_type]}: " \
-            "resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
-        end if connection.peer.non_ascii_hostname
+        if connection.peer.non_ascii_hostname
+          log { "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" }
+        end
 
         hostname = generate_candidates(hostname).each do |name|
           @queries[name] = connection
@@ -507,7 +507,7 @@ module HTTPX
         end
 
         while (connection = @connections.shift)
-          emit_resolve_error(connection, host, error)
+          emit_resolve_error(connection, connection.peer.host, error)
         end
       end
     end

data/lib/httpx/resolver/resolver.rb

@@ -4,6 +4,9 @@ require "resolv"
 require "ipaddr"
 
 module HTTPX
+  # Base class for all internal internet name resolvers. It handles basic blocks
+  # from the Selectable API.
+  #
   class Resolver::Resolver
     include Callbacks
     include Loggable
@@ -36,6 +39,7 @@ module HTTPX
       @family = family
       @record_type = RECORD_TYPES[family]
       @options = options
+      @connections = []
 
       set_resolver_callbacks
     end