httpx 1.5.1 → 1.6.0

data/lib/httpx/selector.rb

@@ -3,6 +3,21 @@
 require "io/wait"
 
 module HTTPX
+  #
+  # Implements the selector loop, where it registers and monitors "Selectable" objects.
+  #
+  # A Selectable object is an object which can calculate the **interests** (<tt>:r</tt>, <tt>:w</tt> or <tt>:rw</tt>,
+  # respectively "read", "write" or "read-write") it wants to monitor for, and returns (via <tt>to_io</tt> method) an
+  # IO object which can be passed to functions such as IO.select . More exhaustively, a Selectable **must** implement
+  # the following methods:
+  #
+  # state :: returns the state as a Symbol, must return <tt>:closed</tt> when disposed of resources.
+  # to_io :: returns the IO object.
+  # call :: gets called when the IO is ready.
+  # interests :: returns the current interests to monitor for, as described above.
+  # timeout :: returns nil or an integer, representing how long to wait for interests.
+  # handle_socket_timeout(Numeric) :: called when waiting for interest times out.
+  #
   class Selector
     extend Forwardable
 
@@ -123,38 +138,75 @@ module HTTPX
     private
 
     def select(interval, &block)
+      has_no_selectables = @selectables.empty?
       # do not cause an infinite loop here.
       #
       # this may happen if timeout calculation actually triggered an error which causes
       # the connections to be reaped (such as the total timeout error) before #select
       # gets called.
-      return if interval.nil? && @selectables.empty?
+      return if interval.nil? && has_no_selectables
 
-      return select_one(interval, &block) if @selectables.size == 1
-
-      select_many(interval, &block)
-    end
-
-    def select_many(interval, &block)
+      # @type var r: (selectable | Array[selectable])?
+      # @type var w: (selectable | Array[selectable])?
       r, w = nil
 
-      # first, we group IOs based on interest type. On call to #interests however,
-      # things might already happen, and new IOs might be registered, so we might
-      # have to start all over again. We do this until we group all selectables
       @selectables.delete_if do |io|
         interests = io.interests
 
+        is_closed = io.state == :closed
+
+        next(is_closed) if is_closed
+
         io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
 
-        (r ||= []) << io if READABLE.include?(interests)
-        (w ||= []) << io if WRITABLE.include?(interests)
+        if READABLE.include?(interests)
+          r = r.nil? ? io : (Array(r) << io)
+        end
+
+        if WRITABLE.include?(interests)
+          w = w.nil? ? io : (Array(w) << io)
+        end
 
-        io.state == :closed
+        is_closed
       end
 
-      # TODO: what to do if there are no selectables?
+      case r
+      when Array
+        w = Array(w) unless w.nil?
+
+        select_many(r, w, interval, &block)
+      when nil
+        case w
+        when Array
+          select_many(r, w, interval, &block)
+        when nil
+          return unless interval && has_no_selectables
+
+          # no selectables
+          # TODO: replace with sleep?
+          select_many(r, w, interval, &block)
+        else
+          select_one(w, :w, interval, &block)
+        end
 
-      readers, writers = IO.select(r, w, nil, interval)
+      else
+        case w
+        when Array
+          select_many(Array(r), w, interval, &block)
+        when nil
+          select_one(r, :r, interval, &block)
+        else
+          if r == w
+            select_one(r, :rw, interval, &block)
+          else
+            select_many(Array(r), Array(w), interval, &block)
+          end
+        end
+      end
+    end
+
+    def select_many(r, w, interval, &block)
+      readers, writers = ::IO.select(r, w, nil, interval)
 
       if readers.nil? && writers.nil? && interval
         [*r, *w].each { |io| io.handle_socket_timeout(interval) }
@@ -175,32 +227,20 @@ module HTTPX
       end
     end
 
-    def select_one(interval)
-      io = @selectables.first
-
-      return unless io
-
-      interests = io.interests
-
-      io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
-
-      result = case interests
-               when :r then io.to_io.wait_readable(interval)
-               when :w then io.to_io.wait_writable(interval)
-               when :rw then io.to_io.wait(interval, :read_write)
-               when nil then return
-      end
+    def select_one(io, interests, interval)
+      result =
+        case interests
+        when :r then io.to_io.wait_readable(interval)
+        when :w then io.to_io.wait_writable(interval)
+        when :rw then rw_wait(io, interval)
+        end
 
       unless result || interval.nil?
         io.handle_socket_timeout(interval) unless @is_timer_interval
         return
       end
-      # raise TimeoutError.new(interval, "timed out while waiting on select")
 
       yield io
-      # rescue IOError, SystemCallError
-      #   @selectables.reject!(&:closed?)
-      #   raise unless @selectables.empty?
     end
 
     def next_timeout
@@ -220,5 +260,23 @@ module HTTPX
 
       connection_interval
     end
+
+    if RUBY_ENGINE == "jruby"
+      def rw_wait(io, interval)
+        io.to_io.wait(interval, :read_write)
+      end
+    elsif IO.const_defined?(:READABLE)
+      def rw_wait(io, interval)
+        io.to_io.wait(IO::READABLE | IO::WRITABLE, interval)
+      end
+    else
+      def rw_wait(io, interval)
+        if interval
+          io.to_io.wait(interval, :read_write)
+        else
+          io.to_io.wait(:read_write)
+        end
+      end
+    end
   end
 end

data/lib/httpx/session.rb

@@ -71,12 +71,8 @@ module HTTPX
 
         select_connection(connection, selector)
       end
-      begin
-        @closing = true
-        selector.terminate
-      ensure
-        @closing = false
-      end
+
+      selector_close(selector)
     end
 
     # performs one, or multple requests; it accepts:
@@ -136,6 +132,9 @@ module HTTPX
     alias_method :select_resolver, :select_connection
 
     def deselect_connection(connection, selector, cloned = false)
+      connection.log(level: 2) do
+        "deregistering connection##{connection.object_id}(#{connection.state}) from selector##{selector.object_id}"
+      end
       selector.deregister(connection)
 
       # when connections coalesce
@@ -145,14 +144,19 @@ module HTTPX
 
       return if @closing && connection.state == :closed
 
+      connection.log(level: 2) { "check-in connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
       @pool.checkin_connection(connection)
     end
 
     def deselect_resolver(resolver, selector)
+      resolver.log(level: 2) do
+        "deregistering resolver##{resolver.object_id}(#{resolver.state}) from selector##{selector.object_id}"
+      end
       selector.deregister(resolver)
 
       return if @closing && resolver.closed?
 
+      resolver.log(level: 2) { "check-in resolver##{resolver.object_id}(#{resolver.state}) in pool##{@pool.object_id}" }
       @pool.checkin_resolver(resolver)
     end
 
@@ -194,7 +198,12 @@ module HTTPX
         end
       when :closing, :closed
         connection.idling
-        select_connection(connection, selector)
+        if connection.addresses?
+          select_connection(connection, selector)
+        else
+          # if addresses expired, resolve again
+          resolve_connection(connection, selector)
+        end
       else
         pin_connection(connection, selector)
       end
@@ -204,10 +213,23 @@ module HTTPX
 
     private
 
+    def selector_close(selector)
+      begin
+        @closing = true
+        selector.terminate
+      ensure
+        @closing = false
+      end
+    end
+
+    # tries deactivating connections in the +selector+, deregistering the ones that have been deactivated.
     def deactivate(selector)
-      selector.each_connection do |connection|
-        connection.deactivate
-        deselect_connection(connection, selector) if connection.state == :inactive
+      selector.each_connection.select do |c|
+        c.deactivate
+
+        c.state == :inactive
+      end.each do |c| # rubocop:disable Style/MultilineBlockChain
+        deselect_connection(c, selector)
       end
     end
 
@@ -221,7 +243,11 @@ module HTTPX
     def fetch_response(request, _selector, _options)
       response = request.response
 
-      response if response && response.finished?
+      return unless response && response.finished?
+
+      log(level: 2) { "response fetched" }
+
+      response
     end
 
     # sends the +request+ to the corresponding HTTPX::Connection
@@ -303,8 +329,7 @@ module HTTPX
 
     # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
     def receive_requests(requests, selector)
-      # @type var responses: Array[response]
-      responses = []
+      responses = [] # : Array[response]
 
       # guarantee ordered responses
       loop do
@@ -313,7 +338,7 @@ module HTTPX
         return responses unless request
 
         catch(:coalesced) { selector.next_tick } until (response = fetch_response(request, selector, request.options))
-        request.emit(:complete, response)
+        request.complete!(response)
 
         responses << response
         requests.shift
@@ -326,18 +351,36 @@ module HTTPX
         # handshake error, and the error responses have already been emitted, but there was no
         # opportunity to traverse the requests, hence we're returning only a fraction of the errors
         # we were supposed to. This effectively fetches the existing responses and return them.
-        while (request = requests.shift)
-          response = fetch_response(request, selector, request.options)
-          request.emit(:complete, response) if response
-          responses << response
+        exit_from_loop = true
+
+        requests_to_remove = [] # : Array[Request]
+
+        requests.each do |req|
+          response = fetch_response(req, selector, request.options)
+
+          if exit_from_loop && response
+            req.complete!(response)
+            responses << response
+            requests_to_remove << req
+          else
+            # fetch_response may resend requests. when that happens, we need to go back to the initial
+            # loop and process the selector. we still do a pass-through on the remainder of requests, so
+            # that every request that need to be resent, is resent.
+            exit_from_loop = false
+
+            raise Error, "something went wrong, responses not found and requests not resent" if selector.empty?
+          end
         end
-        break
+
+        break if exit_from_loop
+
+        requests -= requests_to_remove
       end
       responses
     end
 
     def resolve_connection(connection, selector)
-      if connection.addresses || connection.open?
+      if connection.addresses? || connection.open?
         #
         # there are two cases in which we want to activate initialization of
         # connection immediately:
@@ -380,14 +423,16 @@ module HTTPX
     end
 
     def find_resolver_for(connection, selector)
-      resolver = selector.find_resolver(connection.options)
-
-      unless resolver
-        resolver = @pool.checkout_resolver(connection.options)
-        resolver.current_session = self
-        resolver.current_selector = selector
+      if (resolver = selector.find_resolver(connection.options))
+        resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
+        return resolver
       end
 
+      resolver = @pool.checkout_resolver(connection.options)
+      resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
+      resolver.current_session = self
+      resolver.current_selector = selector
+
       resolver
     end
 
@@ -397,7 +442,10 @@ module HTTPX
       unless conn1.coalescable?(conn2)
         conn2.log(level: 2) { "not coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
         select_connection(conn2, selector)
-        @pool.checkin_connection(conn1) if from_pool
+        if from_pool
+          conn1.log(level: 2) { "check-in connection##{conn1.object_id}(#{conn1.state}) in pool##{@pool.object_id}" }
+          @pool.checkin_connection(conn1)
+        end
         return false
       end
 
@@ -422,13 +470,18 @@ module HTTPX
 
     def selector_store
       th_current = Thread.current
-      th_current.thread_variable_get(:httpx_persistent_selector_store) || begin
+
+      thread_selector_store(th_current) || begin
         {}.compare_by_identity.tap do |store|
           th_current.thread_variable_set(:httpx_persistent_selector_store, store)
         end
       end
     end
 
+    def thread_selector_store(th)
+      th.thread_variable_get(:httpx_persistent_selector_store)
+    end
+
     @default_options = Options.new
     @default_options.freeze
     @plugins = []
@@ -452,6 +505,8 @@ module HTTPX
         label = pl
         # raise Error, "Cannot add a plugin to a frozen config" if frozen?
         pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
+        raise ArgumentError, "Invalid plugin type: #{pl.class.inspect}" unless pl.is_a?(Module)
+
         if !@plugins.include?(pl)
           @plugins << pl
           pl.load_dependencies(self, &block) if pl.respond_to?(:load_dependencies)
@@ -463,9 +518,13 @@ module HTTPX
 
           opts = @default_options
           opts.extend_with_plugin_classes(pl)
-          if defined?(pl::OptionsMethods)
 
-            (pl::OptionsMethods.instance_methods - Object.instance_methods).each do |meth|
+          if defined?(pl::OptionsMethods)
+            # when a class gets dup'ed, the #initialize_dup callbacks isn't triggered.
+            # moreover, and because #method_added does not get triggered on mixin include,
+            # the callback is also forcefully manually called here.
+            opts.options_class.instance_variable_set(:@options_names, opts.options_class.options_names.dup)
+            (pl::OptionsMethods.instance_methods + pl::OptionsMethods.private_instance_methods - Object.instance_methods).each do |meth|
               opts.options_class.method_added(meth)
             end
             @default_options = opts.options_class.new(opts)

data/lib/httpx/session_extensions.rb

@@ -11,8 +11,9 @@ module HTTPX
     options = proxy_session.class.default_options.to_hash
     original_verbosity = $VERBOSE
     $VERBOSE = nil
-    const_set(:Options, proxy_session.class.default_options.options_class)
-    options[:options_class] = Class.new(options[:options_class])
+    new_options_class = proxy_session.class.default_options.options_class.dup
+    const_set(:Options, new_options_class)
+    options[:options_class] = Class.new(new_options_class).freeze
     options.freeze
     Options.send(:const_set, :DEFAULT_OPTIONS, options)
     Session.instance_variable_set(:@default_options, Options.new(options))

data/lib/httpx/transcoder/form.rb

@@ -48,11 +48,7 @@ module HTTPX
       end
 
       def encode(form)
-        if multipart?(form)
-          Multipart::Encoder.new(form)
-        else
-          Encoder.new(form)
-        end
+        Encoder.new(form)
       end
 
       def decode(response)
@@ -67,14 +63,6 @@ module HTTPX
           raise Error, "invalid form mime type (#{content_type})"
         end
       end
-
-      def multipart?(data)
-        data.any? do |_, v|
-          Multipart::MULTIPART_VALUE_COND.call(v) ||
-            (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
-            (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
-        end
-      end
     end
   end
 end

data/lib/httpx/transcoder/multipart/mime_type_detector.rb

@@ -44,7 +44,7 @@ module HTTPX
 
           Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
             begin
-              ::IO.copy_stream(file, stdin.binmode)
+              IO.copy_stream(file, stdin.binmode)
             rescue Errno::EPIPE
             end
             file.rewind

data/lib/httpx/transcoder/multipart.rb

@@ -13,5 +13,19 @@ module HTTPX::Transcoder
           value.key?(:body) &&
           (value.key?(:filename) || value.key?(:content_type)))
     end
+
+    module_function
+
+    def multipart?(form_data)
+      form_data.any? do |_, v|
+        Multipart::MULTIPART_VALUE_COND.call(v) ||
+          (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
+          (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
+      end
+    end
+
+    def encode(form_data)
+      Encoder.new(form_data)
+    end
   end
 end

data/lib/httpx/transcoder/utils/deflater.rb

@@ -63,7 +63,7 @@ module HTTPX
         buffer = Response::Buffer.new(
           threshold_size: Options::MAX_BODY_THRESHOLD_SIZE
         )
-        ::IO.copy_stream(self, buffer)
+        IO.copy_stream(self, buffer)
 
         buffer.rewind if buffer.respond_to?(:rewind)
 

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.5.1"
+  VERSION = "1.6.0"
 end

data/sig/callbacks.rbs

@@ -8,7 +8,7 @@ module HTTPX
     def once: (Symbol) { (*untyped) -> void } -> ^(*untyped) -> void
     def emit: (Symbol, *untyped) -> void
 
-    def callbacks_for?: (Symbol) -> bool
+    def callbacks_for?: (Symbol) -> boolish
     def callbacks: () -> Hash[Symbol, Array[_Callable]]
                   | (Symbol) -> Array[_Callable]
   end

data/sig/chainable.rbs

@@ -24,6 +24,7 @@ module HTTPX
               | (:upgrade, ?options) -> Plugins::sessionUpgrade
               | (:h2c, ?options) -> Plugins::sessionUpgrade
               | (:h2, ?options) -> Plugins::sessionUpgrade
+              | (:fiber_concurrency, ?options) -> Plugins::sessionFiberConcurrency
               | (:persistent, ?options) -> Plugins::sessionPersistent
               | (:proxy, ?options) -> (Plugins::sessionProxy & Plugins::httpProxy)
               | (:push_promise, ?options) -> Plugins::sessionPushPromise

data/sig/connection/http1.rbs

@@ -54,6 +54,8 @@ module HTTPX
 
     def ping: () -> void
 
+    def waiting_for_ping?: () -> bool
+
     def timeout: () -> Numeric?
 
     private

data/sig/connection/http2.rbs

@@ -37,6 +37,8 @@ module HTTPX
 
     def ping: () -> void
 
+    def waiting_for_ping?: () -> bool
+
     alias reset init_connection
 
     def timeout: () -> Numeric?
@@ -97,6 +99,8 @@ module HTTPX
 
     def on_pong: (string ping) -> void
 
+    def teardown: (?Request? request) -> void
+
     class Error < ::HTTPX::Error
       def initialize: (Integer id, Symbol | StandardError error) -> void
     end

data/sig/connection.rbs

@@ -50,17 +50,17 @@ module HTTPX
     @main_sibling: bool
 
 
-    def addresses: () -> Array[ipaddr]?
+    def addresses: () -> Array[Resolver::Entry]?
 
     def peer: () -> URI::Generic
 
-    def addresses=: (Array[ipaddr] addresses) -> void
+    def addresses=: (Array[Resolver::Entry] addresses) -> void
 
     def send: (Request request) -> void
 
-    def match?: (URI::Generic uri, Options options) -> bool
+    def addresses?: () -> boolish
 
-    def expired?: () -> boolish
+    def match?: (URI::Generic uri, Options options) -> bool
 
     def mergeable?: (Connection connection) -> bool
 
@@ -82,7 +82,7 @@ module HTTPX
 
     def interests: () -> io_interests?
 
-    def to_io: () -> ::IO
+    def to_io: () -> IO
 
     def call: () -> void
 
@@ -140,7 +140,7 @@ module HTTPX
 
     def build_altsvc_connection: (URI::Generic alt_origin, String origin, Hash[String, String] alt_params) -> void
 
-    def build_socket: (?Array[ipaddr]? addrs) -> (TCP | SSL | UNIX)
+    def build_socket: (?Array[Resolver::Entry]? addrs) -> (TCP | SSL | UNIX)
 
     def on_error: (HTTPX::TimeoutError | Error | StandardError error, ?Request? request) -> void
 

data/sig/errors.rbs

@@ -51,13 +51,15 @@ module HTTPX
   class HTTPError < Error
     attr_reader response: Response
 
+    def status: () -> Integer
+
     private
 
     def initialize: (Response response) -> void
   end
 
   class NativeResolveError < ResolveError
-    attr_reader connection: Connection
+    attr_accessor connection: Connection?
     attr_reader host: String
 
     private

data/sig/io/ssl.rbs

@@ -12,7 +12,7 @@ module HTTPX
     attr_writer ssl_session: OpenSSL::SSL::Session?
 
     # TODO: lift when https://github.com/ruby/rbs/issues/1497 fixed
-    # def initialize: (URI::Generic origin, Array[ipaddr]? addresses, options options) ?{ (self) -> void } -> void
+    # def initialize: (URI::Generic origin, Array[Resolver::Entry]? addresses, options options) ?{ (self) -> void } -> void
 
     def session_new_cb: { (OpenSSL::SSL::Session sess) -> void } -> void
     def can_verify_peer?: () -> bool

data/sig/io/tcp.rbs

@@ -2,11 +2,11 @@ module HTTPX
   class TCP
     include Loggable
 
-    attr_reader ip: ipaddr?
+    attr_reader ip: Resolver::Entry?
 
     attr_reader port: Integer
 
-    attr_reader addresses: Array[ipaddr]
+    attr_reader addresses: Array[Resolver::Entry]
 
     attr_reader state: Symbol
 
@@ -14,18 +14,26 @@ module HTTPX
 
     alias host ip
 
+    @io: Socket
+
     @hostname: String
 
     @options: Options
 
     @fallback_protocol: String
 
+    @keep_open: bool
+
+    @ip_index: Integer
+
     # TODO: lift when https://github.com/ruby/rbs/issues/1497 fixed
-    def initialize: (URI::Generic origin, Array[ipaddr]? addresses, Options options) ?{ (instance) -> void } -> void
+    def initialize: (URI::Generic origin, Array[Resolver::Entry]? addresses, Options options) ?{ (instance) -> void } -> void
 
-    def add_addresses: (Array[ipaddr] addrs) -> void
+    def add_addresses: (Array[Resolver::Entry] addrs) -> void
 
-    def to_io: () -> ::IO
+    def addresses?: () -> bool
+
+    def to_io: () -> IO
 
     def protocol: () -> String
 
@@ -46,8 +54,6 @@ module HTTPX
 
     def connected?: () -> bool
 
-    def expired?: () -> boolish
-
     def closed?: () -> bool
 
     # :nocov: