httpx 1.4.4 → 1.5.0

data/lib/httpx/resolver/system.rb

@@ -3,6 +3,15 @@
 require "resolv"
 
 module HTTPX
+  # Implementation of a synchronous name resolver which relies on the system resolver,
+  # which is lib'c getaddrinfo function (abstracted in ruby via Addrinfo.getaddrinfo).
+  #
+  # Its main advantage is relying on the reference implementation for name resolution
+  # across most/all OSs which deploy ruby (it's what TCPSocket also uses), its main
+  # disadvantage is the inability to set timeouts / check socket for readiness events,
+  # hence why it relies on using the Timeout module, which poses a lot of problems for
+  # the selector loop, specially when network is unstable.
+  #
   class Resolver::System < Resolver::Resolver
     using URIExtensions
 
@@ -23,14 +32,13 @@ module HTTPX
     attr_reader :state
 
     def initialize(options)
-      super(nil, options)
+      super(0, options)
       @resolver_options = @options.resolver_options
       resolv_options = @resolver_options.dup
       timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
       @_timeouts = Array(timeouts)
       @timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
       resolv_options.delete(:cache)
-      @connections = []
       @queries = []
       @ips = []
       @pipe_mutex = Thread::Mutex.new
@@ -100,7 +108,14 @@ module HTTPX
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
-      on_error(error)
+      @queries.each do |host, connection|
+        @connections.delete(connection)
+        emit_resolve_error(connection, host, error)
+      end
+
+      while (connection = @connections.shift)
+        emit_resolve_error(connection, connection.peer.host, error)
+      end
     end
 
     private
@@ -131,19 +146,22 @@ module HTTPX
         case event
         when DONE
           *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
-          @queries.delete(pair)
-          _, connection = pair
-          @connections.delete(connection)
+          if pair
+            @queries.delete(pair)
+            family, connection = pair
+            @connections.delete(connection)
 
-          family, connection = pair
-          catch(:coalesced) { emit_addresses(connection, family, addrs) }
+            catch(:coalesced) { emit_addresses(connection, family, addrs) }
+          end
         when ERROR
           *pair, error = @pipe_mutex.synchronize { @ips.pop }
-          @queries.delete(pair)
-          @connections.delete(connection)
+          if pair && error
+            @queries.delete(pair)
+            @connections.delete(connection)
 
-          _, connection = pair
-          emit_resolve_error(connection, connection.peer.host, error)
+            _, connection = pair
+            emit_resolve_error(connection, connection.peer.host, error)
+          end
         end
       end
 
@@ -152,11 +170,16 @@ module HTTPX
       resolve
     end
 
-    def resolve(connection = @connections.first)
+    def resolve(connection = nil, hostname = nil)
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      connection ||= @connections.first
+
       raise Error, "no URI to resolve" unless connection
+
       return unless @queries.empty?
 
-      hostname = connection.peer.host
+      hostname ||= connection.peer.host
       scheme = connection.origin.scheme
       log do
         "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"

data/lib/httpx/resolver.rb

@@ -92,8 +92,8 @@ module HTTPX
       end
 
       ips = entries.flat_map do |address|
-        if address.key?("alias")
-          lookup(address["alias"], lookups, ttl)
+        if (als = address["alias"])
+          lookup(als, lookups, ttl)
         else
           IPAddr.new(address["data"])
         end

data/lib/httpx/response/body.rb

@@ -11,6 +11,9 @@ module HTTPX
     # Array of encodings contained in the response "content-encoding" header.
     attr_reader :encodings
 
+    attr_reader :buffer
+    protected :buffer
+
     # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
     def initialize(response, options)
       @response = response
@@ -148,18 +151,17 @@ module HTTPX
     end
 
     def ==(other)
-      object_id == other.object_id || begin
-        if other.respond_to?(:read)
-          _with_same_buffer_pos { FileUtils.compare_stream(@buffer, other) }
-        else
-          to_s == other.to_s
-        end
-      end
+      super || case other
+               when Response::Body
+                 @buffer == other.buffer
+               else
+                 @buffer = other
+               end
     end
 
     # :nocov:
     def inspect
-      "#<HTTPX::Response::Body:#{object_id} " \
+      "#<#{self.class}:#{object_id} " \
         "@state=#{@state} " \
         "@length=#{@length}>"
     end
@@ -226,19 +228,6 @@ module HTTPX
       @state = nextstate
     end
 
-    def _with_same_buffer_pos # :nodoc:
-      return yield unless @buffer && @buffer.respond_to?(:pos)
-
-      # @type ivar @buffer: StringIO | Tempfile
-      current_pos = @buffer.pos
-      @buffer.rewind
-      begin
-        yield
-      ensure
-        @buffer.pos = current_pos
-      end
-    end
-
     class << self
       def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
         case encoding

data/lib/httpx/response/buffer.rb

@@ -7,6 +7,9 @@ require "tempfile"
 module HTTPX
   # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
   class Response::Buffer < SimpleDelegator
+    attr_reader :buffer
+    protected :buffer
+
     # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
     # the initial +bytesize+, and the +encoding+.
     def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
@@ -20,7 +23,14 @@ module HTTPX
     def initialize_dup(other)
       super
 
-      @buffer = other.instance_variable_get(:@buffer).dup
+      # create new descriptor in READ-ONLY mode
+      @buffer =
+        case other.buffer
+        when StringIO
+          StringIO.new(other.buffer.string, mode: File::RDONLY)
+        else
+          other.buffer.class.new(other.buffer.path, encoding: Encoding::BINARY, mode: File::RDONLY)
+        end
     end
 
     # size in bytes of the buffered content.
@@ -46,7 +56,7 @@ module HTTPX
         end
       when Tempfile
         rewind
-        content = _with_same_buffer_pos { @buffer.read }
+        content = @buffer.read
         begin
           content.force_encoding(@encoding)
         rescue ArgumentError # ex: unknown encoding name - utf
@@ -61,6 +71,30 @@ module HTTPX
       @buffer.unlink if @buffer.respond_to?(:unlink)
     end
 
+    def ==(other)
+      super || begin
+        return false unless other.is_a?(Response::Buffer)
+
+        if @buffer.nil?
+          other.buffer.nil?
+        elsif @buffer.respond_to?(:read) &&
+              other.respond_to?(:read)
+          buffer_pos = @buffer.pos
+          other_pos = other.buffer.pos
+          @buffer.rewind
+          other.buffer.rewind
+          begin
+            FileUtils.compare_stream(@buffer, other.buffer)
+          ensure
+            @buffer.pos = buffer_pos
+            other.buffer.pos = other_pos
+          end
+        else
+          to_s == other.to_s
+        end
+      end
+    end
+
     private
 
     # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
@@ -82,15 +116,5 @@ module HTTPX
 
       __setobj__(@buffer)
     end
-
-    def _with_same_buffer_pos # :nodoc:
-      current_pos = @buffer.pos
-      @buffer.rewind
-      begin
-        yield
-      ensure
-        @buffer.pos = current_pos
-      end
-    end
   end
 end

data/lib/httpx/response.rb

@@ -71,6 +71,14 @@ module HTTPX
       @content_type = nil
     end
 
+    # dupped initialization
+    def initialize_dup(orig)
+      super
+      # if a response gets dupped, the body handle must also get dupped to prevent
+      # two responses from using the same file handle to read.
+      @body = orig.body.dup
+    end
+
     # closes the respective +@request+ and +@body+.
     def close
       @request.close
@@ -126,7 +134,7 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<Response:#{object_id} " \
+      "#<#{self.class}:#{object_id} " \
         "HTTP/#{version} " \
         "@status=#{@status} " \
         "@headers=#{@headers} " \
@@ -275,6 +283,8 @@ module HTTPX
       true
     end
 
+    def finish!; end
+
     # raises the wrapped exception.
     def raise_for_status
       raise @error

data/lib/httpx/selector.rb

@@ -35,14 +35,21 @@ module HTTPX
         end
 
         begin
-          select(timeout, &:call)
+          select(timeout) do |c|
+            c.log(level: 2) { "[#{c.state}] selected#{" after #{timeout} secs" unless timeout.nil?}..." }
+
+            c.call
+          end
+
           @timers.fire
         rescue TimeoutError => e
           @timers.fire(e)
         end
       end
     rescue StandardError => e
-      emit_error(e)
+      each_connection do |c|
+        c.emit(:error, e)
+      end
     rescue Exception # rubocop:disable Lint/RescueException
       each_connection do |conn|
         conn.force_reset
@@ -77,9 +84,10 @@ module HTTPX
       return enum_for(__method__) unless block
 
       @selectables.each do |c|
-        if c.is_a?(Resolver::Resolver)
+        case c
+        when Resolver::Resolver
           c.each_connection(&block)
-        else
+        when Connection
           yield c
         end
       end
@@ -133,6 +141,8 @@ module HTTPX
       @selectables.delete_if do |io|
         interests = io.interests
 
+        io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
+
         (r ||= []) << io if READABLE.include?(interests)
         (w ||= []) << io if WRITABLE.include?(interests)
 
@@ -169,6 +179,8 @@ module HTTPX
 
       interests = io.interests
 
+      io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
+
       result = case interests
                when :r then io.to_io.wait_readable(interval)
                when :w then io.to_io.wait_writable(interval)
@@ -205,13 +217,5 @@ module HTTPX
 
       connection_interval
     end
-
-    def emit_error(e)
-      @selectables.each do |c|
-        next if c.is_a?(Resolver::Resolver)
-
-        c.emit(:error, e)
-      end
-    end
   end
 end

data/lib/httpx/session.rb

@@ -15,11 +15,11 @@ module HTTPX
     # When pass a block, it'll yield itself to it, then closes after the block is evaluated.
     def initialize(options = EMPTY_HASH, &blk)
       @options = self.class.default_options.merge(options)
-      @responses = {}
       @persistent = @options.persistent
       @pool = @options.pool_class.new(@options.pool_options)
       @wrapped = false
       @closing = false
+      INSTANCES[self] = self if @persistent && @options.close_on_fork && INSTANCES
       wrap(&blk) if blk
     end
 
@@ -174,11 +174,15 @@ module HTTPX
     # returns the HTTPX::Connection through which the +request+ should be sent through.
     def find_connection(request_uri, selector, options)
       if (connection = selector.find_connection(request_uri, options))
+        connection.idling if connection.state == :closed
+        connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
         return connection
       end
 
       connection = @pool.checkout_connection(request_uri, options)
 
+      connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
+
       case connection.state
       when :idle
         do_init_connection(connection, selector)
@@ -207,11 +211,6 @@ module HTTPX
       end
     end
 
-    # callback executed when a response for a given request has been received.
-    def on_response(request, response)
-      @responses[request] = response
-    end
-
     # callback executed when an HTTP/2 promise frame has been received.
     def on_promise(_, stream)
       log(level: 2) { "#{stream.id}: refusing stream!" }
@@ -220,7 +219,9 @@ module HTTPX
 
     # returns the corresponding HTTP::Response to the given +request+ if it has been received.
     def fetch_response(request, _selector, _options)
-      @responses.delete(request)
+      response = request.response
+
+      response if response && response.finished?
     end
 
     # sends the +request+ to the corresponding HTTPX::Connection
@@ -237,7 +238,9 @@ module HTTPX
 
       raise error unless error.is_a?(Error)
 
-      request.emit(:response, ErrorResponse.new(request, error))
+      response = ErrorResponse.new(request, error)
+      request.response = response
+      request.emit(:response, response)
     end
 
     # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
@@ -267,7 +270,6 @@ module HTTPX
     end
 
     def set_request_callbacks(request)
-      request.on(:response, &method(:on_response).curry(2)[request])
       request.on(:promise, &method(:on_promise))
     end
 
@@ -362,15 +364,12 @@ module HTTPX
 
       return select_connection(connection, selector) unless found_connection
 
-      if found_connection.open?
-        coalesce_connections(found_connection, connection, selector, from_pool)
-      else
-        found_connection.once(:open) do
-          next unless found_connection.current_session == self
-
-          coalesce_connections(found_connection, connection, selector, from_pool)
-        end
+      connection.log(level: 2) do
+        "try coalescing from #{from_pool ? "pool##{@pool.object_id}" : "selector##{selector.object_id}"} " \
+          "(conn##{found_connection.object_id}[#{found_connection.origin}])"
       end
+
+      coalesce_connections(found_connection, connection, selector, from_pool)
     end
 
     def on_resolver_close(resolver, selector)
@@ -396,14 +395,16 @@ module HTTPX
     # (it is known via +from_pool+), then it adds its to the +selector+.
     def coalesce_connections(conn1, conn2, selector, from_pool)
       unless conn1.coalescable?(conn2)
+        conn2.log(level: 2) { "not coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
         select_connection(conn2, selector)
         @pool.checkin_connection(conn1) if from_pool
         return false
       end
 
-      conn2.coalesced_connection = conn1
+      conn2.log(level: 2) { "coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
+      conn2.coalesce!(conn1)
       select_connection(conn1, selector) if from_pool
-      deselect_connection(conn2, selector)
+      conn2.disconnect
       true
     end
 
@@ -448,6 +449,7 @@ module HTTPX
       #   session_with_custom = session.plugin(CustomPlugin)
       #
       def plugin(pl, options = nil, &block)
+        label = pl
         # raise Error, "Cannot add a plugin to a frozen config" if frozen?
         pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
         if !@plugins.include?(pl)
@@ -472,9 +474,36 @@ module HTTPX
           @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
           @default_options = @default_options.merge(options) if options
 
+          if pl.respond_to?(:subplugins)
+            pl.subplugins.transform_keys(&Plugins.method(:load_plugin)).each do |main_pl, sub_pl|
+              # in case the main plugin has already been loaded, then apply subplugin functionality
+              # immediately
+              next unless @plugins.include?(main_pl)
+
+              plugin(sub_pl, options, &block)
+            end
+          end
+
           pl.configure(self, &block) if pl.respond_to?(:configure)
 
+          if label.is_a?(Symbol)
+            # in case an already-loaded plugin complements functionality of
+            # the plugin currently being loaded, loaded it now
+            @plugins.each do |registered_pl|
+              next if registered_pl == pl
+
+              next unless registered_pl.respond_to?(:subplugins)
+
+              sub_pl = registered_pl.subplugins[label]
+
+              next unless sub_pl
+
+              plugin(sub_pl, options, &block)
+            end
+          end
+
           @default_options.freeze
+          set_temporary_name("#{superclass}/#{pl}") if respond_to?(:set_temporary_name) # ruby 3.4 only
         elsif options
           # this can happen when two plugins are loaded, an one of them calls the other under the hood,
           # albeit changing some default.
@@ -483,9 +512,40 @@ module HTTPX
 
           @default_options.freeze
         end
+
         self
       end
     end
+
+    # setup of the support for close_on_fork sessions.
+    # adapted from https://github.com/mperham/connection_pool/blob/main/lib/connection_pool.rb#L48
+    if Process.respond_to?(:fork)
+      INSTANCES = ObjectSpace::WeakMap.new
+      private_constant :INSTANCES
+
+      def self.after_fork
+        INSTANCES.each_value(&:close)
+        nil
+      end
+
+      if ::Process.respond_to?(:_fork)
+        module ForkTracker
+          def _fork
+            pid = super
+            Session.after_fork if pid.zero?
+            pid
+          end
+        end
+        Process.singleton_class.prepend(ForkTracker)
+      end
+    else
+      INSTANCES = nil
+      private_constant :INSTANCES
+
+      def self.after_fork
+        # noop
+      end
+    end
   end
 
   # session may be overridden by certain adapters.

data/lib/httpx/timers.rb

@@ -7,17 +7,16 @@ module HTTPX
     end
 
     def after(interval_in_secs, cb = nil, &blk)
-      return unless interval_in_secs
-
       callback = cb || blk
 
+      raise Error, "timer must have a callback" unless callback
+
       # I'm assuming here that most requests will have the same
       # request timeout, as in most cases they share common set of
       # options. A user setting different request timeouts for 100s of
       # requests will already have a hard time dealing with that.
-      unless (interval = @intervals.find { |t| t.interval == interval_in_secs })
+      unless (interval = @intervals.bsearch { |t| t.interval == interval_in_secs })
         interval = Interval.new(interval_in_secs)
-        interval.on_empty { @intervals.delete(interval) }
         @intervals << interval
         @intervals.sort!
       end
@@ -30,6 +29,8 @@ module HTTPX
     end
 
     def wait_interval
+      drop_elapsed!
+
       return if @intervals.empty?
 
       @next_interval_at = Utils.now
@@ -43,11 +44,25 @@ module HTTPX
 
       elapsed_time = Utils.elapsed_time(@next_interval_at)
 
+      drop_elapsed!(elapsed_time)
+
       @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
 
       @next_interval_at = nil if @intervals.empty?
     end
 
+    private
+
+    def drop_elapsed!(elapsed_time = 0)
+      # check first, if not elapsed, then return
+      first_interval = @intervals.first
+
+      return unless first_interval && first_interval.elapsed?(elapsed_time)
+
+      # TODO: would be nice to have a drop_while!
+      @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
+    end
+
     class Timer
       def initialize(interval, callback)
         @interval = interval
@@ -67,15 +82,6 @@ module HTTPX
       def initialize(interval)
         @interval = interval
         @callbacks = []
-        @on_empty = nil
-      end
-
-      def on_empty(&blk)
-        @on_empty = blk
-      end
-
-      def cancel
-        @on_empty.call
       end
 
       def <=>(other)
@@ -98,18 +104,20 @@ module HTTPX
 
       def delete(callback)
         @callbacks.delete(callback)
-        @on_empty.call if @callbacks.empty?
       end
 
       def no_callbacks?
         @callbacks.empty?
       end
 
-      def elapsed?
-        @interval <= 0
+      def elapsed?(elapsed = 0)
+        (@interval - elapsed) <= 0 || @callbacks.empty?
       end
 
       def elapse(elapsed)
+        # same as elapsing
+        return 0 if @callbacks.empty?
+
         @interval -= elapsed
 
         if @interval <= 0

data/lib/httpx/transcoder/multipart/decoder.rb

@@ -12,6 +12,7 @@ module HTTPX
         def initialize(filename, content_type)
           @original_filename = filename
           @content_type = content_type
+          @current = nil
           @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
           super(@file)
         end
@@ -68,11 +69,12 @@ module HTTPX
             # raise Error, "couldn't parse part headers" unless idx
             return unless idx
 
+            # @type var head: String
             head = @buffer.byteslice(0..idx + 4 - 1)
 
             @buffer = @buffer.byteslice(head.bytesize..-1)
 
-            content_type = head[MULTIPART_CONTENT_TYPE, 1]
+            content_type = head[MULTIPART_CONTENT_TYPE, 1] || "text/plain"
             if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
               name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
               name.gsub!(/\\(.)/, "\\1")
@@ -83,7 +85,7 @@ module HTTPX
 
             filename = HTTPX::Utils.get_filename(head)
 
-            name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
+            name = filename || +"#{content_type}[]" if name.nil? || name.empty?
 
             @current = name
 

data/lib/httpx/transcoder/multipart/encoder.rb

@@ -20,7 +20,7 @@ module HTTPX
       end
 
       def to_s
-        read
+        read || ""
       ensure
         rewind
       end
@@ -37,6 +37,7 @@ module HTTPX
       def rewind
         form = @form.each_with_object([]) do |(key, val), aux|
           if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
+            # @type var val: File
             val = val.reopen(val.path, File::RDONLY)
           end
           val.rewind if val.respond_to?(:rewind)

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.4.4"
+  VERSION = "1.5.0"
 end