httpx 1.4.3 → 1.5.0

data/lib/httpx/session.rb

@@ -15,11 +15,11 @@ module HTTPX
     # When pass a block, it'll yield itself to it, then closes after the block is evaluated.
     def initialize(options = EMPTY_HASH, &blk)
       @options = self.class.default_options.merge(options)
-      @responses = {}
       @persistent = @options.persistent
       @pool = @options.pool_class.new(@options.pool_options)
       @wrapped = false
       @closing = false
+      INSTANCES[self] = self if @persistent && @options.close_on_fork && INSTANCES
       wrap(&blk) if blk
     end
 
@@ -174,11 +174,15 @@ module HTTPX
     # returns the HTTPX::Connection through which the +request+ should be sent through.
     def find_connection(request_uri, selector, options)
       if (connection = selector.find_connection(request_uri, options))
+        connection.idling if connection.state == :closed
+        connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
         return connection
       end
 
       connection = @pool.checkout_connection(request_uri, options)
 
+      connection.log(level: 2) { "found connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
+
       case connection.state
       when :idle
         do_init_connection(connection, selector)
@@ -188,14 +192,9 @@ module HTTPX
         else
           pin_connection(connection, selector)
         end
-      when :closed
+      when :closing, :closed
         connection.idling
         select_connection(connection, selector)
-      when :closing
-        connection.once(:close) do
-          connection.idling
-          select_connection(connection, selector)
-        end
       else
         pin_connection(connection, selector)
       end
@@ -212,11 +211,6 @@ module HTTPX
       end
     end
 
-    # callback executed when a response for a given request has been received.
-    def on_response(request, response)
-      @responses[request] = response
-    end
-
     # callback executed when an HTTP/2 promise frame has been received.
     def on_promise(_, stream)
       log(level: 2) { "#{stream.id}: refusing stream!" }
@@ -225,7 +219,9 @@ module HTTPX
 
     # returns the corresponding HTTP::Response to the given +request+ if it has been received.
     def fetch_response(request, _selector, _options)
-      @responses.delete(request)
+      response = request.response
+
+      response if response && response.finished?
     end
 
     # sends the +request+ to the corresponding HTTPX::Connection
@@ -242,7 +238,9 @@ module HTTPX
 
       raise error unless error.is_a?(Error)
 
-      request.emit(:response, ErrorResponse.new(request, error))
+      response = ErrorResponse.new(request, error)
+      request.response = response
+      request.emit(:response, response)
     end
 
     # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
@@ -272,7 +270,6 @@ module HTTPX
     end
 
     def set_request_callbacks(request)
-      request.on(:response, &method(:on_response).curry(2)[request])
       request.on(:promise, &method(:on_promise))
     end
 
@@ -367,13 +364,12 @@ module HTTPX
 
       return select_connection(connection, selector) unless found_connection
 
-      if found_connection.open?
-        coalesce_connections(found_connection, connection, selector, from_pool)
-      else
-        found_connection.once(:open) do
-          coalesce_connections(found_connection, connection, selector, from_pool)
-        end
+      connection.log(level: 2) do
+        "try coalescing from #{from_pool ? "pool##{@pool.object_id}" : "selector##{selector.object_id}"} " \
+          "(conn##{found_connection.object_id}[#{found_connection.origin}])"
       end
+
+      coalesce_connections(found_connection, connection, selector, from_pool)
     end
 
     def on_resolver_close(resolver, selector)
@@ -399,14 +395,16 @@ module HTTPX
     # (it is known via +from_pool+), then it adds its to the +selector+.
     def coalesce_connections(conn1, conn2, selector, from_pool)
       unless conn1.coalescable?(conn2)
+        conn2.log(level: 2) { "not coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
         select_connection(conn2, selector)
         @pool.checkin_connection(conn1) if from_pool
         return false
       end
 
-      conn2.coalesced_connection = conn1
+      conn2.log(level: 2) { "coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
+      conn2.coalesce!(conn1)
       select_connection(conn1, selector) if from_pool
-      deselect_connection(conn2, selector)
+      conn2.disconnect
       true
     end
 
@@ -451,6 +449,7 @@ module HTTPX
       #   session_with_custom = session.plugin(CustomPlugin)
       #
       def plugin(pl, options = nil, &block)
+        label = pl
         # raise Error, "Cannot add a plugin to a frozen config" if frozen?
         pl = Plugins.load_plugin(pl) if pl.is_a?(Symbol)
         if !@plugins.include?(pl)
@@ -475,9 +474,36 @@ module HTTPX
           @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
           @default_options = @default_options.merge(options) if options
 
+          if pl.respond_to?(:subplugins)
+            pl.subplugins.transform_keys(&Plugins.method(:load_plugin)).each do |main_pl, sub_pl|
+              # in case the main plugin has already been loaded, then apply subplugin functionality
+              # immediately
+              next unless @plugins.include?(main_pl)
+
+              plugin(sub_pl, options, &block)
+            end
+          end
+
           pl.configure(self, &block) if pl.respond_to?(:configure)
 
+          if label.is_a?(Symbol)
+            # in case an already-loaded plugin complements functionality of
+            # the plugin currently being loaded, loaded it now
+            @plugins.each do |registered_pl|
+              next if registered_pl == pl
+
+              next unless registered_pl.respond_to?(:subplugins)
+
+              sub_pl = registered_pl.subplugins[label]
+
+              next unless sub_pl
+
+              plugin(sub_pl, options, &block)
+            end
+          end
+
           @default_options.freeze
+          set_temporary_name("#{superclass}/#{pl}") if respond_to?(:set_temporary_name) # ruby 3.4 only
         elsif options
           # this can happen when two plugins are loaded, an one of them calls the other under the hood,
           # albeit changing some default.
@@ -486,9 +512,40 @@ module HTTPX
 
           @default_options.freeze
         end
+
         self
       end
     end
+
+    # setup of the support for close_on_fork sessions.
+    # adapted from https://github.com/mperham/connection_pool/blob/main/lib/connection_pool.rb#L48
+    if Process.respond_to?(:fork)
+      INSTANCES = ObjectSpace::WeakMap.new
+      private_constant :INSTANCES
+
+      def self.after_fork
+        INSTANCES.each_value(&:close)
+        nil
+      end
+
+      if ::Process.respond_to?(:_fork)
+        module ForkTracker
+          def _fork
+            pid = super
+            Session.after_fork if pid.zero?
+            pid
+          end
+        end
+        Process.singleton_class.prepend(ForkTracker)
+      end
+    else
+      INSTANCES = nil
+      private_constant :INSTANCES
+
+      def self.after_fork
+        # noop
+      end
+    end
   end
 
   # session may be overridden by certain adapters.

data/lib/httpx/timers.rb

@@ -7,17 +7,16 @@ module HTTPX
     end
 
     def after(interval_in_secs, cb = nil, &blk)
-      return unless interval_in_secs
-
       callback = cb || blk
 
+      raise Error, "timer must have a callback" unless callback
+
       # I'm assuming here that most requests will have the same
       # request timeout, as in most cases they share common set of
       # options. A user setting different request timeouts for 100s of
       # requests will already have a hard time dealing with that.
-      unless (interval = @intervals.find { |t| t.interval == interval_in_secs })
+      unless (interval = @intervals.bsearch { |t| t.interval == interval_in_secs })
         interval = Interval.new(interval_in_secs)
-        interval.on_empty { @intervals.delete(interval) }
         @intervals << interval
         @intervals.sort!
       end
@@ -30,6 +29,8 @@ module HTTPX
     end
 
     def wait_interval
+      drop_elapsed!
+
       return if @intervals.empty?
 
       @next_interval_at = Utils.now
@@ -43,11 +44,25 @@ module HTTPX
 
       elapsed_time = Utils.elapsed_time(@next_interval_at)
 
+      drop_elapsed!(elapsed_time)
+
       @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
 
       @next_interval_at = nil if @intervals.empty?
     end
 
+    private
+
+    def drop_elapsed!(elapsed_time = 0)
+      # check first, if not elapsed, then return
+      first_interval = @intervals.first
+
+      return unless first_interval && first_interval.elapsed?(elapsed_time)
+
+      # TODO: would be nice to have a drop_while!
+      @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
+    end
+
     class Timer
       def initialize(interval, callback)
         @interval = interval
@@ -67,15 +82,6 @@ module HTTPX
       def initialize(interval)
         @interval = interval
         @callbacks = []
-        @on_empty = nil
-      end
-
-      def on_empty(&blk)
-        @on_empty = blk
-      end
-
-      def cancel
-        @on_empty.call
       end
 
       def <=>(other)
@@ -98,18 +104,20 @@ module HTTPX
 
       def delete(callback)
         @callbacks.delete(callback)
-        @on_empty.call if @callbacks.empty?
       end
 
       def no_callbacks?
         @callbacks.empty?
       end
 
-      def elapsed?
-        @interval <= 0
+      def elapsed?(elapsed = 0)
+        (@interval - elapsed) <= 0 || @callbacks.empty?
       end
 
       def elapse(elapsed)
+        # same as elapsing
+        return 0 if @callbacks.empty?
+
         @interval -= elapsed
 
         if @interval <= 0

data/lib/httpx/transcoder/multipart/decoder.rb

@@ -12,6 +12,7 @@ module HTTPX
         def initialize(filename, content_type)
           @original_filename = filename
           @content_type = content_type
+          @current = nil
           @file = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
           super(@file)
         end
@@ -68,11 +69,12 @@ module HTTPX
             # raise Error, "couldn't parse part headers" unless idx
             return unless idx
 
+            # @type var head: String
             head = @buffer.byteslice(0..idx + 4 - 1)
 
             @buffer = @buffer.byteslice(head.bytesize..-1)
 
-            content_type = head[MULTIPART_CONTENT_TYPE, 1]
+            content_type = head[MULTIPART_CONTENT_TYPE, 1] || "text/plain"
             if (name = head[MULTIPART_CONTENT_DISPOSITION, 1])
               name = /\A"(.*)"\Z/ =~ name ? Regexp.last_match(1) : name.dup
               name.gsub!(/\\(.)/, "\\1")
@@ -83,7 +85,7 @@ module HTTPX
 
             filename = HTTPX::Utils.get_filename(head)
 
-            name = filename || +"#{content_type || "text/plain"}[]" if name.nil? || name.empty?
+            name = filename || +"#{content_type}[]" if name.nil? || name.empty?
 
             @current = name
 

data/lib/httpx/transcoder/multipart/encoder.rb

@@ -20,7 +20,7 @@ module HTTPX
       end
 
       def to_s
-        read
+        read || ""
       ensure
         rewind
       end
@@ -37,6 +37,7 @@ module HTTPX
       def rewind
         form = @form.each_with_object([]) do |(key, val), aux|
           if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
+            # @type var val: File
             val = val.reopen(val.path, File::RDONLY)
           end
           val.rewind if val.respond_to?(:rewind)

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.4.3"
+  VERSION = "1.5.0"
 end

data/sig/buffer.rbs

@@ -14,7 +14,7 @@ module HTTPX
     def capacity: () -> Integer
 
     # delegated
-    def <<: (string data) -> String
+    def <<: (String data) -> String
     def empty?: () -> bool
     def bytesize: () -> (Integer | Float)
     def clear: () -> void

data/sig/chainable.rbs

@@ -21,14 +21,16 @@ module HTTPX
               | (:cookies, ?options) -> Plugins::sessionCookies
               | (:expect, ?options) -> Session
               | (:follow_redirects, ?options) -> Plugins::sessionFollowRedirects
-              | (:upgrade, ?options) -> Session
-              | (:h2c, ?options) -> Session
+              | (:upgrade, ?options) -> Plugins::sessionUpgrade
+              | (:h2c, ?options) -> Plugins::sessionUpgrade
+              | (:h2, ?options) -> Plugins::sessionUpgrade
               | (:persistent, ?options) -> Plugins::sessionPersistent
               | (:proxy, ?options) -> (Plugins::sessionProxy & Plugins::httpProxy)
               | (:push_promise, ?options) -> Plugins::sessionPushPromise
               | (:retries, ?options) -> Plugins::sessionRetries
               | (:rate_limiter, ?options) -> Session
               | (:stream, ?options) -> Plugins::sessionStream
+              | (:stream_bidi, ?options) -> Plugins::sessionStreamBidi
               | (:aws_sigv4, ?options) -> Plugins::awsSigV4Session
               | (:grpc, ?options) -> Plugins::grpcSession
               | (:response_cache, ?options) -> Plugins::sessionResponseCache
@@ -39,6 +41,7 @@ module HTTPX
               | (:ssrf_filter, ?options) -> Plugins::sessionSsrf
               | (:webdav, ?options) -> Plugins::sessionWebDav
               | (:xml, ?options) -> Plugins::sessionXML
+              | (:query, ?options) -> Plugins::sessionQuery
               | (Symbol | Module, ?options) { (Class) -> void } -> Session
               | (Symbol | Module, ?options) -> Session
 

data/sig/connection/http2.rbs

@@ -16,6 +16,8 @@ module HTTPX
     @drains: Hash[Request, String]
     @pings: Array[String]
     @buffer: Buffer
+    @handshake_completed: bool
+    @wait_for_handshake: bool
 
     def interests: () -> io_interests?
 
@@ -27,8 +29,6 @@ module HTTPX
 
     def <<: (string) -> void
 
-    def can_buffer_more_requests?: () -> bool
-
     def send: (Request request, ?bool head) -> bool
 
     def consume: () -> void
@@ -45,6 +45,8 @@ module HTTPX
 
     def initialize: (Buffer buffer, Options options) -> untyped
 
+    def can_buffer_more_requests?: () -> bool
+
     def send_pending: () -> void
 
     def set_protocol_headers: (Request) -> _Each[[String, String]]
@@ -63,6 +65,10 @@ module HTTPX
 
     def join_body: (::HTTP2::Stream stream, Request request) -> void
 
+    def send_chunk: (Request request, ::HTTP2::Stream stream, String chunk, String? next_chunk) -> void
+
+    def end_stream?: (Request request, String? next_chunk) -> void
+
     def on_stream_headers: (::HTTP2::Stream stream, Request request, Array[[String, String]] headers) -> void
 
     def on_stream_trailers: (::HTTP2::Stream stream, Response response, Array[[String, String]] headers) -> void
@@ -92,12 +98,15 @@ module HTTPX
     def on_pong: (string ping) -> void
 
     class Error < ::HTTPX::Error
+      def initialize: (Integer id, Symbol | StandardError error) -> void
     end
 
     class GoawayError < Error
+      def initialize: () -> void
     end
 
     class PingError < Error
+      def initialize: () -> void
     end
   end
 end

data/sig/connection.rbs

@@ -20,6 +20,7 @@ module HTTPX
 
 
     attr_reader type: io_type
+    attr_reader io: TCP | SSL | UNIX | nil
     attr_reader origin: http_uri
     attr_reader origins: Array[String]
     attr_reader state: Symbol
@@ -39,7 +40,6 @@ module HTTPX
     @keep_alive_timeout: Numeric?
     @timeout: Numeric?
     @current_timeout: Numeric?
-    @io: TCP | SSL | UNIX
     @parser: Object & _Parser
     @connected_at: Float
     @response_received_at: Float
@@ -64,6 +64,8 @@ module HTTPX
 
     def mergeable?: (Connection connection) -> bool
 
+    def coalesce!: (instance connection) -> void
+
     def coalescable?: (Connection connection) -> bool
 
     def create_idle: (?Hash[Symbol, untyped] options) -> instance
@@ -104,8 +106,6 @@ module HTTPX
 
     def handle_socket_timeout: (Numeric interval) -> void
 
-    def coalesced_connection=: (instance connection) -> void
-
     def sibling=: (instance? connection) -> void
 
     def handle_connect_error: (StandardError error) -> void
@@ -164,6 +164,6 @@ module HTTPX
 
     def set_request_timeout: (Symbol label, Request request, Numeric timeout, Symbol start_event, Symbol | Array[Symbol] finish_events) { () -> void } -> void
 
-    def self.parser_type: (String protocol) -> (singleton(HTTP1) | singleton(HTTP2))
+    def parser_type: (String protocol) -> (singleton(HTTP1) | singleton(HTTP2))
   end
 end

data/sig/errors.rbs

@@ -18,9 +18,6 @@ module HTTPX
   end
 
   class PoolTimeoutError < TimeoutError
-    attr_reader origin: String
-
-    def initialize: (String origin, Numeric timeout) -> void
   end
 
   class ConnectTimeoutError < TimeoutError

data/sig/headers.rbs

@@ -2,7 +2,7 @@ module HTTPX
   class Headers
     include _ToS
 
-    @headers: Hash[String, Array[String]]
+    @headers: Hash[String, Array[_ToS]]
 
     def self.new: (?untyped headers) -> instance
 
@@ -11,34 +11,39 @@ module HTTPX
     def []: (String field) -> String?
     def []=: (String field, headers_value value) -> void
 
-    def add: (String field, string value) -> void
-    def delete: (String field) -> Array[String]?
+    def add: (String field, String value) -> void
+    def delete: (String field) -> Array[_ToS]?
 
     def each: (?_Each[[String, String]]? extra_headers) { (String k, String v) -> void } -> void
             | (?_Each[[String, String]]? extra_headers) -> Enumerable[[String, String]]
 
-    def get: (String field) -> Array[String]
+    def get: (String field) -> Array[_ToS]
     def key?: (String downcased_key) -> bool
 
     def merge: (_Each[[String, headers_value]] other) -> Headers
 
-    def same_headers?: (untyped headers) -> bool
+    def empty?: () -> bool
 
     def to_a: () -> Array[[String, String]]
+
     def to_hash: () -> Hash[String, String]
+
     alias to_h to_hash
 
     def inspect: () -> String
 
     private
 
-    def initialize: (?headers?) -> untyped
-    def array_value: (headers_value) -> Array[String]
-    def downcased: (_ToS field) -> String
+    def initialize: (?(headers_input | instance)?) -> void
+
+    def array_value: (headers_value value) -> Array[_ToS]
+
+    def downcased: (header_field field) -> String
   end
 
+  type header_field = string | _ToS
   type headers_value = _ToS | Array[_ToS]
-  type headers_hash = Hash[_ToS, headers_value]
-  type headers_input = headers_hash | Array[[_ToS, string]]
+  type headers_hash = Hash[header_field, headers_value]
+  type headers_input = headers_hash | Array[[header_field, headers_value]]
   type headers = Headers | headers_input
 end

data/sig/httpx.rbs

@@ -2,6 +2,7 @@ module HTTPX
   extend Chainable
 
   EMPTY: Array[untyped]
+  EMPTY_HASH: Hash[untyped, untyped]
 
   VERSION: String
 
@@ -16,7 +17,10 @@ module HTTPX
   type ip_family = Integer #Socket::AF_INET6 | Socket::AF_INET
 
   module Plugins
-    def self?.load_plugin: (Symbol) -> Module
+    self.@plugins: Hash[Symbol, Module]
+    self.@plugins_mutex: Thread::Mutex
+
+    def self?.load_plugin: (Symbol name) -> Module
 
     def self?.register_plugin: (Symbol, Module) -> void
   end

data/sig/io/tcp.rbs

@@ -14,6 +14,12 @@ module HTTPX
 
     alias host ip
 
+    @hostname: String
+
+    @options: Options
+
+    @fallback_protocol: String
+
     # TODO: lift when https://github.com/ruby/rbs/issues/1497 fixed
     def initialize: (URI::Generic origin, Array[ipaddr]? addresses, Options options) ?{ (instance) -> void } -> void
 

data/sig/loggable.rbs

@@ -11,5 +11,7 @@ module HTTPX
     def log: (?level: Integer?, ?color: Symbol?, ?debug_level: Integer, ?debug: _IOLogger?) { () -> String } -> void
 
     def log_exception: (Exception error, ?level: Integer, ?color: Symbol, ?debug_level: Integer, ?debug: _IOLogger?) -> void
+
+    def log_redact: (_ToS text, ?bool should_redact) -> String
   end
 end

data/sig/options.rbs

@@ -13,6 +13,7 @@ module HTTPX
     KEEP_ALIVE_TIMEOUT: Integer
     SETTINGS_TIMEOUT: Integer
     CLOSE_HANDSHAKE_TIMEOUT: Integer
+    SET_TEMPORARY_NAME: ^(Module mod, ?Symbol pl) -> void
     DEFAULT_OPTIONS: Hash[Symbol, untyped]
     REQUEST_BODY_IVARS: Array[Symbol]
 
@@ -25,7 +26,7 @@ module HTTPX
     attr_reader uri: URI?
 
     # headers
-    attr_reader headers: Headers?
+    attr_reader headers: headers?
 
     # timeout
     attr_reader timeout: timeout
@@ -89,6 +90,8 @@ module HTTPX
 
     attr_reader response_body_class: singleton(Response::Body)
 
+    attr_reader options_class: singleton(Options)
+
     attr_reader resolver_class: Symbol | Class
 
     attr_reader ssl: Hash[Symbol, untyped]
@@ -109,6 +112,9 @@ module HTTPX
     # persistent
     attr_reader persistent: bool
 
+    # close_on_fork
+    attr_reader close_on_fork: bool
+
     # resolver_options
     attr_reader resolver_options: Hash[Symbol, untyped]
 

data/sig/plugins/cookies/cookie.rbs

@@ -1,7 +1,5 @@
 module HTTPX
   module Plugins::Cookies
-    type cookie_attributes = Hash[Symbol | String, top]
-
     class Cookie
       include Comparable
 
@@ -33,7 +31,7 @@ module HTTPX
       def cookie_value: () -> String
       alias to_s cookie_value
 
-      def valid_for_uri?: (uri) -> bool
+      def valid_for_uri?: (http_uri uri) -> bool
 
       def self.new: (Cookie) -> instance
                     | (cookie_attributes) -> instance

data/sig/plugins/cookies/jar.rbs

@@ -11,12 +11,12 @@ module HTTPX
 
       def add: (Cookie name, ?String path) -> void
 
-      def []: (uri) -> Array[Cookie]
+      def []: (http_uri) -> Array[Cookie]
 
-      def each: (?uri?) { (Cookie) -> void } -> void
-              | (?uri?) -> Enumerable[Cookie]
+      def each: (?http_uri?) { (Cookie) -> void } -> void
+              | (?http_uri?) -> Enumerable[Cookie]
 
-      def merge: (_Each[cookie] cookies) -> instance
+      def merge: (_Each[cookie] cookies) -> self
 
       private