httpx 1.4.4 → 1.5.0

data/lib/httpx/headers.rb

@@ -11,20 +11,32 @@ module HTTPX
     end
 
     def initialize(headers = nil)
+      if headers.nil? || headers.empty?
+        @headers = headers.to_h
+        return
+      end
+
       @headers = {}
-      return unless headers
 
       headers.each do |field, value|
-        array_value(value).each do |v|
-          add(downcased(field), v)
+        field = downcased(field)
+
+        value = array_value(value)
+
+        current = @headers[field]
+
+        if current.nil?
+          @headers[field] = value
+        else
+          current.concat(value)
         end
       end
     end
 
     # cloned initialization
-    def initialize_clone(orig)
+    def initialize_clone(orig, **kwargs)
       super
-      @headers = orig.instance_variable_get(:@headers).clone
+      @headers = orig.instance_variable_get(:@headers).clone(**kwargs)
     end
 
     # dupped initialization
@@ -39,17 +51,6 @@ module HTTPX
       super
     end
 
-    def same_headers?(headers)
-      @headers.empty? || begin
-        headers.each do |k, v|
-          next unless key?(k)
-
-          return false unless v == self[k]
-        end
-        true
-      end
-    end
-
     # merges headers with another header-quack.
     # the merge rule is, if the header already exists,
     # ignore what the +other+ headers has. Otherwise, set
@@ -119,6 +120,10 @@ module HTTPX
       other == to_hash
     end
 
+    def empty?
+      @headers.empty?
+    end
+
     # the headers store in Hash format
     def to_hash
       Hash[to_a]
@@ -137,7 +142,8 @@ module HTTPX
 
     # :nocov:
     def inspect
-      to_hash.inspect
+      "#<#{self.class}:#{object_id} " \
+        "#{to_hash.inspect}>"
     end
     # :nocov:
 
@@ -160,12 +166,7 @@ module HTTPX
     private
 
     def array_value(value)
-      case value
-      when Array
-        value.map { |val| String(val).strip }
-      else
-        [String(value).strip]
-      end
+      Array(value)
     end
 
     def downcased(field)

data/lib/httpx/io/ssl.rb

@@ -9,7 +9,8 @@ module HTTPX
     # rubocop:disable Style/MutableConstant
     TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
     # https://github.com/jruby/jruby-openssl/issues/284
-    TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby"
+    # TODO: remove when dropping support for jruby-openssl < 0.15.4
+    TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby" && JOpenSSL::VERSION < "0.15.4"
     # rubocop:enable Style/MutableConstant
     TLS_OPTIONS.freeze
 

data/lib/httpx/io/tcp.rb

@@ -167,7 +167,12 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<#{self.class}: #{@ip}:#{@port} (state: #{@state})>"
+      "#<#{self.class}:#{object_id} " \
+        "#{@ip}:#{@port} " \
+        "@state=#{@state} " \
+        "@hostname=#{@hostname} " \
+        "@addresses=#{@addresses} " \
+        "@state=#{@state}>"
     end
     # :nocov:
 
@@ -195,12 +200,9 @@ module HTTPX
     end
 
     def log_transition_state(nextstate)
-      case nextstate
-      when :connected
-        "Connected to #{host} (##{@io.fileno})"
-      else
-        "#{host} #{@state} -> #{nextstate}"
-      end
+      label = host
+      label = "#{label}(##{@io.fileno})" if nextstate == :connected
+      "#{label} #{@state} -> #{nextstate}"
     end
   end
 end

data/lib/httpx/io/unix.rb

@@ -57,7 +57,7 @@ module HTTPX
 
     # :nocov:
     def inspect
-      "#<#{self.class}(path: #{@path}): (state: #{@state})>"
+      "#<#{self.class}:#{object_id} @path=#{@path}) @state=#{@state})>"
     end
     # :nocov:
 

data/lib/httpx/loggable.rb

@@ -15,7 +15,13 @@ module HTTPX
 
     USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
 
-    def log(level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug, &msg)
+    def log(
+      level: @options.debug_level,
+      color: nil,
+      debug_level: @options.debug_level,
+      debug: @options.debug,
+      &msg
+    )
       return unless debug_level >= level
 
       debug_stream = debug || ($stderr if USE_DEBUG_LOG)
@@ -37,5 +43,11 @@ module HTTPX
     def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
       log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
     end
+
+    def log_redact(text, should_redact = @options.debug_redact)
+      return text.to_s unless should_redact
+
+      "[REDACTED]"
+    end
   end
 end

data/lib/httpx/options.rb

@@ -27,10 +27,19 @@ module HTTPX
       [Socket::AF_INET]
     end.freeze
 
+    SET_TEMPORARY_NAME = ->(mod, pl = nil) do
+      if mod.respond_to?(:set_temporary_name) # ruby 3.4 only
+        name = mod.name || "#{mod.superclass.name}(plugin)"
+        name = "#{name}/#{pl}" if pl
+        mod.set_temporary_name(name)
+      end
+    end
+
     DEFAULT_OPTIONS = {
       :max_requests => Float::INFINITY,
       :debug => nil,
       :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
+      :debug_redact => ENV.key?("HTTPX_DEBUG_REDACT"),
       :ssl => EMPTY_HASH,
       :http2_settings => { settings_enable_push: 0 }.freeze,
       :fallback_protocol => "http/1.1",
@@ -47,18 +56,18 @@ module HTTPX
         write_timeout: WRITE_TIMEOUT,
         request_timeout: REQUEST_TIMEOUT,
       },
-      :headers_class => Class.new(Headers),
+      :headers_class => Class.new(Headers, &SET_TEMPORARY_NAME),
       :headers => {},
       :window_size => WINDOW_SIZE,
       :buffer_size => BUFFER_SIZE,
       :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
-      :request_class => Class.new(Request),
-      :response_class => Class.new(Response),
-      :request_body_class => Class.new(Request::Body),
-      :response_body_class => Class.new(Response::Body),
-      :pool_class => Class.new(Pool),
-      :connection_class => Class.new(Connection),
-      :options_class => Class.new(self),
+      :request_class => Class.new(Request, &SET_TEMPORARY_NAME),
+      :response_class => Class.new(Response, &SET_TEMPORARY_NAME),
+      :request_body_class => Class.new(Request::Body, &SET_TEMPORARY_NAME),
+      :response_body_class => Class.new(Response::Body, &SET_TEMPORARY_NAME),
+      :pool_class => Class.new(Pool, &SET_TEMPORARY_NAME),
+      :connection_class => Class.new(Connection, &SET_TEMPORARY_NAME),
+      :options_class => Class.new(self, &SET_TEMPORARY_NAME),
       :transport => nil,
       :addresses => nil,
       :persistent => false,
@@ -66,6 +75,7 @@ module HTTPX
       :resolver_options => { cache: true }.freeze,
       :pool_options => EMPTY_HASH,
       :ip_families => ip_address_families,
+      :close_on_fork => false,
     }.freeze
 
     class << self
@@ -92,6 +102,7 @@ module HTTPX
     #
     # :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
     # :debug_level :: the log level of messages (can be 1, 2, or 3).
+    # :debug_redact :: whether header/body payload should be redacted (defaults to <tt>false</tt>).
     # :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::IO::SSL)
     # :http2_settings :: a hash of options to be passed to a HTTP2::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
     # :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
@@ -128,6 +139,8 @@ module HTTPX
     # :base_path :: path to prefix given relative paths with (ex: "/v2")
     # :max_concurrent_requests :: max number of requests which can be set concurrently
     # :max_requests :: max number of requests which can be made on socket before it reconnects.
+    # :close_on_fork :: whether the session automatically closes when the process is fork (defaults to <tt>false</tt>).
+    #                   it only works if the session is persistent (and ruby 3.1 or higher is used).
     #
     # This list of options are enhanced with each loaded plugin, see the plugin docs for details.
     def initialize(options = {})
@@ -136,13 +149,18 @@ module HTTPX
     end
 
     def freeze
-      super
       @origin.freeze
       @base_path.freeze
       @timeout.freeze
       @headers.freeze
       @addresses.freeze
       @supported_compression_formats.freeze
+      @ssl.freeze
+      @http2_settings.freeze
+      @pool_options.freeze
+      @resolver_options.freeze
+      @ip_families.freeze
+      super
     end
 
     def option_origin(value)
@@ -165,41 +183,6 @@ module HTTPX
       Array(value).map(&:to_s)
     end
 
-    def option_max_concurrent_requests(value)
-      raise TypeError, ":max_concurrent_requests must be positive" unless value.positive?
-
-      value
-    end
-
-    def option_max_requests(value)
-      raise TypeError, ":max_requests must be positive" unless value.positive?
-
-      value
-    end
-
-    def option_window_size(value)
-      value = Integer(value)
-
-      raise TypeError, ":window_size must be positive" unless value.positive?
-
-      value
-    end
-
-    def option_buffer_size(value)
-      value = Integer(value)
-
-      raise TypeError, ":buffer_size must be positive" unless value.positive?
-
-      value
-    end
-
-    def option_body_threshold_size(value)
-      bytes = Integer(value)
-      raise TypeError, ":body_threshold_size must be positive" unless bytes.positive?
-
-      bytes
-    end
-
     def option_transport(value)
       transport = value.to_s
       raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
@@ -215,17 +198,42 @@ module HTTPX
       Array(value)
     end
 
+    # number options
+    %i[
+      max_concurrent_requests max_requests window_size buffer_size
+      body_threshold_size debug_level
+    ].each do |option|
+      class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+        # converts +v+ into an Integer before setting the +#{option}+ option.
+        def option_#{option}(value)                                             # def option_max_requests(v)
+          value = Integer(value) unless value.infinite?
+          raise TypeError, ":#{option} must be positive" unless value.positive? # raise TypeError, ":max_requests must be positive" unless value.positive?
+
+          value
+        end
+      OUT
+    end
+
+    # hashable options
+    %i[ssl http2_settings resolver_options pool_options].each do |option|
+      class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+        # converts +v+ into an Hash before setting the +#{option}+ option.
+        def option_#{option}(value) # def option_ssl(v)
+          Hash[value]
+        end
+      OUT
+    end
+
     %i[
-      ssl http2_settings
       request_class response_class headers_class request_body_class
       response_body_class connection_class options_class
       pool_class pool_options
-      io fallback_protocol debug debug_level resolver_class resolver_options
+      io fallback_protocol debug debug_redact resolver_class
       compress_request_body decompress_response_body
-      persistent
+      persistent close_on_fork
     ].each do |method_name|
       class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-        # sets +v+ as the value of #{method_name}
+        # sets +v+ as the value of the +#{method_name}+ option
         def option_#{method_name}(v); v; end # def option_smth(v); v; end
       OUT
     end
@@ -296,35 +304,42 @@ module HTTPX
     def extend_with_plugin_classes(pl)
       if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
         @request_class = @request_class.dup
+        SET_TEMPORARY_NAME[@request_class, pl]
         @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
         @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
       end
       if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
         @response_class = @response_class.dup
+        SET_TEMPORARY_NAME[@response_class, pl]
         @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
         @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
       end
       if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
         @headers_class = @headers_class.dup
+        SET_TEMPORARY_NAME[@headers_class, pl]
         @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
         @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
       end
       if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
         @request_body_class = @request_body_class.dup
+        SET_TEMPORARY_NAME[@request_body_class, pl]
         @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
         @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
       end
       if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
         @response_body_class = @response_body_class.dup
+        SET_TEMPORARY_NAME[@response_body_class, pl]
         @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
         @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
       end
       if defined?(pl::PoolMethods)
         @pool_class = @pool_class.dup
+        SET_TEMPORARY_NAME[@pool_class, pl]
         @pool_class.__send__(:include, pl::PoolMethods)
       end
       if defined?(pl::ConnectionMethods)
         @connection_class = @connection_class.dup
+        SET_TEMPORARY_NAME[@connection_class, pl]
         @connection_class.__send__(:include, pl::ConnectionMethods)
       end
       return unless defined?(pl::OptionsMethods)

data/lib/httpx/parser/http1.rb

@@ -23,7 +23,7 @@ module HTTPX
 
       def reset!
         @state = :idle
-        @headers.clear
+        @headers = {}
         @content_length = nil
         @_has_trailers = nil
       end

data/lib/httpx/plugins/aws_sigv4.rb

@@ -158,6 +158,7 @@ module HTTPX
         def load_dependencies(*)
           require "set"
           require "digest/sha2"
+          require "cgi/escape"
         end
 
         def configure(klass)

data/lib/httpx/plugins/callbacks.rb

@@ -8,6 +8,13 @@ module HTTPX
     # https://gitlab.com/os85/httpx/-/wikis/Events
     #
     module Callbacks
+      CALLBACKS = %i[
+        connection_opened connection_closed
+        request_error
+        request_started request_body_chunk request_completed
+        response_started response_body_chunk response_completed
+      ].freeze
+
       # connection closed user-space errors happen after errors can be surfaced to requests,
       # so they need to pierce through the scheduler, which is only possible by simulating an
       # interrupt.
@@ -16,12 +23,7 @@ module HTTPX
       module InstanceMethods
         include HTTPX::Callbacks
 
-        %i[
-          connection_opened connection_closed
-          request_error
-          request_started request_body_chunk request_completed
-          response_started response_body_chunk response_completed
-        ].each do |meth|
+        CALLBACKS.each do |meth|
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
             def on_#{meth}(&blk)   # def on_connection_opened(&blk)
               on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
@@ -32,6 +34,17 @@ module HTTPX
 
         private
 
+        def branch(options, &blk)
+          super(options).tap do |sess|
+            CALLBACKS.each do |cb|
+              next unless callbacks_for?(cb)
+
+              sess.callbacks(cb).concat(callbacks(cb))
+            end
+            sess.wrap(&blk) if blk
+          end
+        end
+
         def do_init_connection(connection, selector)
           super
           connection.on(:open) do

data/lib/httpx/plugins/circuit_breaker.rb

@@ -70,10 +70,11 @@ module HTTPX
           short_circuit_responses
         end
 
-        def on_response(request, response)
-          emit(:circuit_open, request) if try_circuit_open(request, response)
-
+        def set_request_callbacks(request)
           super
+          request.on(:response) do |response|
+            emit(:circuit_open, request) if try_circuit_open(request, response)
+          end
         end
 
         def try_circuit_open(request, response)

data/lib/httpx/plugins/cookies/jar.rb

@@ -59,8 +59,6 @@ module HTTPX
 
         return @cookies.each(&blk) unless uri
 
-        uri = URI(uri)
-
         now = Time.now
         tpath = uri.path
 

data/lib/httpx/plugins/cookies/set_cookie_parser.rb

@@ -83,7 +83,7 @@ module HTTPX
           scanner.skip(RE_WSP)
 
           name, value = scan_name_value(scanner, true)
-          value = nil if name.empty?
+          value = nil if name && name.empty?
 
           attrs = {}
 
@@ -98,15 +98,18 @@ module HTTPX
 
               aname, avalue = scan_name_value(scanner, true)
 
-              next if aname.empty? || value.nil?
+              next if (aname.nil? || aname.empty?) || value.nil?
 
               aname.downcase!
 
               case aname
               when "expires"
+                next unless avalue
+
                 # RFC 6265 5.2.1
-                (avalue &&= Time.parse(avalue)) || next
+                (avalue = Time.parse(avalue)) || next
               when "max-age"
+                next unless avalue
                 # RFC 6265 5.2.2
                 next unless /\A-?\d+\z/.match?(avalue)
 
@@ -119,7 +122,7 @@ module HTTPX
                 # RFC 6265 5.2.4
                 # A relative path must be ignored rather than normalizing it
                 # to "/".
-                next unless avalue.start_with?("/")
+                next unless avalue && avalue.start_with?("/")
               when "secure", "httponly"
                 # RFC 6265 5.2.5, 5.2.6
                 avalue = true

data/lib/httpx/plugins/cookies.rb

@@ -48,15 +48,15 @@ module HTTPX
 
         private
 
-        def on_response(_request, response)
-          if response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
+        def set_request_callbacks(request)
+          super
+          request.on(:response) do |response|
+            next unless response && response.respond_to?(:headers) && (set_cookie = response.headers["set-cookie"])
 
             log { "cookies: set-cookie is over #{Cookie::MAX_LENGTH}" } if set_cookie.bytesize > Cookie::MAX_LENGTH
 
             @options.cookies.parse(set_cookie)
           end
-
-          super
         end
       end
 

data/lib/httpx/plugins/follow_redirects.rb

@@ -149,9 +149,11 @@ module HTTPX
             retry_start = Utils.now
             log { "redirecting after #{redirect_after} secs..." }
             selector.after(redirect_after) do
-              if request.response
+              if (response = request.response)
+                response.finish!
+                retry_request.response = response
                 # request has terminated abruptly meanwhile
-                retry_request.emit(:response, request.response)
+                retry_request.emit(:response, response)
               else
                 log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                 send_request(retry_request, selector, options)

data/lib/httpx/plugins/grpc/call.rb

@@ -15,7 +15,7 @@ module HTTPX
         end
 
         def inspect
-          "#GRPC::Call(#{grpc_response})"
+          "#{self.class}(#{grpc_response})"
         end
 
         def to_s

data/lib/httpx/plugins/h2c.rb

@@ -42,6 +42,12 @@ module HTTPX
         end
       end
 
+      module RequestMethods
+        def valid_h2c_verb?
+          VALID_H2C_VERBS.include?(@verb)
+        end
+      end
+
       module ConnectionMethods
         using URIExtensions
 
@@ -53,7 +59,7 @@ module HTTPX
         def send(request)
           return super if @h2c_handshake
 
-          return super unless VALID_H2C_VERBS.include?(request.verb) && request.scheme == "http"
+          return super unless request.valid_h2c_verb? && request.scheme == "http"
 
           return super if @upgrade_protocol == "h2c"
 

data/lib/httpx/plugins/persistent.rb

@@ -24,7 +24,7 @@ module HTTPX
         else
           1
         end
-        klass.plugin(:retries, max_retries: max_retries, retry_change_requests: true)
+        klass.plugin(:retries, max_retries: max_retries)
       end
 
       def self.extra_options(options)
@@ -34,6 +34,27 @@ module HTTPX
       module InstanceMethods
         private
 
+        def repeatable_request?(request, _)
+          super || begin
+            response = request.response
+
+            return false unless response && response.is_a?(ErrorResponse)
+
+            error = response.error
+
+            Retries::RECONNECTABLE_ERRORS.any? { |klass| error.is_a?(klass) }
+          end
+        end
+
+        def retryable_error?(ex)
+          super &&
+            # under the persistent plugin rules, requests are only retried for connection related errors,
+            # which do not include request timeout related errors. This only gets overriden if the end user
+            # manually changed +:max_retries+ to something else, which means it is aware of the
+            # consequences.
+            (!ex.is_a?(RequestTimeoutError) || @options.max_retries != 1)
+        end
+
         def get_current_selector
           super(&nil) || begin
             return unless block_given?

data/lib/httpx/plugins/proxy/http.rb

@@ -60,7 +60,7 @@ module HTTPX
               return unless @io.connected?
 
               @parser || begin
-                @parser = self.class.parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+                @parser = parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
                 parser = @parser
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
@@ -138,6 +138,8 @@ module HTTPX
             else
               pending = @pending + @parser.pending
               while (req = pending.shift)
+                response.finish!
+                req.response = response
                 req.emit(:response, response)
               end
               reset

data/lib/httpx/plugins/query.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for using the experimental QUERY HTTP method
+    #
+    # https://gitlab.com/os85/httpx/wikis/Query
+    module Query
+      def self.subplugins
+        {
+          retries: QueryRetries,
+        }
+      end
+
+      module InstanceMethods
+        def query(*uri, **options)
+          request("QUERY", uri, **options)
+        end
+      end
+
+      module QueryRetries
+        module InstanceMethods
+          private
+
+          def repeatable_request?(request, options)
+            super || request.verb == "QUERY"
+          end
+        end
+      end
+    end
+
+    register_plugin :query, Query
+  end
+end