httpx 0.20.0 → 1.3.1

data/lib/httpx/options.rb

@@ -3,18 +3,22 @@
 require "socket"
 
 module HTTPX
+  # Contains a set of options which are passed and shared across from session to its requests or
+  # responses.
   class Options
+    BUFFER_SIZE = 1 << 14
     WINDOW_SIZE = 1 << 14 # 16K
     MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
-    CONNECT_TIMEOUT = 60
-    OPERATION_TIMEOUT = 60
     KEEP_ALIVE_TIMEOUT = 20
     SETTINGS_TIMEOUT = 10
+    CLOSE_HANDSHAKE_TIMEOUT = 10
+    CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
+    REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
 
     # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
     ip_address_families = begin
       list = Socket.ip_address_list
-      if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+      if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? && !a.ipv6_unique_local? }
         [Socket::AF_INET6, Socket::AF_INET]
       else
         [Socket::AF_INET]
@@ -24,29 +28,37 @@ module HTTPX
     end
 
     DEFAULT_OPTIONS = {
+      :max_requests => Float::INFINITY,
       :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
       :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
       :ssl => {},
       :http2_settings => { settings_enable_push: 0 },
       :fallback_protocol => "http/1.1",
+      :supported_compression_formats => %w[gzip deflate],
+      :decompress_response_body => true,
+      :compress_request_body => true,
       :timeout => {
         connect_timeout: CONNECT_TIMEOUT,
         settings_timeout: SETTINGS_TIMEOUT,
+        close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
         operation_timeout: OPERATION_TIMEOUT,
         keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
+        read_timeout: READ_TIMEOUT,
+        write_timeout: WRITE_TIMEOUT,
+        request_timeout: REQUEST_TIMEOUT,
       },
+      :headers_class => Class.new(Headers),
       :headers => {},
       :window_size => WINDOW_SIZE,
+      :buffer_size => BUFFER_SIZE,
       :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
       :request_class => Class.new(Request),
       :response_class => Class.new(Response),
-      :headers_class => Class.new(Headers),
       :request_body_class => Class.new(Request::Body),
       :response_body_class => Class.new(Response::Body),
       :connection_class => Class.new(Connection),
       :options_class => Class.new(self),
       :transport => nil,
-      :transport_options => nil,
       :addresses => nil,
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
@@ -54,28 +66,6 @@ module HTTPX
       :ip_families => ip_address_families,
     }.freeze
 
-    begin
-      module HashExtensions
-        refine Hash do
-          def >=(other)
-            Hash[other] <= self
-          end
-
-          def <=(other)
-            other = Hash[other]
-            return false unless size <= other.size
-
-            each do |k, v|
-              v2 = other.fetch(k) { return false }
-              return false unless v2 == v
-            end
-            true
-          end
-        end
-      end
-      using HashExtensions
-    end unless Hash.method_defined?(:>=)
-
     class << self
       def new(options = {})
         # let enhanced options go through
@@ -94,38 +84,50 @@ module HTTPX
 
         attr_reader(optname)
       end
-
-      def def_option(optname, *args, &block)
-        if args.size.zero? && !block
-          class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(v); v; end # def option_smth(v); v; end
-          OUT
-          return
-        end
-
-        deprecated_def_option(optname, *args, &block)
-      end
-
-      def deprecated_def_option(optname, layout = nil, &interpreter)
-        warn "DEPRECATION WARNING: using `def_option(#{optname})` for setting options is deprecated. " \
-             "Define module OptionsMethods and `def option_#{optname}(val)` instead."
-
-        if layout
-          class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(value)  # def option_origin(v)
-              #{layout}                   #   URI(v)
-            end                           # end
-          OUT
-        elsif interpreter
-          define_method(:"option_#{optname}") do |value|
-            instance_exec(value, &interpreter)
-          end
-        end
-      end
     end
 
+    # creates a new options instance from a given hash, which optionally define the following:
+    #
+    # :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
+    # :debug_level :: the log level of messages (can be 1, 2, or 3).
+    # :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::IO::SSL)
+    # :http2_settings :: a hash of options to be passed to a HTTP2::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
+    # :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
+    #                       like ALPN (defaults to <tt>"http/1.1"</tt>)
+    # :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
+    # :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
+    # :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
+    # :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
+    #             <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>,  <tt>:read_timeout</tt>,  <tt>:write_timeout</tt>
+    #             and <tt>:request_timeout</tt>
+    # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
+    # :window_size :: number of bytes to read from a socket
+    # :buffer_size :: internal read and write buffer size in bytes
+    # :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
+    # :request_class :: class used to instantiate a request
+    # :response_class :: class used to instantiate a response
+    # :headers_class :: class used to instantiate headers
+    # :request_body_class :: class used to instantiate a request body
+    # :response_body_class :: class used to instantiate a response body
+    # :connection_class :: class used to instantiate connections
+    # :options_class :: class used to instantiate options
+    # :transport :: type of transport to use (set to "unix" for UNIX sockets)
+    # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
+    #               paths should be used for UNIX sockets instead)
+    # :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
+    # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
+    # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
+    #                    using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
+    # :resolver_options :: hash of options passed to the resolver
+    # :ip_families :: which socket families are supported (system-dependent)
+    # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
+    # :base_path :: path to prefix given relative paths with (ex: "/v2")
+    # :max_concurrent_requests :: max number of requests which can be set concurrently
+    # :max_requests :: max number of requests which can be made on socket before it reconnects.
+    #
+    # This list of options are enhanced with each loaded plugin, see the plugin docs for details.
     def initialize(options = {})
-      __initialize__(options)
+      do_initialize(options)
       freeze
     end
 
@@ -136,6 +138,7 @@ module HTTPX
       @timeout.freeze
       @headers.freeze
       @addresses.freeze
+      @supported_compression_formats.freeze
     end
 
     def option_origin(value)
@@ -147,18 +150,15 @@ module HTTPX
     end
 
     def option_headers(value)
-      Headers.new(value)
+      headers_class.new(value)
     end
 
     def option_timeout(value)
-      timeouts = Hash[value]
-
-      if timeouts.key?(:loop_timeout)
-        warn ":loop_timeout is deprecated, use :operation_timeout instead"
-        timeouts[:operation_timeout] = timeouts.delete(:loop_timeout)
-      end
+      Hash[value]
+    end
 
-      timeouts
+    def option_supported_compression_formats(value)
+      Array(value).map(&:to_s)
     end
 
     def option_max_concurrent_requests(value)
@@ -174,16 +174,31 @@ module HTTPX
     end
 
     def option_window_size(value)
-      Integer(value)
+      value = Integer(value)
+
+      raise TypeError, ":window_size must be positive" unless value.positive?
+
+      value
+    end
+
+    def option_buffer_size(value)
+      value = Integer(value)
+
+      raise TypeError, ":buffer_size must be positive" unless value.positive?
+
+      value
     end
 
     def option_body_threshold_size(value)
-      Integer(value)
+      bytes = Integer(value)
+      raise TypeError, ":body_threshold_size must be positive" unless bytes.positive?
+
+      bytes
     end
 
     def option_transport(value)
       transport = value.to_s
-      raise TypeError, "\#{transport} is an unsupported transport type" unless IO.registry.key?(transport)
+      raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
 
       transport
     end
@@ -197,53 +212,82 @@ module HTTPX
     end
 
     %i[
-      params form json body ssl http2_settings
+      ssl http2_settings
       request_class response_class headers_class request_body_class
       response_body_class connection_class options_class
-      io fallback_protocol debug debug_level transport_options resolver_class resolver_options
+      io fallback_protocol debug debug_level resolver_class resolver_options
+      compress_request_body decompress_response_body
       persistent
     ].each do |method_name|
-      def_option(method_name)
+      class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+        # sets +v+ as the value of #{method_name}
+        def option_#{method_name}(v); v; end # def option_smth(v); v; end
+      OUT
     end
 
-    REQUEST_IVARS = %i[@params @form @json @body].freeze
-    private_constant :REQUEST_IVARS
+    REQUEST_BODY_IVARS = %i[@headers].freeze
 
     def ==(other)
-      ivars = instance_variables | other.instance_variables
+      super || options_equals?(other)
+    end
+
+    def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
+      # headers and other request options do not play a role, as they are
+      # relevant only for the request.
+      ivars = instance_variables - ignore_ivars
+      other_ivars = other.instance_variables - ignore_ivars
+
+      return false if ivars.size != other_ivars.size
+
+      return false if ivars.sort != other_ivars.sort
+
       ivars.all? do |ivar|
-        case ivar
-        when :@headers
-          # currently, this is used to pick up an available matching connection.
-          # the headers do not play a role, as they are relevant only for the request.
-          true
-        when *REQUEST_IVARS
-          true
-        else
-          instance_variable_get(ivar) == other.instance_variable_get(ivar)
-        end
+        instance_variable_get(ivar) == other.instance_variable_get(ivar)
       end
     end
 
+    OTHER_LOOKUP = ->(obj, k, ivar_map) {
+      case obj
+      when Hash
+        obj[ivar_map[k]]
+      else
+        obj.instance_variable_get(k)
+      end
+    }
     def merge(other)
-      raise ArgumentError, "#{other} is not a valid set of options" unless other.respond_to?(:to_hash)
+      ivar_map = nil
+      other_ivars = case other
+                    when Hash
+                      ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
+                      ivar_map.keys
+                    else
+                      other.instance_variables
+      end
 
-      h2 = other.to_hash
-      return self if h2.empty?
+      return self if other_ivars.empty?
 
-      h1 = to_hash
+      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == OTHER_LOOKUP[other, ivar, ivar_map] }
 
-      return self if h1 >= h2
+      opts = dup
 
-      merged = h1.merge(h2) do |_k, v1, v2|
-        if v1.respond_to?(:merge) && v2.respond_to?(:merge)
-          v1.merge(v2)
-        else
-          v2
+      other_ivars.each do |ivar|
+        v = OTHER_LOOKUP[other, ivar, ivar_map]
+
+        unless v
+          opts.instance_variable_set(ivar, v)
+          next
         end
+
+        v = opts.__send__(:"option_#{ivar[1..-1]}", v)
+
+        orig_v = instance_variable_get(ivar)
+
+        v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
+
+        opts.instance_variable_set(ivar, v)
       end
 
-      self.class.new(merged)
+      opts
     end
 
     def to_hash
@@ -252,40 +296,54 @@ module HTTPX
       end
     end
 
-    if RUBY_VERSION > "2.4.0"
-      def initialize_dup(other)
-        instance_variables.each do |ivar|
-          instance_variable_set(ivar, other.instance_variable_get(ivar).dup)
-        end
+    def extend_with_plugin_classes(pl)
+      if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
+        @request_class = @request_class.dup
+        @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
+        @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
       end
-    else
-      def initialize_dup(other)
-        instance_variables.each do |ivar|
-          value = other.instance_variable_get(ivar)
-          value = case value
-                  when Symbol, Fixnum, TrueClass, FalseClass # rubocop:disable Lint/UnifiedInteger
-                    value
-                  else
-                    value.dup
-          end
-          instance_variable_set(ivar, value)
-        end
+      if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
+        @response_class = @response_class.dup
+        @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
+        @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
+      end
+      if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
+        @headers_class = @headers_class.dup
+        @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
+        @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
+      end
+      if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
+        @request_body_class = @request_body_class.dup
+        @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
+        @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
       end
+      if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
+        @response_body_class = @response_body_class.dup
+        @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
+        @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
+      end
+      if defined?(pl::ConnectionMethods)
+        @connection_class = @connection_class.dup
+        @connection_class.__send__(:include, pl::ConnectionMethods)
+      end
+      return unless defined?(pl::OptionsMethods)
+
+      @options_class = @options_class.dup
+      @options_class.__send__(:include, pl::OptionsMethods)
     end
 
     private
 
-    def __initialize__(options = {})
+    def do_initialize(options = {})
       defaults = DEFAULT_OPTIONS.merge(options)
       defaults.each do |k, v|
         next if v.nil?
 
-        begin
-          value = __send__(:"option_#{k}", v)
-          instance_variable_set(:"@#{k}", value)
-        rescue NoMethodError
-          raise Error, "unknown option: #{k}"
-        end
+        option_method_name = :"option_#{k}"
+        raise Error, "unknown option: #{k}" unless respond_to?(option_method_name)
+
+        value = __send__(option_method_name, v)
+        instance_variable_set(:"@#{k}", value)
       end
     end
   end

data/lib/httpx/parser/http1.rb

@@ -75,6 +75,7 @@ module HTTPX
         buffer = @buffer
 
         while (idx = buffer.index("\n"))
+          # @type var line: String
           line = buffer.byteslice(0..idx)
           raise Error, "wrong header format" if line.start_with?("\s", "\t")
 
@@ -101,9 +102,11 @@ module HTTPX
           separator_index = line.index(":")
           raise Error, "wrong header format" unless separator_index
 
+          # @type var key: String
           key = line.byteslice(0..(separator_index - 1))
 
           key.rstrip! # was lstripped previously!
+          # @type var value: String
           value = line.byteslice((separator_index + 1)..-1)
           value.strip!
           raise Error, "wrong header format" if value.nil?
@@ -118,6 +121,7 @@ module HTTPX
             @observer.on_data(chunk)
           end
         elsif @content_length
+          # @type var data: String
           data = @buffer.byteslice(0, @content_length)
           @buffer = @buffer.byteslice(@content_length..-1) || "".b
           @content_length -= data.bytesize

data/lib/httpx/plugins/{authentication → auth}/basic.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "base64"
+require "httpx/base64"
 
 module HTTPX
   module Plugins
@@ -11,10 +11,6 @@ module HTTPX
           @password = password
         end
 
-        def can_authenticate?(authenticate)
-          authenticate && /Basic .*/.match?(authenticate)
-        end
-
         def authenticate(*)
           "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
         end

data/lib/httpx/plugins/{authentication → auth}/digest.rb

@@ -8,12 +8,11 @@ module HTTPX
   module Plugins
     module Authentication
       class Digest
-        using RegexpExtensions unless Regexp.method_defined?(:match?)
-
-        def initialize(user, password, **)
+        def initialize(user, password, hashed: false, **)
           @user = user
           @password = password
           @nonce = 0
+          @hashed = hashed
         end
 
         def can_authenticate?(authenticate)
@@ -21,7 +20,7 @@ module HTTPX
         end
 
         def authenticate(request, authenticate)
-          "Digest #{generate_header(request.verb.to_s.upcase, request.path, authenticate)}"
+          "Digest #{generate_header(request.verb, request.path, authenticate)}"
         end
 
         private
@@ -30,9 +29,9 @@ module HTTPX
           # discard first token, it's Digest
           auth_info = authenticate[/^(\w+) (.*)/, 2]
 
-          params = Hash[auth_info.split(/ *, */)
-                                 .map { |val| val.split("=") }
-                                 .map { |k, v| [k, v.delete("\"")] }]
+          params = auth_info.split(/ *, */)
+                            .to_h { |val| val.split("=", 2) }
+                            .transform_values { |v| v.delete("\"") }
           nonce = params["nonce"]
           nc = next_nonce
 
@@ -45,7 +44,6 @@ module HTTPX
             raise DigestError, "unknown algorithm \"#{alg}\"" unless algorithm
 
             sess = Regexp.last_match(2)
-            params.delete("algorithm")
           else
             algorithm = ::Digest::MD5
           end
@@ -56,11 +54,13 @@ module HTTPX
           end
 
           a1 = if sess
-            [algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}"),
-             nonce,
-             cnonce].join ":"
+            [
+              (@hashed ? @password : algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}")),
+              nonce,
+              cnonce,
+            ].join ":"
           else
-            "#{@user}:#{params["realm"]}:#{@password}"
+            @hashed ? @password : "#{@user}:#{params["realm"]}:#{@password}"
           end
 
           ha1 = algorithm.hexdigest(a1)
@@ -77,11 +77,11 @@ module HTTPX
             %(response="#{algorithm.hexdigest(request_digest)}"),
           ]
           header << %(realm="#{params["realm"]}") if params.key?("realm")
-          header << %(algorithm=#{params["algorithm"]}") if params.key?("algorithm")
-          header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
+          header << %(algorithm=#{params["algorithm"]}) if params.key?("algorithm")
           header << %(cnonce="#{cnonce}") if cnonce
           header << %(nc=#{nc})
           header << %(qop=#{qop}) if qop
+          header << %(opaque="#{params["opaque"]}") if params.key?("opaque")
           header.join ", "
         end
 

data/lib/httpx/plugins/{authentication → auth}/ntlm.rb

@@ -1,14 +1,12 @@
 # frozen_string_literal: true
 
-require "base64"
+require "httpx/base64"
 require "ntlm"
 
 module HTTPX
   module Plugins
     module Authentication
       class Ntlm
-        using RegexpExtensions unless Regexp.method_defined?(:match?)
-
         def initialize(user, password, domain: nil)
           @user = user
           @password = password

data/lib/httpx/plugins/{authentication → auth}/socks5.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "base64"
-
 module HTTPX
   module Plugins
     module Authentication

data/lib/httpx/plugins/auth.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds a shim +authorization+ method to the session, which will fill
+    # the HTTP Authorization header, and another, +bearer_auth+, which fill the "Bearer " prefix
+    # in its value.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Auth#auth
+    #
+    module Auth
+      module InstanceMethods
+        def authorization(token)
+          with(headers: { "authorization" => token })
+        end
+
+        def bearer_auth(token)
+          authorization("Bearer #{token}")
+        end
+      end
+    end
+    register_plugin :auth, Auth
+  end
+end

data/lib/httpx/plugins/aws_sdk_authentication.rb

@@ -20,9 +20,7 @@ module HTTPX
           true
         end
 
-        def method_missing(*)
-          nil
-        end
+        def method_missing(*); end
       end
 
       #
@@ -74,6 +72,9 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :aws_profile :: AWS account profile to retrieve credentials from.
       module OptionsMethods
         def option_aws_profile(value)
           String(value)