httpx 0.21.0 → 1.2.1

data/lib/httpx/io/udp.rb

@@ -6,11 +6,10 @@ module HTTPX
   class UDP
     include Loggable
 
-    def initialize(uri, _, options)
-      ip = IPAddr.new(uri.host)
-      @host = ip.to_s
-      @port = uri.port
-      @io = UDPSocket.new(ip.family)
+    def initialize(ip, port, options)
+      @host = ip
+      @port = port
+      @io = UDPSocket.new(IPAddr.new(ip).family)
       @options = options
     end
 
@@ -24,45 +23,19 @@ module HTTPX
       true
     end
 
-    if RUBY_VERSION < "2.3"
-      # :nocov:
-      def close
-        @io.close
-      rescue StandardError
-        nil
-      end
-      # :nocov:
-    else
-      def close
-        @io.close
-      end
+    def close
+      @io.close
     end
 
-    # :nocov:
-    if (RUBY_ENGINE == "truffleruby" && RUBY_ENGINE_VERSION < "21.1.0") ||
-       RUBY_VERSION < "2.3"
+    if RUBY_ENGINE == "jruby"
+      # In JRuby, sendmsg_nonblock is not implemented
       def write(buffer)
-        siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s))
+        siz = @io.send(buffer.to_s, 0, @host, @port)
         log { "WRITE: #{siz} bytes..." }
         buffer.shift!(siz)
         siz
-      rescue ::IO::WaitWritable
-        0
-      rescue EOFError
-        nil
-      end
-
-      def read(size, buffer)
-        data, _ = @io.recvfrom_nonblock(size)
-        buffer.replace(data)
-        log { "READ: #{buffer.bytesize} bytes..." }
-        buffer.bytesize
-      rescue ::IO::WaitReadable
-        0
-      rescue IOError
       end
     else
-
       def write(buffer)
         siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
         return 0 if siz == :wait_writable
@@ -73,24 +46,17 @@ module HTTPX
         buffer.shift!(siz)
         siz
       end
+    end
 
-      def read(size, buffer)
-        ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
-        return 0 if ret == :wait_readable
-        return if ret.nil?
+    def read(size, buffer)
+      ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
+      return 0 if ret == :wait_readable
+      return if ret.nil?
 
-        buffer.bytesize
-      rescue IOError
-      end
-    end
+      log { "READ: #{buffer.bytesize} bytes..." }
 
-    # In JRuby, sendmsg_nonblock is not implemented
-    def write(buffer)
-      siz = @io.send(buffer.to_s, 0, @host, @port)
-      log { "WRITE: #{siz} bytes..." }
-      buffer.shift!(siz)
-      siz
-    end if RUBY_ENGINE == "jruby"
-    # :nocov:
+      buffer.bytesize
+    rescue IOError
+    end
   end
 end

data/lib/httpx/io/unix.rb

@@ -1,11 +1,7 @@
 # frozen_string_literal: true
 
-require "forwardable"
-
 module HTTPX
   class UNIX < TCP
-    extend Forwardable
-
     using URIExtensions
 
     attr_reader :path
@@ -13,7 +9,7 @@ module HTTPX
     alias_method :host, :path
 
     def initialize(origin, addresses, options)
-      @addresses = addresses
+      @addresses = []
       @hostname = origin.host
       @state = :idle
       @options = Options.new(options)
@@ -31,14 +27,7 @@ module HTTPX
         @keep_open = true
         @state = :connected
       else
-        if @options.transport_options
-          # :nocov:
-          warn ":transport_options is deprecated, use :addresses instead"
-          @path = @options.transport_options[:path]
-          # :nocov:
-        else
-          @path = addresses.first
-        end
+        @path = addresses.first
       end
       @io ||= build_socket
     end
@@ -60,6 +49,10 @@ module HTTPX
            ::IO::WaitReadable
     end
 
+    def expired?
+      false
+    end
+
     # :nocov:
     def inspect
       "#<#{self.class}(path: #{@path}): (state: #{@state})>"

data/lib/httpx/io.rb

@@ -4,14 +4,8 @@ require "socket"
 require "httpx/io/udp"
 require "httpx/io/tcp"
 require "httpx/io/unix"
-require "httpx/io/ssl"
 
-module HTTPX
-  module IO
-    extend Registry
-    register "udp", UDP
-    register "unix", HTTPX::UNIX
-    register "tcp", TCP
-    register "ssl", SSL
-  end
+begin
+  require "httpx/io/ssl"
+rescue LoadError
 end

data/lib/httpx/loggable.rb

@@ -24,26 +24,11 @@ module HTTPX
       debug_stream << message
     end
 
-    if Exception.instance_methods.include?(:full_message)
-
-      def log_exception(ex, level: @options.debug_level, color: nil)
-        return unless @options.debug
-        return unless @options.debug_level >= level
-
-        log(level: level, color: color) { ex.full_message }
-      end
-
-    else
-
-      def log_exception(ex, level: @options.debug_level, color: nil)
-        return unless @options.debug
-        return unless @options.debug_level >= level
-
-        message = +"#{ex.message} (#{ex.class})"
-        message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
-        log(level: level, color: color) { message }
-      end
+    def log_exception(ex, level: @options.debug_level, color: nil)
+      return unless @options.debug
+      return unless @options.debug_level >= level
 
+      log(level: level, color: color) { ex.full_message }
     end
   end
 end

data/lib/httpx/options.rb

@@ -3,19 +3,22 @@
 require "socket"
 
 module HTTPX
+  # Contains a set of options which are passed and shared across from session to its requests or
+  # responses.
   class Options
+    BUFFER_SIZE = 1 << 14
     WINDOW_SIZE = 1 << 14 # 16K
     MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
-    CONNECT_TIMEOUT = 60
-    OPERATION_TIMEOUT = 60
     KEEP_ALIVE_TIMEOUT = 20
     SETTINGS_TIMEOUT = 10
-    READ_TIMEOUT = WRITE_TIMEOUT = REQUEST_TIMEOUT = Float::INFINITY
+    CLOSE_HANDSHAKE_TIMEOUT = 10
+    CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
+    REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
 
     # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
     ip_address_families = begin
       list = Socket.ip_address_list
-      if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+      if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? && !a.ipv6_unique_local? }
         [Socket::AF_INET6, Socket::AF_INET]
       else
         [Socket::AF_INET]
@@ -25,14 +28,19 @@ module HTTPX
     end
 
     DEFAULT_OPTIONS = {
+      :max_requests => Float::INFINITY,
       :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
       :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
       :ssl => {},
       :http2_settings => { settings_enable_push: 0 },
       :fallback_protocol => "http/1.1",
+      :supported_compression_formats => %w[gzip deflate],
+      :decompress_response_body => true,
+      :compress_request_body => true,
       :timeout => {
         connect_timeout: CONNECT_TIMEOUT,
         settings_timeout: SETTINGS_TIMEOUT,
+        close_handshake_timeout: CLOSE_HANDSHAKE_TIMEOUT,
         operation_timeout: OPERATION_TIMEOUT,
         keep_alive_timeout: KEEP_ALIVE_TIMEOUT,
         read_timeout: READ_TIMEOUT,
@@ -41,6 +49,7 @@ module HTTPX
       },
       :headers => {},
       :window_size => WINDOW_SIZE,
+      :buffer_size => BUFFER_SIZE,
       :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
       :request_class => Class.new(Request),
       :response_class => Class.new(Response),
@@ -50,7 +59,6 @@ module HTTPX
       :connection_class => Class.new(Connection),
       :options_class => Class.new(self),
       :transport => nil,
-      :transport_options => nil,
       :addresses => nil,
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
@@ -58,28 +66,6 @@ module HTTPX
       :ip_families => ip_address_families,
     }.freeze
 
-    begin
-      module HashExtensions
-        refine Hash do
-          def >=(other)
-            Hash[other] <= self
-          end
-
-          def <=(other)
-            other = Hash[other]
-            return false unless size <= other.size
-
-            each do |k, v|
-              v2 = other.fetch(k) { return false }
-              return false unless v2 == v
-            end
-            true
-          end
-        end
-      end
-      using HashExtensions
-    end unless Hash.method_defined?(:>=)
-
     class << self
       def new(options = {})
         # let enhanced options go through
@@ -98,38 +84,54 @@ module HTTPX
 
         attr_reader(optname)
       end
-
-      def def_option(optname, *args, &block)
-        if args.size.zero? && !block
-          class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(v); v; end # def option_smth(v); v; end
-          OUT
-          return
-        end
-
-        deprecated_def_option(optname, *args, &block)
-      end
-
-      def deprecated_def_option(optname, layout = nil, &interpreter)
-        warn "DEPRECATION WARNING: using `def_option(#{optname})` for setting options is deprecated. " \
-             "Define module OptionsMethods and `def option_#{optname}(val)` instead."
-
-        if layout
-          class_eval(<<-OUT, __FILE__, __LINE__ + 1)
-            def option_#{optname}(value)  # def option_origin(v)
-              #{layout}                   #   URI(v)
-            end                           # end
-          OUT
-        elsif interpreter
-          define_method(:"option_#{optname}") do |value|
-            instance_exec(value, &interpreter)
-          end
-        end
-      end
     end
 
+    # creates a new options instance from a given hash, which optionally define the following:
+    #
+    # :debug :: an object which log messages are written to (must respond to <tt><<</tt>)
+    # :debug_level :: the log level of messages (can be 1, 2, or 3).
+    # :ssl :: a hash of options which can be set as params of OpenSSL::SSL::SSLContext (see HTTPX::IO::SSL)
+    # :http2_settings :: a hash of options to be passed to a HTTP2Next::Connection (ex: <tt>{ max_concurrent_streams: 2 }</tt>)
+    # :fallback_protocol :: version of HTTP protocol to use by default in the absence of protocol negotiation
+    #                       like ALPN (defaults to <tt>"http/1.1"</tt>)
+    # :supported_compression_formats :: list of compressions supported by the transcoder layer (defaults to <tt>%w[gzip deflate]</tt>).
+    # :decompress_response_body :: whether to auto-decompress response body (defaults to <tt>true</tt>).
+    # :compress_request_body :: whether to auto-decompress response body (defaults to <tt>true</tt>)
+    # :timeout :: hash of timeout configurations (supports <tt>:connect_timeout</tt>, <tt>:settings_timeout</tt>,
+    #             <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>,  <tt>:read_timeout</tt>,  <tt>:write_timeout</tt>
+    #             and <tt>:request_timeout</tt>
+    # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)
+    # :window_size :: number of bytes to read from a socket
+    # :buffer_size :: internal read and write buffer size in bytes
+    # :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
+    # :request_class :: class used to instantiate a request
+    # :response_class :: class used to instantiate a response
+    # :headers_class :: class used to instantiate headers
+    # :request_body_class :: class used to instantiate a request body
+    # :response_body_class :: class used to instantiate a response body
+    # :connection_class :: class used to instantiate connections
+    # :options_class :: class used to instantiate options
+    # :transport :: type of transport to use (set to "unix" for UNIX sockets)
+    # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
+    #               paths should be used for UNIX sockets instead)
+    # :io :: open socket, or domain/ip-to-socket hash, which requests should be sent to
+    # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
+    # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
+    #                    using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
+    # :resolver_options :: hash of options passed to the resolver
+    # :ip_families :: which socket families are supported (system-dependent)
+    # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
+    # :base_path :: path to prefix given relative paths with (ex: "/v2")
+    # :max_concurrent_requests :: max number of requests which can be set concurrently
+    # :max_requests :: max number of requests which can be made on socket before it reconnects.
+    # :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
+    # :form :: hash of array of key-values which will be form-or-multipart-encoded in requests body payload.
+    # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
+    # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
+    #
+    # This list of options are enhanced with each loaded plugin, see the plugin docs for details.
     def initialize(options = {})
-      __initialize__(options)
+      do_initialize(options)
       freeze
     end
 
@@ -140,6 +142,7 @@ module HTTPX
       @timeout.freeze
       @headers.freeze
       @addresses.freeze
+      @supported_compression_formats.freeze
     end
 
     def option_origin(value)
@@ -155,14 +158,11 @@ module HTTPX
     end
 
     def option_timeout(value)
-      timeouts = Hash[value]
-
-      if timeouts.key?(:loop_timeout)
-        warn ":loop_timeout is deprecated, use :operation_timeout instead"
-        timeouts[:operation_timeout] = timeouts.delete(:loop_timeout)
-      end
+      Hash[value]
+    end
 
-      timeouts
+    def option_supported_compression_formats(value)
+      Array(value).map(&:to_s)
     end
 
     def option_max_concurrent_requests(value)
@@ -178,16 +178,31 @@ module HTTPX
     end
 
     def option_window_size(value)
-      Integer(value)
+      value = Integer(value)
+
+      raise TypeError, ":window_size must be positive" unless value.positive?
+
+      value
+    end
+
+    def option_buffer_size(value)
+      value = Integer(value)
+
+      raise TypeError, ":buffer_size must be positive" unless value.positive?
+
+      value
     end
 
     def option_body_threshold_size(value)
-      Integer(value)
+      bytes = Integer(value)
+      raise TypeError, ":body_threshold_size must be positive" unless bytes.positive?
+
+      bytes
     end
 
     def option_transport(value)
       transport = value.to_s
-      raise TypeError, "\#{transport} is an unsupported transport type" unless IO.registry.key?(transport)
+      raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
 
       transport
     end
@@ -204,50 +219,78 @@ module HTTPX
       params form json xml body ssl http2_settings
       request_class response_class headers_class request_body_class
       response_body_class connection_class options_class
-      io fallback_protocol debug debug_level transport_options resolver_class resolver_options
+      io fallback_protocol debug debug_level resolver_class resolver_options
+      compress_request_body decompress_response_body
       persistent
     ].each do |method_name|
-      def_option(method_name)
+      class_eval(<<-OUT, __FILE__, __LINE__ + 1)
+        def option_#{method_name}(v); v; end # def option_smth(v); v; end
+      OUT
     end
 
-    REQUEST_IVARS = %i[@params @form @xml @json @body].freeze
-    private_constant :REQUEST_IVARS
+    REQUEST_BODY_IVARS = %i[@headers @params @form @xml @json @body].freeze
 
     def ==(other)
-      ivars = instance_variables | other.instance_variables
+      super || options_equals?(other)
+    end
+
+    def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
+      # headers and other request options do not play a role, as they are
+      # relevant only for the request.
+      ivars = instance_variables - ignore_ivars
+      other_ivars = other.instance_variables - ignore_ivars
+
+      return false if ivars.size != other_ivars.size
+
+      return false if ivars.sort != other_ivars.sort
+
       ivars.all? do |ivar|
-        case ivar
-        when :@headers
-          # currently, this is used to pick up an available matching connection.
-          # the headers do not play a role, as they are relevant only for the request.
-          true
-        when *REQUEST_IVARS
-          true
-        else
-          instance_variable_get(ivar) == other.instance_variable_get(ivar)
-        end
+        instance_variable_get(ivar) == other.instance_variable_get(ivar)
       end
     end
 
+    OTHER_LOOKUP = ->(obj, k, ivar_map) {
+      case obj
+      when Hash
+        obj[ivar_map[k]]
+      else
+        obj.instance_variable_get(k)
+      end
+    }
     def merge(other)
-      raise ArgumentError, "#{other} is not a valid set of options" unless other.respond_to?(:to_hash)
+      ivar_map = nil
+      other_ivars = case other
+                    when Hash
+                      ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
+                      ivar_map.keys
+                    else
+                      other.instance_variables
+      end
 
-      h2 = other.to_hash
-      return self if h2.empty?
+      return self if other_ivars.empty?
 
-      h1 = to_hash
+      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == OTHER_LOOKUP[other, ivar, ivar_map] }
 
-      return self if h1 >= h2
+      opts = dup
 
-      merged = h1.merge(h2) do |_k, v1, v2|
-        if v1.respond_to?(:merge) && v2.respond_to?(:merge)
-          v1.merge(v2)
-        else
-          v2
+      other_ivars.each do |ivar|
+        v = OTHER_LOOKUP[other, ivar, ivar_map]
+
+        unless v
+          opts.instance_variable_set(ivar, v)
+          next
         end
+
+        v = opts.__send__(:"option_#{ivar[1..-1]}", v)
+
+        orig_v = instance_variable_get(ivar)
+
+        v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
+
+        opts.instance_variable_set(ivar, v)
       end
 
-      self.class.new(merged)
+      opts
     end
 
     def to_hash
@@ -256,30 +299,45 @@ module HTTPX
       end
     end
 
-    if RUBY_VERSION > "2.4.0"
-      def initialize_dup(other)
-        instance_variables.each do |ivar|
-          instance_variable_set(ivar, other.instance_variable_get(ivar).dup)
-        end
+    def extend_with_plugin_classes(pl)
+      if defined?(pl::RequestMethods) || defined?(pl::RequestClassMethods)
+        @request_class = @request_class.dup
+        @request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
+        @request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
       end
-    else
-      def initialize_dup(other)
-        instance_variables.each do |ivar|
-          value = other.instance_variable_get(ivar)
-          value = case value
-                  when Symbol, Numeric, TrueClass, FalseClass
-                    value
-                  else
-                    value.dup
-          end
-          instance_variable_set(ivar, value)
-        end
+      if defined?(pl::ResponseMethods) || defined?(pl::ResponseClassMethods)
+        @response_class = @response_class.dup
+        @response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
+        @response_class.extend(pl::ResponseClassMethods) if defined?(pl::ResponseClassMethods)
       end
+      if defined?(pl::HeadersMethods) || defined?(pl::HeadersClassMethods)
+        @headers_class = @headers_class.dup
+        @headers_class.__send__(:include, pl::HeadersMethods) if defined?(pl::HeadersMethods)
+        @headers_class.extend(pl::HeadersClassMethods) if defined?(pl::HeadersClassMethods)
+      end
+      if defined?(pl::RequestBodyMethods) || defined?(pl::RequestBodyClassMethods)
+        @request_body_class = @request_body_class.dup
+        @request_body_class.__send__(:include, pl::RequestBodyMethods) if defined?(pl::RequestBodyMethods)
+        @request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
+      end
+      if defined?(pl::ResponseBodyMethods) || defined?(pl::ResponseBodyClassMethods)
+        @response_body_class = @response_body_class.dup
+        @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
+        @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
+      end
+      if defined?(pl::ConnectionMethods)
+        @connection_class = @connection_class.dup
+        @connection_class.__send__(:include, pl::ConnectionMethods)
+      end
+      return unless defined?(pl::OptionsMethods)
+
+      @options_class = @options_class.dup
+      @options_class.__send__(:include, pl::OptionsMethods)
     end
 
     private
 
-    def __initialize__(options = {})
+    def do_initialize(options = {})
       defaults = DEFAULT_OPTIONS.merge(options)
       defaults.each do |k, v|
         next if v.nil?

data/lib/httpx/plugins/{authentication → auth}/basic.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "base64"
+require "httpx/base64"
 
 module HTTPX
   module Plugins
@@ -11,10 +11,6 @@ module HTTPX
           @password = password
         end
 
-        def can_authenticate?(authenticate)
-          authenticate && /Basic .*/.match?(authenticate)
-        end
-
         def authenticate(*)
           "Basic #{Base64.strict_encode64("#{@user}:#{@password}")}"
         end