httpx 0.22.5 → 0.23.0

data/lib/httpx/plugins/internal_telemetry.rb

@@ -32,6 +32,15 @@ module HTTPX
         end
       end
 
+      module NativeResolverMethods
+        def transition(nextstate)
+          state = @state
+          val = super
+          meter_elapsed_time("Resolver::Native: #{state} -> #{nextstate}")
+          val
+        end
+      end
+
       module InstanceMethods
         def self.included(klass)
           klass.prepend TrackTimeMethods
@@ -42,6 +51,13 @@ module HTTPX
           meter_elapsed_time("Session: initializing...")
           super
           meter_elapsed_time("Session: initialized!!!")
+          resolver_type = @options.resolver_class
+          resolver_type = Resolver.resolver_for(resolver_type)
+          return unless resolver_type <= Resolver::Native
+
+          resolver_type.prepend TrackTimeMethods
+          resolver_type.prepend NativeResolverMethods
+          @options = @options.merge(resolver_class: resolver_type)
         end
 
         def close(*)

data/lib/httpx/plugins/multipart.rb

@@ -40,9 +40,21 @@ module HTTPX
           require "httpx/plugins/multipart/part"
           require "httpx/plugins/multipart/mime_type_detector"
         end
+      end
+
+      module RequestBodyMethods
+        private
+
+        def initialize_body(options)
+          return FormTranscoder.encode(options.form) if options.form
+
+          super
+        end
+      end
 
-        def configure(*)
-          Transcoder.register("form", FormTranscoder)
+      module ResponseMethods
+        def form
+          decode(FormTranscoder)
         end
       end
 

data/lib/httpx/plugins/proxy/http.rb

@@ -61,7 +61,7 @@ module HTTPX
               return unless @io.connected?
 
               @parser || begin
-                @parser = registry(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
+                @parser = self.class.parser_type(@io.protocol).new(@write_buffer, @options.merge(max_concurrent_requests: 1))
                 parser = @parser
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
@@ -141,9 +141,9 @@ module HTTPX
 
         module ProxyParser
           def join_headline(request)
-            return super if request.verb == :connect
+            return super if request.verb == "CONNECT"
 
-            "#{request.verb.to_s.upcase} #{request.uri} HTTP/#{@version.join(".")}"
+            "#{request.verb} #{request.uri} HTTP/#{@version.join(".")}"
           end
 
           def set_protocol_headers(request)
@@ -161,7 +161,7 @@ module HTTPX
 
         class ConnectRequest < Request
           def initialize(uri, _options)
-            super(:connect, uri, {})
+            super("CONNECT", uri, {})
             @headers.delete("accept")
           end
 

data/lib/httpx/plugins/proxy.rb

@@ -18,6 +18,43 @@ module HTTPX
       Error = HTTPProxyError
       PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
 
+      class << self
+        def configure(klass)
+          klass.plugin(:"proxy/http")
+          klass.plugin(:"proxy/socks4")
+          klass.plugin(:"proxy/socks5")
+        end
+
+        if URI::Generic.methods.include?(:use_proxy?)
+          def use_proxy?(*args)
+            URI::Generic.use_proxy?(*args)
+          end
+        else
+          # https://github.com/ruby/uri/blob/ae07f956a4bea00b4f54a75bd40b8fa918103eed/lib/uri/generic.rb
+          def use_proxy?(hostname, addr, port, no_proxy)
+            hostname = hostname.downcase
+            dothostname = ".#{hostname}"
+            no_proxy.scan(/([^:,\s]+)(?::(\d+))?/) do |p_host, p_port|
+              if !p_port || port == p_port.to_i
+                if p_host.start_with?(".")
+                  return false if hostname.end_with?(p_host.downcase)
+                else
+                  return false if dothostname.end_with?(".#{p_host.downcase}")
+                end
+                if addr
+                  begin
+                    return false if IPAddr.new(p_host).include?(addr)
+                  rescue IPAddr::InvalidAddressError
+                    next
+                  end
+                end
+              end
+            end
+            true
+          end
+        end
+      end
+
       class Parameters
         attr_reader :uri, :username, :password, :scheme
 
@@ -77,14 +114,6 @@ module HTTPX
         end
       end
 
-      class << self
-        def configure(klass)
-          klass.plugin(:"proxy/http")
-          klass.plugin(:"proxy/socks4")
-          klass.plugin(:"proxy/socks5")
-        end
-      end
-
       module OptionsMethods
         def option_proxy(value)
           value.is_a?(Parameters) ? value : Hash[value]
@@ -94,34 +123,39 @@ module HTTPX
       module InstanceMethods
         private
 
-        def proxy_uris(uri, options)
-          @_proxy_uris ||= begin
-            uris = options.proxy ? Array(options.proxy[:uri]) : []
-            if uris.empty?
-              uri = URI(uri).find_proxy
-              uris << uri if uri
-            end
-            uris
-          end
-          return if @_proxy_uris.empty?
+        def find_connection(request, connections, options)
+          return super unless options.respond_to?(:proxy)
 
-          proxy = options.proxy
+          uri = URI(request.uri)
 
-          return { uri: uri.host } if proxy && proxy.key?(:no_proxy) && !Array(proxy[:no_proxy]).grep(uri.host).empty?
+          proxy_opts = if (next_proxy = uri.find_proxy)
+            { uri: next_proxy }
+          else
+            proxy = options.proxy
 
-          proxy_opts = { uri: @_proxy_uris.first }
-          proxy_opts = options.proxy.merge(proxy_opts) if options.proxy
-          proxy_opts
-        end
+            return super unless proxy
 
-        def find_connection(request, connections, options)
-          return super unless options.respond_to?(:proxy)
+            return super(request, connections, options.merge(proxy: nil)) unless proxy.key?(:uri)
 
-          uri = URI(request.uri)
-          next_proxy = proxy_uris(uri, options)
-          raise Error, "Failed to connect to proxy" unless next_proxy
+            @_proxy_uris ||= Array(proxy[:uri])
+
+            next_proxy = @_proxy_uris.first
+            raise Error, "Failed to connect to proxy" unless next_proxy
+
+            if proxy.key?(:no_proxy)
+              next_proxy = URI(next_proxy)
+
+              no_proxy = proxy[:no_proxy]
+              no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array)
+
+              return super(request, connections, options.merge(proxy: nil)) unless Proxy.use_proxy?(uri.host, next_proxy.host,
+                                                                                                    next_proxy.port, no_proxy)
+            end
+
+            proxy.merge(uri: next_proxy)
+          end
 
-          proxy = Parameters.new(**next_proxy) unless next_proxy[:uri] == uri.host
+          proxy = Parameters.new(**proxy_opts)
 
           proxy_options = options.merge(proxy: proxy)
           connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
@@ -284,7 +318,7 @@ module HTTPX
     register_plugin :proxy, Proxy
   end
 
-  class ProxySSL < IO.registry["ssl"]
+  class ProxySSL < SSL
     def initialize(tcp, request_uri, options)
       @io = tcp.to_io
       super(request_uri, tcp.addresses, options)

data/lib/httpx/plugins/response_cache.rb

@@ -8,7 +8,7 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Response-Cache
     #
     module ResponseCache
-      CACHEABLE_VERBS = %i[get head].freeze
+      CACHEABLE_VERBS = %w[GET HEAD].freeze
       CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
       private_constant :CACHEABLE_VERBS
       private_constant :CACHEABLE_STATUS_CODES
@@ -96,7 +96,7 @@ module HTTPX
 
       module RequestMethods
         def response_cache_key
-          @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}#{@uri}")
+          @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}-#{@uri}")
         end
       end
 

data/lib/httpx/plugins/retries.rb

@@ -11,7 +11,7 @@ module HTTPX
       MAX_RETRIES = 3
       # TODO: pass max_retries in a configure/load block
 
-      IDEMPOTENT_METHODS = %i[get options head put delete].freeze
+      IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
       RETRYABLE_ERRORS = [
         IOError,
         EOFError,
@@ -96,7 +96,7 @@ module HTTPX
                )
                # rubocop:enable Style/MultilineTernaryOperator
              )
-            response.close if response.respond_to?(:close)
+            __try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
             request.retries -= 1
             request.transition(:idle)
@@ -134,15 +134,62 @@ module HTTPX
         def __retryable_error?(ex)
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
+
+        #
+        # Atttempt to set the request to perform a partial range request.
+        # This happens if the peer server accepts byte-range requests, and
+        # the last response contains some body payload.
+        #
+        def __try_partial_retry(request, response)
+          response = response.response if response.is_a?(ErrorResponse)
+
+          return unless response
+
+          unless response.headers.key?("accept-ranges") &&
+                 response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
+                 (original_body = response.body)
+            response.close if response.respond_to?(:close)
+            return
+          end
+
+          request.partial_response = response
+
+          size = original_body.bytesize
+
+          request.headers["range"] = "bytes=#{size}-"
+        end
       end
 
       module RequestMethods
         attr_accessor :retries
 
+        attr_writer :partial_response
+
         def initialize(*args)
           super
           @retries = @options.max_retries
         end
+
+        def response=(response)
+          if @partial_response
+            if response.is_a?(Response) && response.status == 206
+              response.from_partial_response(@partial_response)
+            else
+              @partial_response.close
+            end
+            @partial_response = nil
+          end
+
+          super
+        end
+      end
+
+      module ResponseMethods
+        def from_partial_response(response)
+          @status = response.status
+          @headers = response.headers
+          @body = response.body
+        end
       end
     end
     register_plugin :retries, Retries

data/lib/httpx/plugins/upgrade/h2.rb

@@ -10,8 +10,8 @@ module HTTPX
     #
     module H2
       class << self
-        def configure(klass)
-          klass.default_options.upgrade_handlers.register "h2", self
+        def extra_options(options)
+          options.merge(upgrade_handlers: options.upgrade_handlers.merge("h2" => self))
         end
 
         def call(connection, _request, _response)
@@ -32,7 +32,7 @@ module HTTPX
 
           @parser = Connection::HTTP2.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
-          @upgrade_protocol = :h2
+          @upgrade_protocol = "h2"
 
           # what's happening here:
           # a deviation from the state machine is done to perform the actions when a

data/lib/httpx/plugins/upgrade.rb

@@ -15,16 +15,13 @@ module HTTPX
         end
 
         def extra_options(options)
-          upgrade_handlers = Module.new do
-            extend Registry
-          end
-          options.merge(upgrade_handlers: upgrade_handlers)
+          options.merge(upgrade_handlers: {})
         end
       end
 
       module OptionsMethods
         def option_upgrade_handlers(value)
-          raise TypeError, ":upgrade_handlers must be a registry" unless value.respond_to?(:registry)
+          raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
 
           value
         end
@@ -41,9 +38,9 @@ module HTTPX
 
             upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
 
-            return response unless upgrade_protocol && options.upgrade_handlers.registry.key?(upgrade_protocol)
+            return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
 
-            protocol_handler = options.upgrade_handlers.registry(upgrade_protocol)
+            protocol_handler = options.upgrade_handlers[upgrade_protocol]
 
             return response unless protocol_handler
 

data/lib/httpx/plugins/webdav.rb

@@ -10,11 +10,11 @@ module HTTPX
     module WebDav
       module InstanceMethods
         def copy(src, dest)
-          request(:copy, src, headers: { "destination" => @options.origin.merge(dest) })
+          request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
         end
 
         def move(src, dest)
-          request(:move, src, headers: { "destination" => @options.origin.merge(dest) })
+          request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
         end
 
         def lock(path, timeout: nil, &blk)
@@ -30,7 +30,7 @@ module HTTPX
                 "<D:locktype><D:write/></D:locktype>" \
                 "<D:owner>null</D:owner>" \
                 "</D:lockinfo>"
-          response = request(:lock, path, headers: headers, xml: xml)
+          response = request("LOCK", path, headers: headers, xml: xml)
 
           return response unless response.is_a?(Response)
 
@@ -46,11 +46,11 @@ module HTTPX
         end
 
         def unlock(path, lock_token)
-          request(:unlock, path, headers: { "lock-token" => lock_token })
+          request("UNLOCK", path, headers: { "lock-token" => lock_token })
         end
 
         def mkcol(dir)
-          request(:mkcol, dir)
+          request("MKCOL", dir)
         end
 
         def propfind(path, xml = nil)
@@ -64,13 +64,13 @@ module HTTPX
                    xml
           end
 
-          request(:propfind, path, headers: { "depth" => "1" }, xml: body)
+          request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
         end
 
         def proppatch(path, xml)
           body = "<?xml version=\"1.0\"?>" \
                  "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
-          request(:proppatch, path, xml: body)
+          request("PROPPATCH", path, xml: body)
         end
         # %i[ orderpatch acl report search]
       end

data/lib/httpx/pool.rb

@@ -244,7 +244,7 @@ module HTTPX
     def find_resolver_for(connection)
       connection_options = connection.options
       resolver_type = connection_options.resolver_class
-      resolver_type = Resolver.registry(resolver_type) if resolver_type.is_a?(Symbol)
+      resolver_type = Resolver.resolver_for(resolver_type)
 
       @resolvers[resolver_type] ||= begin
         resolver_manager = if resolver_type.multi?

data/lib/httpx/request.rb

@@ -19,7 +19,11 @@ module HTTPX
     def_delegator :@body, :empty?
 
     def initialize(verb, uri, options = {})
-      @verb    = verb.to_s.downcase.to_sym
+      if verb.is_a?(Symbol)
+        warn "DEPRECATION WARNING: Using symbols for `verb` is deprecated, and will not be supported in httpx 1.0. " \
+             "Use \"#{verb.to_s.upcase}\" instead."
+      end
+      @verb    = verb.to_s.upcase
       @options = Options.new(options)
       @uri     = Utils.to_uri(uri)
       if @uri.relative?
@@ -116,7 +120,7 @@ module HTTPX
 
       query = []
       if (q = @options.params)
-        query << Transcoder.registry("form").encode(q)
+        query << Transcoder::Form.encode(q)
       end
       query << @uri.query if @uri.query
       @query = query.join("&")
@@ -138,7 +142,7 @@ module HTTPX
     # :nocov:
     def inspect
       "#<HTTPX::Request:#{object_id} " \
-        "#{@verb.to_s.upcase} " \
+        "#{@verb} " \
         "#{uri} " \
         "@headers=#{@headers} " \
         "@body=#{@body}>"
@@ -156,15 +160,7 @@ module HTTPX
 
       def initialize(headers, options)
         @headers = headers
-        @body = if options.body
-          Transcoder.registry("body").encode(options.body)
-        elsif options.form
-          Transcoder.registry("form").encode(options.form)
-        elsif options.json
-          Transcoder.registry("json").encode(options.json)
-        elsif options.xml
-          Transcoder.registry("xml").encode(options.xml)
-        end
+        @body = initialize_body(options)
         return if @body.nil?
 
         @headers["content-type"] ||= @body.content_type
@@ -207,7 +203,7 @@ module HTTPX
 
       def stream(body)
         encoded = body
-        encoded = Transcoder.registry("chunker").encode(body.enum_for(:each)) if chunked?
+        encoded = Transcoder::Chunker.encode(body.enum_for(:each)) if chunked?
         encoded
       end
 
@@ -231,6 +227,20 @@ module HTTPX
           "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
       end
       # :nocov:
+
+      private
+
+      def initialize_body(options)
+        if options.body
+          Transcoder::Body.encode(options.body)
+        elsif options.form
+          Transcoder::Form.encode(options.form)
+        elsif options.json
+          Transcoder::JSON.encode(options.json)
+        elsif options.xml
+          Transcoder::Xml.encode(options.xml)
+        end
+      end
     end
 
     def transition(nextstate)

data/lib/httpx/resolver/https.rb

@@ -104,7 +104,7 @@ module HTTPX
         resolver_connection.send(request)
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError => e
-        @queries.delete(hostname)
+        reset_hostname(hostname)
         emit_resolve_error(connection, connection.origin.host, e)
       end
     end
@@ -113,7 +113,7 @@ module HTTPX
       response.raise_for_status
     rescue StandardError => e
       hostname = @requests.delete(request)
-      connection = @queries.delete(hostname)
+      connection = reset_hostname(hostname)
       emit_resolve_error(connection, connection.origin.host, e)
     else
       # @type var response: HTTPX::Response
@@ -128,30 +128,35 @@ module HTTPX
     end
 
     def parse(request, response)
-      begin
-        answers = decode_response_body(response)
-      rescue Resolv::DNS::DecodeError => e
-        host, connection = @queries.first
-        @queries.delete(host)
-        emit_resolve_error(connection, connection.origin.host, e)
-        return
-      end
+      code, result = decode_response_body(response)
 
-      if answers.nil?
+      case code
+      when :ok
+        parse_addresses(result)
+      when :no_domain_found
         # Indicates no such domain was found.
 
         host = @requests.delete(request)
-        connection = @queries.delete(host)
+        connection = reset_hostname(host)
 
-        emit_resolve_error(connection) unless @queries.value?(connection)
-      elsif answers.empty?
-        # no address found, eliminate candidates
+        emit_resolve_error(connection)
+      when :dns_error
         host = @requests.delete(request)
-        connection = @queries.delete(host)
+        connection = reset_hostname(host)
 
-        # eliminate other candidates
-        @queries.delete_if { |_, conn| connection == conn }
+        emit_resolve_error(connection)
+      when :decode_error
+        host, connection = @queries.first
+        reset_hostname(host)
+        emit_resolve_error(connection, connection.origin.host, result)
+      end
+    end
 
+    def parse_addresses(answers)
+      if answers.empty?
+        # no address found, eliminate candidates
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
         emit_resolve_error(connection)
         return
 
@@ -162,7 +167,7 @@ module HTTPX
             if address.key?("alias")
               alias_address = answers[address["alias"]]
               if alias_address.nil?
-                @queries.delete(address["name"])
+                reset_hostname(address["name"])
                 if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
                   @connections.delete(connection)
                 else
@@ -179,7 +184,7 @@ module HTTPX
           next if addresses.empty?
 
           hostname.delete_suffix!(".") if hostname.end_with?(".")
-          connection = @queries.delete(hostname)
+          connection = reset_hostname(hostname, reset_candidates: false)
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
@@ -224,5 +229,17 @@ module HTTPX
         raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
       end
     end
+
+    def reset_hostname(hostname, reset_candidates: true)
+      connection = @queries.delete(hostname)
+
+      return connection unless connection && reset_candidates
+
+      # eliminate other candidates
+      candidates = @queries.select { |_, conn| connection == conn }.keys
+      @queries.delete_if { |h, _| candidates.include?(h) }
+
+      connection
+    end
   end
 end