httpx 0.22.4 → 0.23.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d5e3e33caae5726e0c7630721296abac27073ed1fc071a7bb8c9467a2b1b2e0
-  data.tar.gz: f4344aee8002ecbe64d6e3321e1ca9f18507e2fd70223890211e05ab15967479
+  metadata.gz: 71f84ab5eb126b7c22ceeabf6ef36935e678376ad79e3585a1c8af0cdadcb36c
+  data.tar.gz: 98169826112b92768e6592ec9dcc5a0ebf0dcc34f063a4480d17b4ee4bc3f345
 SHA512:
-  metadata.gz: 9c42940a7467f91f66779bdf26b83043fb99b17033d69cee7d7656946abc03a064f1b9952889b3c783b757429d5cc6d476575ff9ab347524738152f846c3ea91
-  data.tar.gz: 6199caed20de123a10af893a85c2657cc4b4bfb6645592a5155743e74a37ff754d3fcf404c8bb28582bead974b35c6f872d40d8b5ac77806bdc4b34992847e7a
+  metadata.gz: 3ebd7512c81d95e87284d5a622d473df66a618597b502eccb17cb7ec738364f46d2dec85b1b63940b6182356ccd11caaef0867f56f671320d3a9f9c04e6ad47b
+  data.tar.gz: e98bb305329b8982414eb95d2455ef436703ba328175ddd383170c68463aa7d1b83721ac9ffd74fdd91190b0ce055668eac8c67c23648ac7493dfdc01d96e3a6

data/doc/release_notes/0_22_5.md

@@ -0,0 +1,6 @@
+# 0.22.5
+
+## Bugfixes
+
+* `datadog` and `sentry` integrations did not account for `Connection#send` being possibly called multiple times (something possible for connection coalescing, max requests exhaustion, or Happy Eyeballs 2), and were registering multiple `on(:response)` callbacks. Requests are now marked when decorated the first time.
+* Happy Eyeballs handshake "connect errors" routine is now taking both name resolution errors, as well as TLS handshake errors, into account, when the handshake fails.

data/doc/release_notes/0_23_0.md

@@ -0,0 +1,42 @@
+# 0.23.0
+
+## Features
+
+### `:retries` plugin: resumable requests
+
+The `:retries` plugin will now support scenarios where, if the request being retried supports the `range` header, and a partial response has been already buffered, the retry will resume from there and only download the missing data.
+
+#### HTTPX::ErrorResponse#response
+
+As a result, ´HTTPX::ErrorResponse#response` has also been introduced; error responses may have an actual response. This happens in cases where the request failed **after** a partial response was initiated.
+
+#### `:buffer_size` option
+
+A nnew option, `:buffer_size`, can be used to tweak the buffers used by the read/write socket routines (16k by default, you can lower it in memory-constrained environments).
+
+## Improvements
+
+### `:native` resolver falls back to TCP for truncated messages
+
+The `:native` resolver will repeat DNS queries to a nameserver via TCP when the first attempt is marked as truncated. This behaviour is both aligned with `getaddrinfo` and the `resolv` standard library.
+
+This introduces a new `resolver_options` option, `:socket_type`, which can now be `:tcp` if it is to remain the default.
+
+## Chore
+
+### HTTPX.build_request should receive upcased string (i.e. "GET")
+
+Functions which receive an HTTP verb should be given he verb in "upcased string" format now. The usage of symbols is still possible, but a deprecation warning will be emitted, and support will be removed in v1.0.0 .
+
+### Remove HTTPX::Registry
+
+These internal registries were a bit magical to use, difficult to debug, not thread-safe, and overall a nuisance when it came to type checking. While there is the possibility that someone was relying on it existing, nothing had ever been publicly documented.
+
+## Bugfixes
+
+* fixed proxy discovery using proxy env vars (`HTTPS_PROXY`, `NO_PROXY`...) being enabled/disabled based on first host uused in the session;
+* fixed `:no_proxy` option usage inn the `:proxy` plugin.
+* fixed `webmock` adapter to correctly disable it when `Webmock.disable!` is called.
+* fixed bug in `:digest_authentication` plugin when enabled and no credentials were passed.
+* fixed several bugs in the `sentry` adapter around breadcrumb handling.
+* fixed `:native` resolver candidate calculation by putting absolute domain at the bottom of the list.

data/lib/httpx/adapters/datadog.rb

@@ -68,7 +68,7 @@ module TRACING_MODULE # rubocop:disable Naming/ClassAndModuleCamelCase
 
             @request.on(:response, &method(:finish))
 
-            verb = @request.verb.to_s.upcase
+            verb = @request.verb
             uri = @request.uri
 
             @span = build_span
@@ -158,9 +158,19 @@ module TRACING_MODULE # rubocop:disable Naming/ClassAndModuleCamelCase
           end
         end
 
+        module RequestMethods
+          def __datadog_enable_trace!
+            return super if @__datadog_enable_trace
+
+            RequestTracer.new(self).call
+            @__datadog_enable_trace = true
+          end
+        end
+
         module ConnectionMethods
           def send(request)
-            RequestTracer.new(request).call
+            request.__datadog_enable_trace!
+
             super
           end
         end

data/lib/httpx/adapters/faraday.rb

@@ -44,7 +44,7 @@ module Faraday
             headers: env.request_headers,
             body: env.body,
           }
-          [meth, env.url, request_options]
+          [meth.to_s.upcase, env.url, request_options]
         end
 
         def options_from_env(env)

data/lib/httpx/adapters/sentry.rb

@@ -43,8 +43,8 @@ module HTTPX::Plugins
 
         request_info = extract_request_info(req)
 
-        data = if response.is_a?(HTTPX::ErrorResponse)
-          { error: res.message, **request_info }
+        data = if res.is_a?(HTTPX::ErrorResponse)
+          { error: res.error.message, **request_info }
         else
           { status: res.status, **request_info }
         end
@@ -64,7 +64,7 @@ module HTTPX::Plugins
         request_info = extract_request_info(req)
         sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
         if res.is_a?(HTTPX::ErrorResponse)
-          sentry_span.set_data(:error, res.message)
+          sentry_span.set_data(:error, res.error.message)
         else
           sentry_span.set_data(:status, res.status)
         end
@@ -75,7 +75,7 @@ module HTTPX::Plugins
         uri = req.uri
 
         result = {
-          method: req.verb.to_s.upcase,
+          method: req.verb,
         }
 
         if ::Sentry.configuration.send_default_pii
@@ -89,9 +89,19 @@ module HTTPX::Plugins
       end
     end
 
+    module RequestMethods
+      def __sentry_enable_trace!
+        return super if @__sentry_enable_trace
+
+        Tracer.call(self)
+        @__sentry_enable_trace = true
+      end
+    end
+
     module ConnectionMethods
       def send(request)
-        Tracer.call(request)
+        request.__sentry_enable_trace!
+
         super
       end
     end

data/lib/httpx/adapters/webmock.rb

@@ -23,7 +23,7 @@ module WebMock
           uri.path = uri.normalized_path.gsub("[^:]//", "/")
 
           WebMock::RequestSignature.new(
-            request.verb,
+            request.verb.downcase.to_sym,
             uri.to_s,
             body: request.body.each.to_a.join,
             headers: request.headers.to_h
@@ -122,7 +122,7 @@ module WebMock
 
       class << self
         def enable!
-          @original_session = HTTPX::Session
+          @original_session ||= HTTPX::Session
 
           webmock_session = HTTPX.plugin(Plugin)
 

data/lib/httpx/buffer.rb

@@ -31,6 +31,10 @@ module HTTPX
       @buffer.bytesize >= @limit
     end
 
+    def capacity
+      @limit - @buffer.bytesize
+    end
+
     def shift!(fin)
       @buffer = @buffer.byteslice(fin..-1) || "".b
     end

data/lib/httpx/callbacks.rb

@@ -22,12 +22,12 @@ module HTTPX
       callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
     end
 
-    protected
-
     def callbacks_for?(type)
-      @callbacks.key?(type) && !@callbacks[type].empty?
+      @callbacks.key?(type) && @callbacks[type].any?
     end
 
+    protected
+
     def callbacks(type = nil)
       return @callbacks unless type
 

data/lib/httpx/chainable.rb

@@ -2,11 +2,11 @@
 
 module HTTPX
   module Chainable
-    %i[head get post put delete trace options connect patch].each do |meth|
+    %w[head get post put delete trace options connect patch].each do |meth|
       class_eval(<<-MOD, __FILE__, __LINE__ + 1)
-        def #{meth}(*uri, **options)        # def get(*uri, **options)
-          request(:#{meth}, uri, **options) #   request(:get, uri, **options)
-        end                                 # end
+        def #{meth}(*uri, **options)                # def get(*uri, **options)
+          request("#{meth.upcase}", uri, **options) #   request("GET", uri, **options)
+        end                                         # end
       MOD
     end
 

data/lib/httpx/connection/http1.rb

@@ -106,6 +106,7 @@ module HTTPX
 
     def on_headers(h)
       @request = @requests.first
+
       return if @request.response
 
       log(level: 2) { "headers received" }
@@ -312,7 +313,7 @@ module HTTPX
     end
 
     def join_headline(request)
-      "#{request.verb.to_s.upcase} #{request.path} HTTP/#{@version.join(".")}"
+      "#{request.verb} #{request.path} HTTP/#{@version.join(".")}"
     end
 
     def join_headers(request)
@@ -367,5 +368,4 @@ module HTTPX
       UPCASED[field] || field.split("-").map(&:capitalize).join("-")
     end
   end
-  Connection.register "http/1.1", Connection::HTTP1
 end

data/lib/httpx/connection/http2.rb

@@ -208,7 +208,7 @@ module HTTPX
     def set_protocol_headers(request)
       {
         ":scheme" => request.scheme,
-        ":method" => request.verb.to_s.upcase,
+        ":method" => request.verb,
         ":path" => request.path,
         ":authority" => request.authority,
       }
@@ -317,7 +317,7 @@ module HTTPX
         emit(:response, request, response)
       else
         response = request.response
-        if response && response.status == 421
+        if response && response.is_a?(Response) && response.status == 421
           ex = MisdirectedRequestError.new(response)
           ex.set_backtrace(caller)
           emit(:error, request, ex)
@@ -412,5 +412,4 @@ module HTTPX
       end
     end
   end
-  Connection.register "h2", Connection::HTTP2
 end

data/lib/httpx/connection.rb

@@ -29,7 +29,6 @@ module HTTPX
   #
   class Connection
     extend Forwardable
-    include Registry
     include Loggable
     include Callbacks
 
@@ -39,8 +38,6 @@ module HTTPX
     require "httpx/connection/http2"
     require "httpx/connection/http1"
 
-    BUFFER_SIZE = 1 << 14
-
     def_delegator :@io, :closed?
 
     def_delegator :@write_buffer, :empty?
@@ -57,15 +54,15 @@ module HTTPX
       @origin = Utils.to_uri(uri.origin)
       @options = Options.new(options)
       @window_size = @options.window_size
-      @read_buffer = Buffer.new(BUFFER_SIZE)
-      @write_buffer = Buffer.new(BUFFER_SIZE)
+      @read_buffer = Buffer.new(@options.buffer_size)
+      @write_buffer = Buffer.new(@options.buffer_size)
       @pending = []
       on(:error, &method(:on_error))
       if @options.io
         # if there's an already open IO, get its
         # peer address, and force-initiate the parser
         transition(:already_open)
-        @io = IO.registry(@type).new(@origin, nil, @options)
+        @io = build_socket
         parser
       else
         transition(:idle)
@@ -84,7 +81,7 @@ module HTTPX
       if @io
         @io.add_addresses(addrs)
       else
-        @io = IO.registry(@type).new(@origin, addrs, @options)
+        @io = build_socket(addrs)
       end
     end
 
@@ -104,7 +101,7 @@ module HTTPX
           # was the result of coalescing. To prevent blind trust in the case where the
           # origin came from an ORIGIN frame, we're going to verify the hostname with the
           # SSL certificate
-          (@origins.size == 1 || @origin == uri.origin || (@io && @io.verify_hostname(uri.host)))
+          (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
         ) && @options == options
       ) || (match_altsvcs?(uri) && match_altsvc_options?(uri, options))
     end
@@ -118,7 +115,7 @@ module HTTPX
 
       (
         (open? && @origin == connection.origin) ||
-        !(@io.addresses & connection.addresses).empty?
+        !(@io.addresses & (connection.addresses || [])).empty?
       ) && @options == connection.options
     end
 
@@ -453,7 +450,7 @@ module HTTPX
     end
 
     def build_parser(protocol = @io.protocol)
-      parser = registry(protocol).new(@write_buffer, @options)
+      parser = self.class.parser_type(protocol).new(@write_buffer, @options)
       set_parser_callbacks(parser)
       parser
     end
@@ -538,12 +535,13 @@ module HTTPX
       # connect errors, exit gracefully
       error = ConnectionError.new(e.message)
       error.set_backtrace(e.backtrace)
-      connecting? && callbacks(:connect_error).any? ? emit(:connect_error, error) : handle_error(error)
+      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, error) : handle_error(error)
       @state = :closed
       emit(:close)
     rescue TLSError => e
       # connect errors, exit gracefully
       handle_error(e)
+      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, e) : handle_error(e)
       @state = :closed
       emit(:close)
     end
@@ -595,6 +593,17 @@ module HTTPX
       remove_instance_variable(:@timeout) if defined?(@timeout)
     end
 
+    def build_socket(addrs = nil)
+      transport_type = case @type
+                       when "tcp" then TCP
+                       when "ssl" then SSL
+                       when "unix" then UNIX
+                       else
+                         raise Error, "unsupported transport (#{@type})"
+      end
+      transport_type.new(@origin, addrs, @options)
+    end
+
     def on_error(error)
       if error.instance_of?(TimeoutError)
 
@@ -663,5 +672,16 @@ module HTTPX
       error = error_type.new(request, request.response, read_timeout)
       on_error(error)
     end
+
+    class << self
+      def parser_type(protocol)
+        case protocol
+        when "h2" then HTTP2
+        when "http/1.1" then HTTP1
+        else
+          raise Error, "unsupported protocol (##{protocol})"
+        end
+      end
+    end
   end
 end

data/lib/httpx/io/udp.rb

@@ -78,6 +78,8 @@ module HTTPX
         return 0 if ret == :wait_readable
         return if ret.nil?
 
+        log { "READ: #{buffer.bytesize} bytes..." }
+
         buffer.bytesize
       rescue IOError
       end

data/lib/httpx/io/unix.rb

@@ -1,11 +1,7 @@
 # frozen_string_literal: true
 
-require "forwardable"
-
 module HTTPX
   class UNIX < TCP
-    extend Forwardable
-
     using URIExtensions
 
     attr_reader :path
@@ -13,7 +9,7 @@ module HTTPX
     alias_method :host, :path
 
     def initialize(origin, addresses, options)
-      @addresses = addresses
+      @addresses = []
       @hostname = origin.host
       @state = :idle
       @options = Options.new(options)

data/lib/httpx/io.rb

@@ -5,13 +5,3 @@ require "httpx/io/udp"
 require "httpx/io/tcp"
 require "httpx/io/unix"
 require "httpx/io/ssl"
-
-module HTTPX
-  module IO
-    extend Registry
-    register "udp", UDP
-    register "unix", HTTPX::UNIX
-    register "tcp", TCP
-    register "ssl", SSL
-  end
-end

data/lib/httpx/options.rb

@@ -4,6 +4,7 @@ require "socket"
 
 module HTTPX
   class Options
+    BUFFER_SIZE = 1 << 14
     WINDOW_SIZE = 1 << 14 # 16K
     MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
     CONNECT_TIMEOUT = 60
@@ -41,6 +42,7 @@ module HTTPX
       },
       :headers => {},
       :window_size => WINDOW_SIZE,
+      :buffer_size => BUFFER_SIZE,
       :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
       :request_class => Class.new(Request),
       :response_class => Class.new(Response),
@@ -178,7 +180,19 @@ module HTTPX
     end
 
     def option_window_size(value)
-      Integer(value)
+      value = Integer(value)
+
+      raise TypeError, ":window_size must be positive" unless value.positive?
+
+      value
+    end
+
+    def option_buffer_size(value)
+      value = Integer(value)
+
+      raise TypeError, ":buffer_size must be positive" unless value.positive?
+
+      value
     end
 
     def option_body_threshold_size(value)
@@ -187,7 +201,7 @@ module HTTPX
 
     def option_transport(value)
       transport = value.to_s
-      raise TypeError, "\#{transport} is an unsupported transport type" unless IO.registry.key?(transport)
+      raise TypeError, "#{transport} is an unsupported transport type" unless %w[unix].include?(transport)
 
       transport
     end

data/lib/httpx/plugins/authentication/digest.rb

@@ -21,7 +21,7 @@ module HTTPX
         end
 
         def authenticate(request, authenticate)
-          "Digest #{generate_header(request.verb.to_s.upcase, request.path, authenticate)}"
+          "Digest #{generate_header(request.verb, request.path, authenticate)}"
         end
 
         private

data/lib/httpx/plugins/aws_sdk_authentication.rb

@@ -20,9 +20,7 @@ module HTTPX
           true
         end
 
-        def method_missing(*)
-          nil
-        end
+        def method_missing(*); end
       end
 
       #

data/lib/httpx/plugins/aws_sigv4.rb

@@ -71,7 +71,7 @@ module HTTPX
           end.join
 
           # canonical request
-          creq = "#{request.verb.to_s.upcase}" \
+          creq = "#{request.verb}" \
                  "\n#{request.canonical_path}" \
                  "\n#{request.canonical_query}" \
                  "\n#{canonical_headers}" \

data/lib/httpx/plugins/compression/brotli.rb

@@ -5,13 +5,13 @@ module HTTPX
     module Compression
       module Brotli
         class << self
-          def load_dependencies(_klass)
+          def load_dependencies(klass)
             require "brotli"
+            klass.plugin(:compression)
           end
 
-          def configure(klass)
-            klass.plugin(:compression)
-            klass.default_options.encodings.register "br", self
+          def extra_options(options)
+            options.merge(encodings: options.encodings.merge("br" => self))
           end
         end
 

data/lib/httpx/plugins/compression/deflate.rb

@@ -4,14 +4,19 @@ module HTTPX
   module Plugins
     module Compression
       module Deflate
-        def self.load_dependencies(_klass)
-          require "stringio"
-          require "zlib"
-        end
+        class << self
+          def load_dependencies(_klass)
+            require "stringio"
+            require "zlib"
+          end
 
-        def self.configure(klass)
-          klass.plugin(:"compression/gzip")
-          klass.default_options.encodings.register "deflate", self
+          def configure(klass)
+            klass.plugin(:"compression/gzip")
+          end
+
+          def extra_options(options)
+            options.merge(encodings: options.encodings.merge("deflate" => self))
+          end
         end
 
         module Deflater

data/lib/httpx/plugins/compression/gzip.rb

@@ -6,12 +6,14 @@ module HTTPX
   module Plugins
     module Compression
       module GZIP
-        def self.load_dependencies(*)
-          require "zlib"
-        end
+        class << self
+          def load_dependencies(*)
+            require "zlib"
+          end
 
-        def self.configure(klass)
-          klass.default_options.encodings.register "gzip", self
+          def extra_options(options)
+            options.merge(encodings: options.encodings.merge("gzip" => self))
+          end
         end
 
         class Deflater

data/lib/httpx/plugins/compression.rb

@@ -20,10 +20,7 @@ module HTTPX
         end
 
         def extra_options(options)
-          encodings = Module.new do
-            extend Registry
-          end
-          options.merge(encodings: encodings)
+          options.merge(encodings: {})
         end
       end
 
@@ -36,7 +33,7 @@ module HTTPX
         end
 
         def option_encodings(value)
-          raise TypeError, ":encodings must be a registry" unless value.respond_to?(:registry)
+          raise TypeError, ":encodings must be an Hash" unless value.is_a?(Hash)
 
           value
         end
@@ -49,7 +46,7 @@ module HTTPX
           if @headers.key?("range")
             @headers.delete("accept-encoding")
           else
-            @headers["accept-encoding"] ||= @options.encodings.registry.keys
+            @headers["accept-encoding"] ||= @options.encodings.keys
           end
         end
       end
@@ -65,7 +62,9 @@ module HTTPX
           @headers.get("content-encoding").each do |encoding|
             next if encoding == "identity"
 
-            @body = Encoder.new(@body, options.encodings.registry(encoding).deflater)
+            next unless options.encodings.key?(encoding)
+
+            @body = Encoder.new(@body, options.encodings[encoding].deflater)
           end
           @headers["content-length"] = @body.bytesize unless unbounded_body?
         end
@@ -95,7 +94,9 @@ module HTTPX
           @_inflaters = @headers.get("content-encoding").filter_map do |encoding|
             next if encoding == "identity"
 
-            inflater = @options.encodings.registry(encoding).inflater(compressed_length)
+            next unless @options.encodings.key?(encoding)
+
+            inflater = @options.encodings[encoding].inflater(compressed_length)
             # do not uncompress if there is no decoder available. In fact, we can't reliably
             # continue decompressing beyond that, so ignore.
             break unless inflater

data/lib/httpx/plugins/digest_authentication.rb

@@ -39,10 +39,7 @@ module HTTPX
           requests.flat_map do |request|
             digest = request.options.digest
 
-            unless digest
-              super(request)
-              next
-            end
+            next super(request) unless digest
 
             probe_response = wrap { super(request).first }
 

data/lib/httpx/plugins/follow_redirects.rb

@@ -103,7 +103,7 @@ module HTTPX
                                           max_redirects: max_redirects - 1)
           end
 
-          build_request(:get, redirect_uri, retry_options)
+          build_request("GET", redirect_uri, retry_options)
         end
 
         def __get_location_from_response(response)

data/lib/httpx/plugins/grpc/message.rb

@@ -47,7 +47,9 @@ module HTTPX
             data = message.byteslice(5..size + 5 - 1)
             if compressed == 1
               encodings.reverse_each do |algo|
-                inflater = encoders.registry(algo).inflater(size)
+                next unless encoders.key?(algo)
+
+                inflater = encoders[algo].inflater(size)
                 data = inflater.inflate(data)
                 size = data.bytesize
               end