httpx 1.3.4 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59239a38c693a32e279f32e5f153f606fdc83e0cdce02e6079947cc8f8d6e0d0
-  data.tar.gz: c386fbdd0409186f50390a2379c3945add29a7cafac805bbd0c780992e933437
+  metadata.gz: b8a0ae955506767cc7b2f0a8134dd920bc937f88aeba2f72f691489e4d2199ab
+  data.tar.gz: 96503529f27fddf76b41250f3a7a0686c93d9d6c1091255275ed1d5b2beada11
 SHA512:
-  metadata.gz: 5d142fdfe649e0447f3043e5eebd52776c71c44c4b141607b061895c520e32a5d1da0c5c678c6fd6cdb9f572e1d3c43165d8c26bf006420c2b89ce66c547c986
-  data.tar.gz: 77435864f6afd01fdb7fbc8e1019928411a50032c11751220062f1fff804355d0220823f18cbef430e47bc26f3c28fab826513a242222ccdb003eae949a41b97
+  metadata.gz: d16170323b9f5d016f496e848be6b6fb50046402ca644c2a016d05fe1af4a5b045a400b31530c8fa38f85a61351125062edce01bd04c975bc45c3205545aef71
+  data.tar.gz: a800e8814449fcf2d3149aadc5f335f458f7dc7fa537d7c676d8460b6bfcfd1a9cc0a96f3350b9492846fea563fca15605f228ef539b39ef36985df05431d193

data/doc/release_notes/1_4_0.md

@@ -0,0 +1,43 @@
+# 1.4.0
+
+## Features
+
+### `:content_digest` plugin
+
+The `:content_digest` can be used to calculate the digest of request payloads and set them in the `"content-digest"` header; it can also validate the integrity of responses which declare the same `"content-digest"` header.
+
+More info under https://honeyryderchuck.gitlab.io/httpx/wiki/Content-Digest
+
+## Per-session connection pools
+
+This architectural changes moves away from per-thread shared connection pools, and into per-session (also thread-safe) connection pools. Unlike before, this enables connections from a session to be reused across threads, as well as limiting the number of connections that can be open on a given origin peer. This fixes long-standing issues, such as reusing connections under a fiber scheduler loop (such as the one from the gem `async`).
+
+A new `:pool_options` option is introduced, which can be passed an hash with the following sub-options:
+
+* `:max_connections_per_origin`: maximum number of connections a pool allows (unbounded by default, for backwards compatibility).
+* `:pool_timeout`: the number of seconds a session will wait for a connection to be checked out (default: 5)
+
+More info under https://honeyryderchuck.gitlab.io/httpx/wiki/Connection-Pools
+
+
+## Improvements
+
+* `:aws_sigv4` plugin: improved digest calculation on compressed request bodies by buffering content to a tempfile.
+* `HTTPX::Response#json` will parse payload from extended json MIME types (like `application/ld+json`, `application/hal+json`, ...).
+
+## Bugfixes
+
+* `:aws_sigv4` plugin: do not try to rewind a request body which yields chunks.
+* fixed request encoding when `:json` param is passed, and the `oj` gem is used (by using the `:compat` flag).
+* native resolver: on message truncation, bubble up tcp handshake errors as resolve errors.
+* allow `HTTPX::Response#json` to accept extended JSON mime types (such as responses with `content-type: application/ld+json`)
+
+## Chore
+
+* default options are now fully frozen (in case anyone relies on overriding them).
+
+### `:xml` plugin
+
+XML encoding/decoding (via `:xml` request param, and `HTTPX::Response#xml`) is now available via the `:xml` plugin.
+
+Using `HTTPX::Response#xml` without the plugin will issue a deprecation warning.

data/doc/release_notes/1_4_1.md

@@ -0,0 +1,19 @@
+# 1.4.1
+
+## Bugfixes
+
+* several `datadog` integration bugfixes
+    * only load the `datadog` integration when the `datadog` sdk is loaded (and not other gems that may define the `Datadog` module, like `dogstatsd`)
+    * do not trace if datadog integration is loaded but disabled
+    * distributed headers are now sent along (when the configuration is enabled, which it is by default)
+* fix for handling multiple `GOAWAY` frames coming from the server (node.js servers seem to send multiple frames on connection timeout)
+* fix regression for when a url is used with `httpx` which is not `http://` or `https://` (should raise `HTTPX::UnsupportedSchemaError`)
+* worked around `IO.copy_stream` which was emitting incorrect bytes for HTTP/2 requests which bodies larger than the maximum supported frame size.
+* multipart requests: make sure that a body declared as `Pathname` is opened for reading in binary mode.
+* `webmock` integration: ensure that request events are emitted (such as plugins and integrations relying in it, such as `datadog` and the OTel integration)
+* native resolver: do not propagate successful name resolutions for connections which were already closed.
+* native resolver: fixed name resolution stalling, in a multi-request to multi-origin scenario, when a resolution timeout would happen.
+
+## Chore
+
+* refactor of the happy eyeballs and connection coalescing logic to not rely on callbacks, and instead on instance variable management (makes code more straightforward to read).

data/lib/httpx/adapters/datadog.rb

@@ -27,62 +27,54 @@ module Datadog::Tracing
       # Enables tracing for httpx requests.
       #
       # A span will be created for each request transaction; the span is created lazily only when
-      # receiving a response, and it is fed the start time stored inside the tracer object.
+      # buffering a request, and it is fed the start time stored inside the tracer object.
       #
       module Plugin
-        class RequestTracer
-          include Contrib::HttpAnnotationHelper
+        module RequestTracer
+          extend Contrib::HttpAnnotationHelper
+
+          module_function
 
           SPAN_REQUEST = "httpx.request"
 
-          # initializes the tracer object on the +request+.
-          def initialize(request)
-            @request = request
-            @start_time = nil
+          # initializes tracing on the +request+.
+          def call(request)
+            return unless configuration(request).enabled
+
+            span = nil
 
             # request objects are reused, when already buffered requests get rerouted to a different
             # connection due to connection issues, or when they already got a response, but need to
             # be retried. In such situations, the original span needs to be extended for the former,
             # while a new is required for the latter.
-            request.on(:idle) { reset }
+            request.on(:idle) do
+              span = nil
+            end
             # the span is initialized when the request is buffered in the parser, which is the closest
             # one gets to actually sending the request.
-            request.on(:headers) { call }
-          end
-
-          # sets up the span start time, while preparing the on response callback.
-          def call(*args)
-            return if @start_time
-
-            start(*args)
+            request.on(:headers) do
+              next if span
 
-            @request.once(:response, &method(:finish))
-          end
+              span = initialize_span(request, now)
+            end
 
-          private
+            request.on(:response) do |response|
+              unless span
+                next unless response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
 
-          # just sets the span init time. It can be passed a +start_time+ in cases where
-          # this is collected outside the request transaction.
-          def start(start_time = now)
-            @start_time = start_time
-          end
+                # handles the case when the +error+ happened during name resolution, which means
+                # that the tracing start point hasn't been triggered yet; in such cases, the approximate
+                # initial resolving time is collected from the connection, and used as span start time,
+                # and the tracing object in inserted before the on response callback is called.
+                span = initialize_span(request, response.error.connection.init_time)
 
-          # resets the start time for already finished request transactions.
-          def reset
-            return unless @start_time
+              end
 
-            start
+              finish(response, span)
+            end
           end
 
-          # creates the span from the collected +@start_time+ to what the +response+ state
-          # contains. It also resets internal state to allow this object to be reused.
-          def finish(response)
-            return unless @start_time
-
-            span = initialize_span
-
-            return unless span
-
+          def finish(response, span)
             if response.is_a?(::HTTPX::ErrorResponse)
               span.set_error(response.error)
             else
@@ -92,40 +84,40 @@ module Datadog::Tracing
             end
 
             span.finish
-          ensure
-            @start_time = nil
           end
 
           # return a span initialized with the +@request+ state.
-          def initialize_span
-            verb = @request.verb
-            uri = @request.uri
+          def initialize_span(request, start_time)
+            verb = request.verb
+            uri = request.uri
+
+            config = configuration(request)
 
-            span = create_span(@request)
+            span = create_span(request, config, start_time)
 
             span.resource = verb
 
             # Add additional request specific tags to the span.
 
-            span.set_tag(TAG_URL, @request.path)
+            span.set_tag(TAG_URL, request.path)
             span.set_tag(TAG_METHOD, verb)
 
             span.set_tag(TAG_TARGET_HOST, uri.host)
-            span.set_tag(TAG_TARGET_PORT, uri.port.to_s)
+            span.set_tag(TAG_TARGET_PORT, uri.port)
 
             # Tag as an external peer service
             span.set_tag(TAG_PEER_SERVICE, span.service)
 
-            if configuration[:distributed_tracing]
+            if config[:distributed_tracing]
               propagate_trace_http(
-                Datadog::Tracing.active_trace.to_digest,
-                @request.headers
+                Datadog::Tracing.active_trace,
+                request.headers
               )
             end
 
             # Set analytics sample rate
-            if Contrib::Analytics.enabled?(configuration[:analytics_enabled])
-              Contrib::Analytics.set_sample_rate(span, configuration[:analytics_sample_rate])
+            if Contrib::Analytics.enabled?(config[:analytics_enabled])
+              Contrib::Analytics.set_sample_rate(span, config[:analytics_sample_rate])
             end
 
             span
@@ -138,34 +130,34 @@ module Datadog::Tracing
             ::Datadog::Core::Utils::Time.now.utc
           end
 
-          def configuration
-            @configuration ||= Datadog.configuration.tracing[:httpx, @request.uri.host]
+          def configuration(request)
+            Datadog.configuration.tracing[:httpx, request.uri.host]
           end
 
           if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
-            def propagate_trace_http(digest, headers)
-              Datadog::Tracing::Contrib::HTTP.inject(digest, headers)
+            def propagate_trace_http(trace, headers)
+              Datadog::Tracing::Contrib::HTTP.inject(trace, headers)
             end
 
-            def create_span(request)
+            def create_span(request, configuration, start_time)
               Datadog::Tracing.trace(
                 SPAN_REQUEST,
-                service: service_name(request.uri.host, configuration, Datadog.configuration_for(self)),
+                service: service_name(request.uri.host, configuration),
                 type: TYPE_OUTBOUND,
-                start_time: @start_time
+                start_time: start_time
               )
             end
           else
-            def propagate_trace_http(digest, headers)
-              Datadog::Tracing::Propagation::HTTP.inject!(digest, headers)
+            def propagate_trace_http(trace, headers)
+              Datadog::Tracing::Propagation::HTTP.inject!(trace.to_digest, headers)
             end
 
-            def create_span(request)
+            def create_span(request, configuration, start_time)
               Datadog::Tracing.trace(
                 SPAN_REQUEST,
-                service: service_name(request.uri.host, configuration, Datadog.configuration_for(self)),
+                service: service_name(request.uri.host, configuration),
                 span_type: TYPE_OUTBOUND,
-                start_time: @start_time
+                start_time: start_time
               )
             end
           end
@@ -178,7 +170,7 @@ module Datadog::Tracing
 
             return unless Datadog::Tracing.enabled?
 
-            RequestTracer.new(self)
+            RequestTracer.call(self)
           end
         end
 
@@ -190,26 +182,6 @@ module Datadog::Tracing
 
             @init_time = ::Datadog::Core::Utils::Time.now.utc
           end
-
-          # handles the case when the +error+ happened during name resolution, which meanns
-          # that the tracing logic hasn't been injected yet; in such cases, the approximate
-          # initial resolving time is collected from the connection, and used as span start time,
-          # and the tracing object in inserted before the on response callback is called.
-          def handle_error(error, request = nil)
-            return super unless Datadog::Tracing.enabled?
-
-            return super unless error.respond_to?(:connection)
-
-            @pending.each do |req|
-              next if request and request == req
-
-              RequestTracer.new(req).call(error.connection.init_time)
-            end
-
-            RequestTracer.new(request).call(error.connection.init_time) if request
-
-            super
-          end
         end
       end
 

data/lib/httpx/adapters/faraday.rb

@@ -108,9 +108,11 @@ module Faraday
             ssl_options
           end
         else
+          # :nocov:
           def ssl_options_from_env(*)
             {}
           end
+          # :nocov:
         end
       end
 

data/lib/httpx/adapters/webmock.rb

@@ -52,16 +52,20 @@ module WebMock
       end
 
       module InstanceMethods
-        def init_connection(*)
-          connection = super
+        private
+
+        def do_init_connection(connection, selector)
+          super
+
           connection.once(:unmock_connection) do
             unless connection.addresses
-              connection.__send__(:callbacks)[:connect_error].clear
-              pool.__send__(:unregister_connection, connection)
+              # reset Happy Eyeballs, fail early
+              connection.sibling = nil
+
+              deselect_connection(connection, selector)
             end
-            pool.__send__(:resolve_connection, connection)
+            resolve_connection(connection, selector)
           end
-          connection
         end
       end
 
@@ -100,6 +104,10 @@ module WebMock
           super
         end
 
+        def terminate
+          force_reset
+        end
+
         def send(request)
           request_signature = Plugin.build_webmock_request_signature(request)
           WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
@@ -108,6 +116,10 @@ module WebMock
             response = Plugin.build_from_webmock_response(request, mock_response)
             WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, request_signature, mock_response)
             log { "mocking #{request.uri} with #{mock_response.inspect}" }
+            request.transition(:headers)
+            request.transition(:body)
+            request.transition(:trailers)
+            request.transition(:done)
             request.response = response
             request.emit(:response, response)
             response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)

data/lib/httpx/callbacks.rb

@@ -15,11 +15,6 @@ module HTTPX
       self
     end
 
-    def only(type, &block)
-      callbacks(type).clear
-      on(type, &block)
-    end
-
     def emit(type, *args)
       callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
     end

data/lib/httpx/chainable.rb

@@ -73,7 +73,7 @@ module HTTPX
         ].include?(callback)
 
         warn "DEPRECATION WARNING: calling `.#{meth}` on plain HTTPX sessions is deprecated. " \
-             "Use HTTPX.plugin(:callbacks).#{meth} instead."
+             "Use `HTTPX.plugin(:callbacks).#{meth}` instead."
 
         plugin(:callbacks).__send__(meth, *args, **options, &blk)
       else
@@ -101,4 +101,6 @@ module HTTPX
       end
     end
   end
+
+  extend Chainable
 end

data/lib/httpx/connection/http2.rb

@@ -98,10 +98,10 @@ module HTTPX
         @streams.size < @max_requests
     end
 
-    def send(request)
+    def send(request, head = false)
       unless can_buffer_more_requests?
-        @pending << request
-        return
+        head ? @pending.unshift(request) : @pending << request
+        return false
       end
       unless (stream = @streams[request])
         stream = @connection.new_stream
@@ -113,6 +113,7 @@ module HTTPX
       true
     rescue ::HTTP2::Error::StreamLimitExceeded
       @pending.unshift(request)
+      false
     end
 
     def consume
@@ -136,7 +137,7 @@ module HTTPX
 
         emit(:error, req, ex)
       end
-      @pending.each do |req|
+      while (req = @pending.shift)
         next if request && request == req
 
         emit(:error, req, ex)
@@ -154,8 +155,7 @@ module HTTPX
 
     def send_pending
       while (request = @pending.shift)
-        # TODO: this request should go back to top of stack
-        break unless send(request)
+        break unless send(request, true)
       end
     end
 
@@ -327,10 +327,14 @@ module HTTPX
         end
       end
       send(@pending.shift) unless @pending.empty?
+
       return unless @streams.empty? && exhausted?
 
-      close
-      emit(:exhausted) unless @pending.empty?
+      if @pending.empty?
+        close
+      else
+        emit(:exhausted)
+      end
     end
 
     def on_frame(bytes)