httpx 0.16.1 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 005e6856a12b549480cb2fcf8becd893e041026e9df634fa102fc410ab3527ef
-  data.tar.gz: 00722d3ffe0822eb6f434df84db78e9ffb9b53cbd103220546dc611f3763117c
+  metadata.gz: f6b2befec2e4b0093acd45f7ef9ef448ad41516510710616aded46934f1e3981
+  data.tar.gz: a9858adfacbdc27e1097b98b958f7dd1614f1207c74ec4290a54268df6270f69
 SHA512:
-  metadata.gz: e8c2c8c1ff44c845a9773e79b3564ed7491cbbcbfd6aa20ea8e93a7a4a029f5afe177c53693173b493efe54122324448ee14e34807aa37db77d3699cc7bb75e2
-  data.tar.gz: bad8e79d228c37dd3efafcf530ba7188e4759e5aca807c7b693a7102b6a2bc339663bade32d78c80a62e691f8a8378731f864639965554ab78f4f031994f0626
+  metadata.gz: 86276d59efaf3a15efe0a27fbd59bff2d005bb3bab83ee9917599854bf46eba1bfbb016b9df55172c41799d1fe82195e4bb7d82c008c1996814ec2e393be71b3
+  data.tar.gz: bd113e65cf5700f231992bb485f6c59511f114372d53078ea53c019a654e449e5012b2a1a19f889d79cee1eecd24345d3e03f3f9fa50aa6c69060138327e1d09

data/doc/release_notes/0_17_0.md

@@ -0,0 +1,49 @@
+# 0.17.0
+
+## Features
+
+### Response mime type decoders (#json, #form)
+
+https://gitlab.com/honeyryderchuck/httpx/-/wikis/Response-Handling#response-decoding
+
+Two new methods, `#json` and `#form`, were added to `HTTPX::Response`. As the name implies, they'll decode the raw payload into ruby objects you can work with.
+
+```ruby
+# after HTTPX.get("https://api.smth/endpoint-returning-json")
+response.json # same as JSON.dump(response.to_s)
+```
+
+Although not yet documented, integrating custom decoders is also possible (i.e. parsing HTML with `nokogiri` or something similar).
+
+## Improvements
+
+### Connection: reduce interest calculations
+
+Due to it being an intensive task, internal interest calculation in connections was reduce to the bare minimum.
+
+### Immutable Options, internal recycling of instances, improves memory usage in the happy path
+
+A lot of effort went into avoiding generating options objects internally whenever necessary. This means, when sending several requests with the same set of options (the most common case in `httpx` usage), internally only one object is passed around. For that, the following improvements were done:
+
+* `Options#merge` returns the same options the the options being merged are a subset of the current set of options (b126938a6547e09b726dd64298fb488891d938e9).
+* `Session#build_request` bypasses instantiation of options if it receives an `Options` object (which happens internally in the happy path, if users don't call `#build_request` directly) (3d549817cb41d4b904102fdc61afe3ecd9170893).
+* Improving internal `Session` APIs to not pass around options, and instead rely on accessing request options.
+* `Options#to_hash` does not build internal garbage arrays anymore (cc02679b804f63798f5d2136a039be1624e96ab6).
+
+### Reduce regexp operations in the HTTP/1 parser
+
+Some code paths in the HTTP/1 parser still using regular expressions were replaced by string operations accomplishing the same.
+
+### HTTP/1 improvements on the complexity of connection accounting calculations
+
+Managing open HTTP/1 connections relies on operations calculating whether there are requests waiting for completion. This relied on traversing all requests for that connectionn (O(n)); it now only checks the completion state of the first and last request of that connection, given that all requests in HTTP/1 are sequential (O(1)); this optimization brings a big improvement to persistent and pipelined requests (65261217b1270913e4bb93717e8b8dcfa775565a).
+
+## Bugfixes
+
+* fixing HTTP/1 protocol uncompliant exposing multiple values for the "Host" header (e435dd0534314508262184fb03d83124d89d2079).
+
+* Custom response finalizer introduced in 0.16.0 has been reverted. It was brought to my attention that `Tempfile` implementation already takes care of the file on GC (and `httpx` was duplicating), and the approach taken in `httpx` was buggy in several ways (not tolerant to forks, never recycled finalizers...) (aa3be21c890f92a41afcc7931f01dd24cc801f7c).
+
+## Chore
+
+RBS Typing improvements based on latest stdlib signatures additions, such as `openssl`, `digest`, `socket` and others.

data/lib/httpx/adapters/webmock.rb

@@ -19,7 +19,7 @@ module WebMock
       module InstanceMethods
         private
 
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           request_signatures = requests.map do |request|
             request_signature = _build_webmock_request_signature(request)
             WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
@@ -47,7 +47,7 @@ module WebMock
 
           unless real_requests.empty?
             reqs = real_requests.keys
-            reqs.zip(super(*reqs, options)).each do |req, res|
+            reqs.zip(super(*reqs)).each do |req, res|
               idx = real_requests[req]
 
               if WebMock::CallbackRegistry.any_callbacks?

data/lib/httpx/chainable.rb

@@ -78,7 +78,7 @@ module HTTPX
       with(option.to_sym => (args.first || options))
     end
 
-    def respond_to_missing?(meth)
+    def respond_to_missing?(meth, *)
       return super unless meth =~ /\Awith_(.+)/
 
       option = Regexp.last_match(1)

data/lib/httpx/connection/http1.rb

@@ -59,7 +59,12 @@ module HTTPX
     def empty?
       # this means that for every request there's an available
       # partial response, so there are no in-flight requests waiting.
-      @requests.empty? || @requests.all? { |request| !request.response.nil? }
+      @requests.empty? || (
+        # checking all responses can be time-consuming. Alas, as in HTTP/1, responses
+        # do not come out of order, we can get away with checking first and last.
+        !@requests.first.response.nil? &&
+        (@requests.size == 1 || !@requests.last.response.nil?)
+      )
     end
 
     def <<(data)
@@ -260,7 +265,7 @@ module HTTPX
     def set_protocol_headers(request)
       if !request.headers.key?("content-length") &&
          request.body.bytesize == Float::INFINITY
-        request.chunk!
+        request.body.chunk!
       end
 
       connection = request.headers["connection"]
@@ -285,10 +290,9 @@ module HTTPX
         end
       end
 
-      {
-        "host" => (request.headers["host"] || request.authority),
-        "connection" => connection,
-      }
+      extra_headers = { "connection" => connection }
+      extra_headers["host"] = request.authority unless request.headers.key?("host")
+      extra_headers
     end
 
     def headline_uri(request)
@@ -318,7 +322,7 @@ module HTTPX
     end
 
     def join_body(request)
-      return if request.empty?
+      return if request.body.empty?
 
       while (chunk = request.drain_body)
         log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
@@ -327,7 +331,9 @@ module HTTPX
         throw(:buffer_full, request) if @buffer.full?
       end
 
-      raise request.drain_error if request.drain_error
+      return unless (error = request.drain_error)
+
+      raise error
     end
 
     def join_trailers(request)
@@ -354,7 +360,7 @@ module HTTPX
     }.freeze
 
     def capitalized(field)
-      UPCASED[field] || field.to_s.split("-").map(&:capitalize).join("-")
+      UPCASED[field] || field.split("-").map(&:capitalize).join("-")
     end
   end
   Connection.register "http/1.1", Connection::HTTP1

data/lib/httpx/connection/http2.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "securerandom"
-require "io/wait"
 require "http/2/next"
 
 module HTTPX
@@ -56,7 +55,7 @@ module HTTPX
 
       return :w if !@pending.empty? && can_buffer_more_requests?
 
-      return :w if @streams.each_key.any? { |r| r.interests == :w }
+      return :w unless @drains.empty?
 
       if @buffer.empty?
         return if @streams.empty? && @pings.empty?
@@ -218,7 +217,7 @@ module HTTPX
       log(level: 1, color: :yellow) do
         request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
       end
-      stream.headers(request.headers.each(extra_headers), end_stream: request.empty?)
+      stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
     end
 
     def join_trailers(stream, request)
@@ -234,7 +233,7 @@ module HTTPX
     end
 
     def join_body(stream, request)
-      return if request.empty?
+      return if request.body.empty?
 
       chunk = @drains.delete(request) || request.drain_body
       while chunk
@@ -249,7 +248,9 @@ module HTTPX
         chunk = next_chunk
       end
 
-      on_stream_refuse(stream, request, request.drain_error) if request.drain_error
+      return unless (error = request.drain_error)
+
+      on_stream_refuse(stream, request, error)
     end
 
     ######
@@ -257,8 +258,10 @@ module HTTPX
     ######
 
     def on_stream_headers(stream, request, h)
-      if request.response && request.response.version == "2.0"
-        on_stream_trailers(stream, request, h)
+      response = request.response
+
+      if response.is_a?(Response) && response.version == "2.0"
+        on_stream_trailers(stream, response, h)
         return
       end
 
@@ -274,11 +277,11 @@ module HTTPX
       handle(request, stream) if request.expects?
     end
 
-    def on_stream_trailers(stream, request, h)
+    def on_stream_trailers(stream, response, h)
       log(color: :yellow) do
         h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
       end
-      request.response.merge_headers(h)
+      response.merge_headers(h)
     end
 
     def on_stream_data(stream, request, data)
@@ -304,7 +307,7 @@ module HTTPX
         emit(:response, request, response)
       else
         response = request.response
-        if response.status == 421
+        if response && response.status == 421
           ex = MisdirectedRequestError.new(response)
           ex.set_backtrace(caller)
           emit(:error, request, ex)

data/lib/httpx/connection.rb

@@ -313,7 +313,7 @@ module HTTPX
 
             # exit #consume altogether if all outstanding requests have been dealt with
             return if @pending.size.zero? && @inflight.zero?
-          end unless (interests.nil? || interests == :w || @state == :closing) && !epiped
+          end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
 
           #
           # tight write loop.
@@ -360,19 +360,18 @@ module HTTPX
             break if interests == :r || @state == :closing || @state == :closed
 
             write_drained = false
-          end unless interests == :r
+          end unless (ints = interests) == :r
 
           send_pending if @state == :open
 
           # return if socket is drained
-          next unless (interests != :r || read_drained) &&
-                      (interests != :w || write_drained)
+          next unless (ints != :r || read_drained) && (ints != :w || write_drained)
 
           # gotta go back to the event loop. It happens when:
           #
           # * the socket is drained of bytes or it's not the interest of the conn to read;
           # * theres nothing more to write, or it's not in the interest of the conn to write;
-          log(level: 3) { "(#{interests}): WAITING FOR EVENTS..." }
+          log(level: 3) { "(#{ints}): WAITING FOR EVENTS..." }
           return
         end
       end

data/lib/httpx/headers.rb

@@ -57,7 +57,7 @@ module HTTPX
     def merge(other)
       headers = dup
       other.each do |field, value|
-        headers[field] = value
+        headers[downcased(field)] = value
       end
       headers
     end

data/lib/httpx/options.rb

@@ -39,6 +39,28 @@ module HTTPX
       :resolver_options => { cache: true },
     }.freeze
 
+    begin
+      module HashExtensions
+        refine Hash do
+          def >=(other)
+            Hash[other] <= self
+          end
+
+          def <=(other)
+            other = Hash[other]
+            return false unless size <= other.size
+
+            each do |k, v|
+              v2 = other.fetch(k) { return false }
+              return false unless v2 == v
+            end
+            true
+          end
+        end
+      end
+      using HashExtensions
+    end unless Hash.method_defined?(:>=)
+
     class << self
       def new(options = {})
         # let enhanced options go through
@@ -89,7 +111,7 @@ module HTTPX
 
     def initialize(options = {})
       defaults = DEFAULT_OPTIONS.merge(options)
-      defaults.each do |(k, v)|
+      defaults.each do |k, v|
         next if v.nil?
 
         begin
@@ -163,6 +185,7 @@ module HTTPX
     end
 
     REQUEST_IVARS = %i[@params @form @json @body].freeze
+    private_constant :REQUEST_IVARS
 
     def ==(other)
       ivars = instance_variables | other.instance_variables
@@ -180,14 +203,14 @@ module HTTPX
     end
 
     def merge(other)
-      raise ArgumentError, "#{other.inspect} is not a valid set of options" unless other.respond_to?(:to_hash)
+      raise ArgumentError, "#{other} is not a valid set of options" unless other.respond_to?(:to_hash)
 
       h2 = other.to_hash
       return self if h2.empty?
 
       h1 = to_hash
 
-      return self if h1 == h2
+      return self if h1 >= h2
 
       merged = h1.merge(h2) do |_k, v1, v2|
         if v1.respond_to?(:merge) && v2.respond_to?(:merge)
@@ -201,10 +224,9 @@ module HTTPX
     end
 
     def to_hash
-      hash_pairs = instance_variables.map do |ivar|
-        [ivar[1..-1].to_sym, instance_variable_get(ivar)]
+      instance_variables.each_with_object({}) do |ivar, hs|
+        hs[ivar[1..-1].to_sym] = instance_variable_get(ivar)
       end
-      Hash[hash_pairs]
     end
 
     if RUBY_VERSION > "2.4.0"

data/lib/httpx/parser/http1.rb

@@ -60,7 +60,7 @@ module HTTPX
         (m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
           raise(Error, "wrong head line format")
         version, code, _ = m.captures
-        raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
+        raise(Error, "unsupported HTTP version (HTTP/#{version})") unless version && VERSIONS.include?(version)
 
         @http_version = version.split(".").map(&:to_i)
         @status_code = code.to_i
@@ -72,9 +72,14 @@ module HTTPX
 
       def parse_headers
         headers = @headers
-        while (idx = @buffer.index("\n"))
-          line = @buffer.byteslice(0..idx).sub(/\s+\z/, "")
-          @buffer = @buffer.byteslice((idx + 1)..-1)
+        buffer = @buffer
+
+        while (idx = buffer.index("\n"))
+          line = buffer.byteslice(0..idx)
+          raise Error, "wrong header format" if line.start_with?("\s", "\t")
+
+          line.lstrip!
+          buffer = @buffer = buffer.byteslice((idx + 1)..-1)
           if line.empty?
             case @state
             when :headers
@@ -97,9 +102,8 @@ module HTTPX
           raise Error, "wrong header format" unless separator_index
 
           key = line.byteslice(0..(separator_index - 1))
-          raise Error, "wrong header format" if key.start_with?("\s", "\t")
 
-          key.strip!
+          key.rstrip! # was lstripped previously!
           value = line.byteslice((separator_index + 1)..-1)
           value.strip!
           raise Error, "wrong header format" if value.nil?

data/lib/httpx/plugins/digest_authentication.rb

@@ -40,12 +40,12 @@ module HTTPX
 
         alias_method :digest_auth, :digest_authentication
 
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           requests.flat_map do |request|
             digest = request.options.digest
 
             if digest
-              probe_response = wrap { super(request, options).first }
+              probe_response = wrap { super(request).first }
 
               if digest && !probe_response.is_a?(ErrorResponse) &&
                  probe_response.status == 401 && probe_response.headers.key?("www-authenticate") &&
@@ -56,12 +56,12 @@ module HTTPX
                 token = digest.generate_header(request, probe_response)
                 request.headers["authorization"] = "Digest #{token}"
 
-                super(request, options)
+                super(request)
               else
                 probe_response
               end
             else
-              super(request, options)
+              super(request)
             end
           end
         end

data/lib/httpx/plugins/h2c.rb

@@ -24,15 +24,19 @@ module HTTPX
         def call(connection, request, response)
           connection.upgrade_to_h2c(request, response)
         end
+
+        def extra_options(options)
+          options.merge(max_concurrent_requests: 1)
+        end
       end
 
       module InstanceMethods
-        def send_requests(*requests, options)
+        def send_requests(*requests)
           upgrade_request, *remainder = requests
 
           return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
 
-          connection = pool.find_connection(upgrade_request.uri, @options.merge(options))
+          connection = pool.find_connection(upgrade_request.uri, upgrade_request.options)
 
           return super if connection && connection.upgrade_protocol == :h2c
 
@@ -42,7 +46,7 @@ module HTTPX
           upgrade_request.headers["upgrade"] = "h2c"
           upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
 
-          super(upgrade_request, *remainder, options.merge(max_concurrent_requests: 1))
+          super(upgrade_request, *remainder)
         end
       end