httpx 1.4.1 → 1.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8a0ae955506767cc7b2f0a8134dd920bc937f88aeba2f72f691489e4d2199ab
-  data.tar.gz: 96503529f27fddf76b41250f3a7a0686c93d9d6c1091255275ed1d5b2beada11
+  metadata.gz: 36f5b3d4da61a1a6c86602205a6eda217f51b40411865589587a09553eb263cb
+  data.tar.gz: 9b705a6b8bc7ebf1ec2e308ae6b49bfcc3ede818bbca0dcf4b2fb82dde1cace6
 SHA512:
-  metadata.gz: d16170323b9f5d016f496e848be6b6fb50046402ca644c2a016d05fe1af4a5b045a400b31530c8fa38f85a61351125062edce01bd04c975bc45c3205545aef71
-  data.tar.gz: a800e8814449fcf2d3149aadc5f335f458f7dc7fa537d7c676d8460b6bfcfd1a9cc0a96f3350b9492846fea563fca15605f228ef539b39ef36985df05431d193
+  metadata.gz: 72759cee17931e45580673c119fffd7339afdfe149f6a4a14afa410d11c47b0d84146a70660b5ca8ff63b1d0b6c1d16a85008416348cd48cddd3f9a9c94f3c10
+  data.tar.gz: 1233281adc13e03e5b754ef6d5d830e84335f5f8b3cc90dea33e27c9f5751f2e3f7d20e1d6064b9273678612a3e4505d8d54b3038b85855bc25a72c6d73d914c

data/README.md

@@ -157,7 +157,6 @@ All Rubies greater or equal to 2.7, and always latest JRuby and Truffleruby.
 
 * Discuss your contribution in an issue
 * Fork it
-* Make your changes, add some tests
-* Ensure all tests pass (`docker-compose -f docker-compose.yml -f docker-compose-ruby-{RUBY_VERSION}.yml run httpx bundle exec rake test`)
+* Make your changes, add some tests (follow the instructions from [here](test/README.md))
 * Open a Merge Request (that's Pull Request in Github-ish)
 * Wait for feedback

data/doc/release_notes/1_4_2.md

@@ -0,0 +1,20 @@
+# 1.4.2
+
+## Bugfixes
+
+* faraday: use default reason when none is matched by Net::HTTP::STATUS_CODES
+* native resolver: keep sending DNS queries if the socket is available, to avoid busy loops on select
+* native resolver fixes for Happy Eyeballs v2
+  * do not apply resolution delay if the IPv4 IP was not resolved via DNS
+  * ignore ALIAS if DNS response carries IP answers
+  * do not try to query for names already awaiting answer from the resolver
+  * make sure all types of errors are propagated to connections
+  * make sure next candidate is picked up if receiving NX_DOMAIN_NOT_FOUND error from resolver
+* raise error happening before any request is flushed to respective connections (avoids loop on non-actionable selector termination).
+* fix "NoMethodError: undefined method `after' for nil:NilClass", happening for requests flushed into persistent connections which errored, and were retried in a different connection before triggering the timeout callbacks from the previously-closed connection.
+
+
+## Chore
+
+* Refactor of timers to allow for explicit and more performant single timer interval cancellation.
+* default log message restructured to include info about process, thread and caller.

data/doc/release_notes/1_4_3.md

@@ -0,0 +1,11 @@
+# 1.4.3
+
+## Bugfixes
+
+* `webmock` adapter: reassign headers to signature after callbacks are called (these may change the headers before virtual send).
+* do not close request (and its body) right after sending, instead only on response close
+  * prevents retries from failing under the `:retries` plugin
+  * fixes issue when using `faraday-multipart` request bodies
+* retry request with HTTP/1 when receiving an HTTP/2 GOAWAY frame with `HTTP_1_1_REQUIRED` error code.
+* fix wrong method call on HTTP/2 PING frame with unrecognized code.
+* fix EOFError issues on connection termination for long running connections which may have already been terminated by peer and were wrongly trying to complete the HTTP/2 termination handshake.

data/lib/httpx/adapters/faraday.rb

@@ -149,7 +149,7 @@ module Faraday
 
         module ResponseMethods
           def reason
-            Net::HTTP::STATUS_CODES.fetch(@status)
+            Net::HTTP::STATUS_CODES.fetch(@status, "Non-Standard status code")
           end
         end
       end

data/lib/httpx/adapters/webmock.rb

@@ -122,6 +122,8 @@ module WebMock
             request.transition(:done)
             request.response = response
             request.emit(:response, response)
+            request_signature.headers = request.headers.to_h
+
             response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)
           elsif WebMock.net_connect_allowed?(request_signature.uri)
             if WebMock::CallbackRegistry.any_callbacks?

data/lib/httpx/callbacks.rb

@@ -4,7 +4,7 @@ module HTTPX
   module Callbacks
     def on(type, &action)
       callbacks(type) << action
-      self
+      action
     end
 
     def once(type, &block)
@@ -12,10 +12,10 @@ module HTTPX
         block.call(*args, &callback)
         :delete
       end
-      self
     end
 
     def emit(type, *args)
+      log { "emit #{type.inspect} callbacks" } if respond_to?(:log)
       callbacks(type).delete_if { |pr| :delete == pr.call(*args) } # rubocop:disable Style/YodaCondition
     end
 

data/lib/httpx/connection/http2.rb

@@ -16,6 +16,12 @@ module HTTPX
       end
     end
 
+    class PingError < Error
+      def initialize
+        super(0, :ping_error)
+      end
+    end
+
     class GoawayError < Error
       def initialize
         super(0, :no_error)
@@ -125,7 +131,7 @@ module HTTPX
     end
 
     def handle_error(ex, request = nil)
-      if ex.instance_of?(TimeoutError) && !@handshake_completed && @connection.state != :closed
+      if ex.is_a?(OperationTimeoutError) && !@handshake_completed && @connection.state != :closed
         @connection.goaway(:settings_timeout, "closing due to settings timeout")
         emit(:close_handshake)
         settings_ex = SettingsTimeoutError.new(ex.timeout, ex.message)
@@ -311,17 +317,20 @@ module HTTPX
       @streams.delete(request)
 
       if error
-        ex = Error.new(stream.id, error)
-        ex.set_backtrace(caller)
-        response = ErrorResponse.new(request, ex)
-        request.response = response
-        emit(:response, request, response)
+        case error
+        when :http_1_1_required
+          emit(:error, request, error)
+        else
+          ex = Error.new(stream.id, error)
+          ex.set_backtrace(caller)
+          response = ErrorResponse.new(request, ex)
+          request.response = response
+          emit(:response, request, response)
+        end
       else
         response = request.response
         if response && response.is_a?(Response) && response.status == 421
-          ex = MisdirectedRequestError.new(response)
-          ex.set_backtrace(caller)
-          emit(:error, request, ex)
+          emit(:error, request, :http_1_1_required)
         else
           emit(:response, request, response)
         end
@@ -352,7 +361,12 @@ module HTTPX
       is_connection_closed = @connection.state == :closed
       if error
         @buffer.clear if is_connection_closed
-        if error == :no_error
+        case error
+        when :http_1_1_required
+          while (request = @pending.shift)
+            emit(:error, request, error)
+          end
+        when :no_error
           ex = GoawayError.new
           @pending.unshift(*@streams.keys)
           @drains.clear
@@ -360,8 +374,11 @@ module HTTPX
         else
           ex = Error.new(0, error)
         end
-        ex.set_backtrace(caller)
-        handle_error(ex)
+
+        if ex
+          ex.set_backtrace(caller)
+          handle_error(ex)
+        end
       end
       return unless is_connection_closed && @streams.empty?
 
@@ -403,11 +420,9 @@ module HTTPX
     end
 
     def on_pong(ping)
-      if @pings.delete(ping.to_s)
-        emit(:pong)
-      else
-        close(:protocol_error, "ping payload did not match")
-      end
+      raise PingError unless @pings.delete(ping.to_s)
+
+      emit(:pong)
     end
   end
 end

data/lib/httpx/connection.rb

@@ -101,8 +101,6 @@ module HTTPX
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
 
-      @intervals = []
-
       self.addresses = @options.addresses if @options.addresses
     end
 
@@ -337,15 +335,7 @@ module HTTPX
     end
 
     def handle_socket_timeout(interval)
-      @intervals.delete_if(&:elapsed?)
-
-      unless @intervals.empty?
-        # remove the intervals which will elapse
-
-        return
-      end
-
-      error = HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
+      error = OperationTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
       on_error(error)
     end
@@ -379,18 +369,20 @@ module HTTPX
       force_reset(true)
     end
 
-    private
-
-    def connect
-      transition(:open)
-    end
-
     def disconnect
+      return unless @current_session && @current_selector
+
       emit(:close)
       @current_session = nil
       @current_selector = nil
     end
 
+    private
+
+    def connect
+      transition(:open)
+    end
+
     def consume
       return unless @io
 
@@ -431,6 +423,8 @@ module HTTPX
             siz = @io.read(@window_size, @read_buffer)
             log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
             unless siz
+              @write_buffer.clear
+
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
               on_error(ex)
@@ -485,6 +479,8 @@ module HTTPX
             end
             log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
             unless siz
+              @write_buffer.clear
+
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
               on_error(ex)
@@ -616,9 +612,9 @@ module HTTPX
       parser.on(:timeout) do |tout|
         @timeout = tout
       end
-      parser.on(:error) do |request, ex|
-        case ex
-        when MisdirectedRequestError
+      parser.on(:error) do |request, error|
+        case error
+        when :http_1_1_required
           current_session = @current_session
           current_selector = @current_selector
           parser.close
@@ -628,11 +624,15 @@ module HTTPX
           other_connection.merge(self)
           request.transition(:idle)
           other_connection.send(request)
-        else
-          response = ErrorResponse.new(request, ex)
-          request.response = response
-          request.emit(:response, response)
+          next
+        when OperationTimeoutError
+          # request level timeouts should take precedence
+          next unless request.active_timeouts.empty?
         end
+
+        response = ErrorResponse.new(request, error)
+        request.response = response
+        request.emit(:response, response)
       end
     end
 
@@ -654,12 +654,14 @@ module HTTPX
       error.set_backtrace(e.backtrace)
       handle_connect_error(error) if connecting?
       @state = :closed
+      purge_after_closed
       disconnect
     rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
       handle_connect_error(e) if connecting?
       @state = :closed
+      purge_after_closed
       disconnect
     end
 
@@ -812,7 +814,7 @@ module HTTPX
     end
 
     def on_error(error, request = nil)
-      if error.instance_of?(TimeoutError)
+      if error.is_a?(OperationTimeoutError)
 
         # inactive connections do not contribute to the select loop, therefore
         # they should not fail due to such errors.
@@ -857,7 +859,7 @@ module HTTPX
 
       return if read_timeout.nil? || read_timeout.infinite?
 
-      set_request_timeout(request, read_timeout, :done, :response) do
+      set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
         read_timeout_callback(request, read_timeout)
       end
     end
@@ -867,7 +869,7 @@ module HTTPX
 
       return if write_timeout.nil? || write_timeout.infinite?
 
-      set_request_timeout(request, write_timeout, :headers, %i[done response]) do
+      set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
         write_timeout_callback(request, write_timeout)
       end
     end
@@ -877,7 +879,7 @@ module HTTPX
 
       return if request_timeout.nil? || request_timeout.infinite?
 
-      set_request_timeout(request, request_timeout, :headers, :complete) do
+      set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
         read_timeout_callback(request, request_timeout, RequestTimeoutError)
       end
     end
@@ -902,21 +904,18 @@ module HTTPX
       on_error(error, request)
     end
 
-    def set_request_timeout(request, timeout, start_event, finish_events, &callback)
-      request.once(start_event) do
-        interval = @current_selector.after(timeout, callback)
+    def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
+      request.set_timeout_callback(start_event) do
+        timer = @current_selector.after(timeout, callback)
+        request.active_timeouts << label
 
         Array(finish_events).each do |event|
           # clean up request timeouts if the connection errors out
-          request.once(event) do
-            if @intervals.include?(interval)
-              interval.delete(callback)
-              @intervals.delete(interval) if interval.no_callbacks?
-            end
+          request.set_timeout_callback(event) do
+            timer.cancel
+            request.active_timeouts.delete(label)
           end
         end
-
-        @intervals << interval
       end
     end
 

data/lib/httpx/errors.rb

@@ -77,6 +77,9 @@ module HTTPX
   # Error raised when there was a timeout while resolving a domain to an IP.
   class ResolveTimeoutError < TimeoutError; end
 
+  # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
+  class OperationTimeoutError < TimeoutError; end
+
   # Error raised when there was an error while resolving a domain to an IP.
   class ResolveError < Error; end
 
@@ -112,8 +115,4 @@ module HTTPX
       @response.status
     end
   end
-
-  # error raised when a request was sent a server which can't reproduce a response, and
-  # has therefore returned an HTTP response using the 421 status code.
-  class MisdirectedRequestError < HTTPError; end
 end

data/lib/httpx/loggable.rb

@@ -15,20 +15,27 @@ module HTTPX
 
     USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
 
-    def log(level: @options.debug_level, color: nil, &msg)
-      return unless @options.debug_level >= level
+    def log(level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug, &msg)
+      return unless debug_level >= level
 
-      debug_stream = @options.debug || ($stderr if USE_DEBUG_LOG)
+      debug_stream = debug || ($stderr if USE_DEBUG_LOG)
 
       return unless debug_stream
 
-      message = (+"" << msg.call << "\n")
+      klass = self.class
+
+      until (class_name = klass.name)
+        klass = klass.superclass
+      end
+
+      message = +"(pid:#{Process.pid} tid:#{Thread.current.object_id}, self:#{class_name}##{object_id}) "
+      message << msg.call << "\n"
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
       debug_stream << message
     end
 
-    def log_exception(ex, level: @options.debug_level, color: nil)
-      log(level: level, color: color) { ex.full_message }
+    def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
+      log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
     end
   end
 end

data/lib/httpx/plugins/callbacks.rb

@@ -25,6 +25,7 @@ module HTTPX
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
             def on_#{meth}(&blk)   # def on_connection_opened(&blk)
               on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
+              self                 #   self
             end                    # end
           MOD
         end

data/lib/httpx/plugins/circuit_breaker.rb

@@ -36,6 +36,7 @@ module HTTPX
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
         def on_#{meth}(&blk)   # def on_circuit_open(&blk)
           on(:#{meth}, &blk)   #   on(:circuit_open, &blk)
+          self                 #   self
         end                    # end
           MOD
         end

data/lib/httpx/plugins/expect.rb

@@ -84,7 +84,7 @@ module HTTPX
 
           return if expect_timeout.nil? || expect_timeout.infinite?
 
-          set_request_timeout(request, expect_timeout, :expect, %i[body response]) do
+          set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
             # expect timeout expired
             if request.state == :expect && !request.expects?
               Expect.no_expect_store << request.origin

data/lib/httpx/plugins/internal_telemetry.rb

@@ -13,6 +13,12 @@ module HTTPX
     # by the end user in $http_init_time, different diff metrics can be shown. The "point of time" is calculated
     # using the monotonic clock.
     module InternalTelemetry
+      DEBUG_LEVEL = 3
+
+      def self.extra_options(options)
+        options.merge(debug_level: 3)
+      end
+
       module TrackTimeMethods
         private
 
@@ -28,7 +34,19 @@ module HTTPX
           after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
           # $http_init_time = after_time
           elapsed = after_time - prev_time
-          warn(+"\e[31m" << "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m")
+          # klass = self.class
+
+          # until (class_name = klass.name)
+          #   klass = klass.superclass
+          # end
+          log(
+            level: DEBUG_LEVEL,
+            color: :red,
+            debug_level: @options ? @options.debug_level : DEBUG_LEVEL,
+            debug: nil
+          ) do
+            "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m"
+          end
         end
       end
 
@@ -88,6 +106,7 @@ module HTTPX
 
       module RequestMethods
         def self.included(klass)
+          klass.prepend Loggable
           klass.prepend TrackTimeMethods
           super
         end
@@ -114,6 +133,7 @@ module HTTPX
 
       module PoolMethods
         def self.included(klass)
+          klass.prepend Loggable
           klass.prepend TrackTimeMethods
           super
         end

data/lib/httpx/plugins/retries.rb

@@ -167,7 +167,7 @@ module HTTPX
           unless response.headers.key?("accept-ranges") &&
                  response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
                  (original_body = response.body)
-            response.close if response.respond_to?(:close)
+            response.body.close
             return
           end
 

data/lib/httpx/request.rb

@@ -45,9 +45,14 @@ module HTTPX
 
     attr_writer :persistent
 
+    attr_reader :active_timeouts
+
     # will be +true+ when request body has been completely flushed.
     def_delegator :@body, :empty?
 
+    # closes the body
+    def_delegator :@body, :close
+
     # initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
     # an absolute or relative +uri+ (either as String or URI::HTTP object), the
     # request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
@@ -100,6 +105,7 @@ module HTTPX
       @response = nil
       @peer_address = nil
       @persistent = @options.persistent
+      @active_timeouts = []
     end
 
     # the read timeout defined for this requet.
@@ -245,8 +251,10 @@ module HTTPX
         @body.rewind
         @response = nil
         @drainer = nil
+        @active_timeouts.clear
       when :headers
         return unless @state == :idle
+
       when :body
         return unless @state == :headers ||
                       @state == :expect
@@ -268,7 +276,6 @@ module HTTPX
       when :done
         return if @state == :expect
 
-        @body.close
       end
       @state = nextstate
       emit(@state, self)
@@ -279,6 +286,15 @@ module HTTPX
     def expects?
       @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end
+
+    def set_timeout_callback(event, &callback)
+      clb = once(event, &callback)
+
+      # reset timeout callbacks when requests get rerouted to a different connection
+      once(:idle) do
+        callbacks(event).delete(clb)
+      end
+    end
   end
 end