httpx 1.4.0 → 1.4.4

data/lib/httpx/connection.rb

@@ -41,15 +41,17 @@ module HTTPX
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session
+    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
 
-    attr_writer :current_selector, :coalesced_connection
+    attr_writer :current_selector
 
     attr_accessor :current_session, :family
 
+    protected :sibling
+
     def initialize(uri, options)
-      @current_session = @current_selector = @coalesced_connection = nil
-      @exhausted = @cloned = false
+      @current_session = @current_selector = @sibling = @coalesced_connection = nil
+      @exhausted = @cloned = @main_sibling = false
 
       @options = Options.new(options)
       @type = initialize_type(uri, @options)
@@ -70,9 +72,6 @@ module HTTPX
       else
         transition(:idle)
       end
-      on(:activate) do
-        @current_session.select_connection(self, @current_selector)
-      end
       on(:close) do
         next if @exhausted # it'll reset
 
@@ -86,10 +85,13 @@ module HTTPX
       on(:terminate) do
         next if @exhausted # it'll reset
 
+        current_session = @current_session
+        current_selector = @current_selector
+
         # may be called after ":close" above, so after the connection has been checked back in.
-        next unless @current_session
+        next unless current_session && current_selector
 
-        @current_session.deselect_connection(self, @current_selector)
+        current_session.deselect_connection(self, current_selector)
       end
 
       on(:altsvc) do |alt_origin, origin, alt_params|
@@ -99,8 +101,6 @@ module HTTPX
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
 
-      @intervals = []
-
       self.addresses = @options.addresses if @options.addresses
     end
 
@@ -194,6 +194,12 @@ module HTTPX
       end
     end
 
+    def io_connected?
+      return @coalesced_connection.io_connected? if @coalesced_connection
+
+      @io && @io.state == :connected
+    end
+
     def connecting?
       @state == :idle
     end
@@ -290,6 +296,7 @@ module HTTPX
           @pending << request
           transition(:active) if @state == :inactive
           parser.ping
+          request.ping!
           return
         end
 
@@ -329,31 +336,54 @@ module HTTPX
     end
 
     def handle_socket_timeout(interval)
-      @intervals.delete_if(&:elapsed?)
+      error = OperationTimeoutError.new(interval, "timed out while waiting on select")
+      error.set_backtrace(caller)
+      on_error(error)
+    end
 
-      unless @intervals.empty?
-        # remove the intervals which will elapse
+    def coalesced_connection=(connection)
+      @coalesced_connection = connection
 
-        return
-      end
+      close_sibling
+      connection.merge(self)
+    end
 
-      error = HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
-      error.set_backtrace(caller)
-      on_error(error)
+    def sibling=(connection)
+      @sibling = connection
+
+      return unless connection
+
+      @main_sibling = connection.sibling.nil?
+
+      return unless @main_sibling
+
+      connection.sibling = self
     end
 
-    private
+    def handle_connect_error(error)
+      @connect_error = error
 
-    def connect
-      transition(:open)
+      return handle_error(error) unless @sibling && @sibling.connecting?
+
+      @sibling.merge(self)
+
+      force_reset(true)
     end
 
     def disconnect
+      return unless @current_session && @current_selector
+
       emit(:close)
       @current_session = nil
       @current_selector = nil
     end
 
+    private
+
+    def connect
+      transition(:open)
+    end
+
     def consume
       return unless @io
 
@@ -394,6 +424,8 @@ module HTTPX
             siz = @io.read(@window_size, @read_buffer)
             log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
             unless siz
+              @write_buffer.clear
+
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
               on_error(ex)
@@ -448,6 +480,8 @@ module HTTPX
             end
             log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
             unless siz
+              @write_buffer.clear
+
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
               on_error(ex)
@@ -531,20 +565,22 @@ module HTTPX
         @exhausted = true
         current_session = @current_session
         current_selector = @current_selector
-        parser.close
-        @pending.concat(parser.pending)
+        begin
+          parser.close
+          @pending.concat(parser.pending)
+        ensure
+          @current_session = current_session
+          @current_selector = current_selector
+        end
+
         case @state
         when :closed
           idling
           @exhausted = false
-          @current_session = current_session
-          @current_selector = current_selector
         when :closing
-          once(:close) do
+          once(:closed) do
             idling
             @exhausted = false
-            @current_session = current_session
-            @current_selector = current_selector
           end
         end
       end
@@ -577,9 +613,9 @@ module HTTPX
       parser.on(:timeout) do |tout|
         @timeout = tout
       end
-      parser.on(:error) do |request, ex|
-        case ex
-        when MisdirectedRequestError
+      parser.on(:error) do |request, error|
+        case error
+        when :http_1_1_required
           current_session = @current_session
           current_selector = @current_selector
           parser.close
@@ -589,11 +625,16 @@ module HTTPX
           other_connection.merge(self)
           request.transition(:idle)
           other_connection.send(request)
-        else
-          response = ErrorResponse.new(request, ex)
-          request.response = response
-          request.emit(:response, response)
+          next
+        when OperationTimeoutError
+          # request level timeouts should take precedence
+          next unless request.active_timeouts.empty?
         end
+
+        @inflight -= 1
+        response = ErrorResponse.new(request, error)
+        request.response = response
+        request.emit(:response, response)
       end
     end
 
@@ -613,14 +654,16 @@ module HTTPX
       # connect errors, exit gracefully
       error = ConnectionError.new(e.message)
       error.set_backtrace(e.backtrace)
-      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, error) : handle_error(error)
+      handle_connect_error(error) if connecting?
       @state = :closed
+      purge_after_closed
       disconnect
     rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
-      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, e) : handle_error(e)
+      handle_connect_error(e) if connecting?
       @state = :closed
+      purge_after_closed
       disconnect
     end
 
@@ -629,12 +672,12 @@ module HTTPX
       when :idle
         @timeout = @current_timeout = @options.timeout[:connect_timeout]
 
-        @connected_at = nil
+        @connected_at = @response_received_at = nil
       when :open
         return if @state == :closed
 
         @io.connect
-        emit(:tcp_open, self) if @io.state == :connected
+        close_sibling if @io.state == :connected
 
         return unless @io.connected?
 
@@ -667,6 +710,7 @@ module HTTPX
 
         purge_after_closed
         disconnect if @pending.empty?
+
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -677,11 +721,29 @@ module HTTPX
         return unless @state == :inactive
 
         nextstate = :open
-        emit(:activate)
+
+        # activate
+        @current_session.select_connection(self, @current_selector)
       end
       @state = nextstate
     end
 
+    def close_sibling
+      return unless @sibling
+
+      if @sibling.io_connected?
+        reset
+        # TODO: transition connection to closed
+      end
+
+      unless @sibling.state == :closed
+        merge(@sibling) unless @main_sibling
+        @sibling.force_reset(true)
+      end
+
+      @sibling = nil
+    end
+
     def purge_after_closed
       @io.close if @io
       @read_buffer.clear
@@ -754,7 +816,7 @@ module HTTPX
     end
 
     def on_error(error, request = nil)
-      if error.instance_of?(TimeoutError)
+      if error.is_a?(OperationTimeoutError)
 
         # inactive connections do not contribute to the select loop, therefore
         # they should not fail due to such errors.
@@ -783,6 +845,7 @@ module HTTPX
 
       return unless request
 
+      @inflight -= 1
       response = ErrorResponse.new(request, error)
       request.response = response
       request.emit(:response, response)
@@ -799,7 +862,7 @@ module HTTPX
 
       return if read_timeout.nil? || read_timeout.infinite?
 
-      set_request_timeout(request, read_timeout, :done, :response) do
+      set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
         read_timeout_callback(request, read_timeout)
       end
     end
@@ -809,7 +872,7 @@ module HTTPX
 
       return if write_timeout.nil? || write_timeout.infinite?
 
-      set_request_timeout(request, write_timeout, :headers, %i[done response]) do
+      set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
         write_timeout_callback(request, write_timeout)
       end
     end
@@ -819,7 +882,7 @@ module HTTPX
 
       return if request_timeout.nil? || request_timeout.infinite?
 
-      set_request_timeout(request, request_timeout, :headers, :complete) do
+      set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
         read_timeout_callback(request, request_timeout, RequestTimeoutError)
       end
     end
@@ -844,21 +907,18 @@ module HTTPX
       on_error(error, request)
     end
 
-    def set_request_timeout(request, timeout, start_event, finish_events, &callback)
-      request.once(start_event) do
-        interval = @current_selector.after(timeout, callback)
+    def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
+      request.set_timeout_callback(start_event) do
+        timer = @current_selector.after(timeout, callback)
+        request.active_timeouts << label
 
         Array(finish_events).each do |event|
           # clean up request timeouts if the connection errors out
-          request.once(event) do
-            if @intervals.include?(interval)
-              interval.delete(callback)
-              @intervals.delete(interval) if interval.no_callbacks?
-            end
+          request.set_timeout_callback(event) do
+            timer.cancel
+            request.active_timeouts.delete(label)
           end
         end
-
-        @intervals << interval
       end
     end
 

data/lib/httpx/errors.rb

@@ -77,6 +77,9 @@ module HTTPX
   # Error raised when there was a timeout while resolving a domain to an IP.
   class ResolveTimeoutError < TimeoutError; end
 
+  # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
+  class OperationTimeoutError < TimeoutError; end
+
   # Error raised when there was an error while resolving a domain to an IP.
   class ResolveError < Error; end
 
@@ -112,8 +115,4 @@ module HTTPX
       @response.status
     end
   end
-
-  # error raised when a request was sent a server which can't reproduce a response, and
-  # has therefore returned an HTTP response using the 421 status code.
-  class MisdirectedRequestError < HTTPError; end
 end

data/lib/httpx/io/ssl.rb

@@ -92,9 +92,12 @@ module HTTPX
     end
 
     def connect
-      super
-      return if @state == :negotiated ||
-                @state != :connected
+      return if @state == :negotiated
+
+      unless @state == :connected
+        super
+        return unless @state == :connected
+      end
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
         if (hostname_is_ip = (@ip == @sni_hostname))

data/lib/httpx/loggable.rb

@@ -15,20 +15,27 @@ module HTTPX
 
     USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
 
-    def log(level: @options.debug_level, color: nil, &msg)
-      return unless @options.debug_level >= level
+    def log(level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug, &msg)
+      return unless debug_level >= level
 
-      debug_stream = @options.debug || ($stderr if USE_DEBUG_LOG)
+      debug_stream = debug || ($stderr if USE_DEBUG_LOG)
 
       return unless debug_stream
 
-      message = (+"" << msg.call << "\n")
+      klass = self.class
+
+      until (class_name = klass.name)
+        klass = klass.superclass
+      end
+
+      message = +"(pid:#{Process.pid} tid:#{Thread.current.object_id}, self:#{class_name}##{object_id}) "
+      message << msg.call << "\n"
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
       debug_stream << message
     end
 
-    def log_exception(ex, level: @options.debug_level, color: nil)
-      log(level: level, color: color) { ex.full_message }
+    def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
+      log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
     end
   end
 end

data/lib/httpx/plugins/callbacks.rb

@@ -25,6 +25,7 @@ module HTTPX
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
             def on_#{meth}(&blk)   # def on_connection_opened(&blk)
               on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
+              self                 #   self
             end                    # end
           MOD
         end

data/lib/httpx/plugins/circuit_breaker.rb

@@ -36,6 +36,7 @@ module HTTPX
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
         def on_#{meth}(&blk)   # def on_circuit_open(&blk)
           on(:#{meth}, &blk)   #   on(:circuit_open, &blk)
+          self                 #   self
         end                    # end
           MOD
         end

data/lib/httpx/plugins/expect.rb

@@ -84,7 +84,7 @@ module HTTPX
 
           return if expect_timeout.nil? || expect_timeout.infinite?
 
-          set_request_timeout(request, expect_timeout, :expect, %i[body response]) do
+          set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
             # expect timeout expired
             if request.state == :expect && !request.expects?
               Expect.no_expect_store << request.origin

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -29,6 +29,8 @@ module HTTPX
 
           buf = outbuf if outbuf
 
+          buf = buf.b if buf.frozen?
+
           buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
           buf
         end

data/lib/httpx/plugins/internal_telemetry.rb

@@ -13,6 +13,12 @@ module HTTPX
     # by the end user in $http_init_time, different diff metrics can be shown. The "point of time" is calculated
     # using the monotonic clock.
     module InternalTelemetry
+      DEBUG_LEVEL = 3
+
+      def self.extra_options(options)
+        options.merge(debug_level: 3)
+      end
+
       module TrackTimeMethods
         private
 
@@ -28,7 +34,19 @@ module HTTPX
           after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
           # $http_init_time = after_time
           elapsed = after_time - prev_time
-          warn(+"\e[31m" << "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m")
+          # klass = self.class
+
+          # until (class_name = klass.name)
+          #   klass = klass.superclass
+          # end
+          log(
+            level: DEBUG_LEVEL,
+            color: :red,
+            debug_level: @options ? @options.debug_level : DEBUG_LEVEL,
+            debug: nil
+          ) do
+            "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m"
+          end
         end
       end
 
@@ -88,6 +106,7 @@ module HTTPX
 
       module RequestMethods
         def self.included(klass)
+          klass.prepend Loggable
           klass.prepend TrackTimeMethods
           super
         end
@@ -114,6 +133,7 @@ module HTTPX
 
       module PoolMethods
         def self.included(klass)
+          klass.prepend Loggable
           klass.prepend TrackTimeMethods
           super
         end

data/lib/httpx/plugins/retries.rb

@@ -110,7 +110,7 @@ module HTTPX
              )
             __try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
-            request.retries -= 1
+            request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
             request.transition(:idle)
 
             retry_after = options.retry_after
@@ -167,7 +167,7 @@ module HTTPX
           unless response.headers.key?("accept-ranges") &&
                  response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
                  (original_body = response.body)
-            response.close if response.respond_to?(:close)
+            response.body.close
             return
           end
 

data/lib/httpx/plugins/stream.rb

@@ -4,27 +4,39 @@ module HTTPX
   class StreamResponse
     def initialize(request, session)
       @request = request
+      @options = @request.options
       @session = session
-      @response = nil
+      @response_enum = nil
+      @buffered_chunks = []
     end
 
     def each(&block)
       return enum_for(__method__) unless block
 
+      if (response_enum = @response_enum)
+        @response_enum = nil
+        # streaming already started, let's finish it
+
+        while (chunk = @buffered_chunks.shift)
+          block.call(chunk)
+        end
+
+        # consume enum til the end
+        begin
+          while (chunk = response_enum.next)
+            block.call(chunk)
+          end
+        rescue StopIteration
+          return
+        end
+      end
+
       @request.stream = self
 
       begin
         @on_chunk = block
 
-        if @request.response
-          # if we've already started collecting the payload, yield it first
-          # before proceeding.
-          body = @request.response.body
-
-          body.each do |chunk|
-            on_chunk(chunk)
-          end
-        end
+        response = @session.request(@request)
 
         response.raise_for_status
       ensure
@@ -64,27 +76,39 @@ module HTTPX
     # :nocov:
 
     def to_s
-      response.to_s
+      if @request.response
+        @request.response.to_s
+      else
+        @buffered_chunks.join
+      end
     end
 
     private
 
     def response
-      return @response if @response
-
       @request.response || begin
-        @response = @session.request(@request)
+        response_enum = each
+        while (chunk = response_enum.next)
+          @buffered_chunks << chunk
+          break if @request.response
+        end
+        @response_enum = response_enum
+        @request.response
       end
     end
 
-    def respond_to_missing?(meth, *args)
-      response.respond_to?(meth, *args) || super
+    def respond_to_missing?(meth, include_private)
+      if (response = @request.response)
+        response.respond_to_missing?(meth, include_private)
+      else
+        @options.response_class.method_defined?(meth) || (include_private && @options.response_class.private_method_defined?(meth))
+      end || super
     end
 
-    def method_missing(meth, *args, &block)
+    def method_missing(meth, *args, **kwargs, &block)
       return super unless response.respond_to?(meth)
 
-      response.__send__(meth, *args, &block)
+      response.__send__(meth, *args, **kwargs, &block)
     end
   end
 

data/lib/httpx/request/body.rb

@@ -52,7 +52,11 @@ module HTTPX
 
       body = stream(@body)
       if body.respond_to?(:read)
-        ::IO.copy_stream(body, ProcIO.new(block))
+        while (chunk = body.read(16_384))
+          block.call(chunk)
+        end
+        # TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
+        # ::IO.copy_stream(body, ProcIO.new(block))
       elsif body.respond_to?(:each)
         body.each(&block)
       else
@@ -60,6 +64,10 @@ module HTTPX
       end
     end
 
+    def close
+      @body.close if @body.respond_to?(:close)
+    end
+
     # if the +@body+ is rewindable, it rewinnds it.
     def rewind
       return if empty?
@@ -142,17 +150,4 @@ module HTTPX
       end
     end
   end
-
-  # Wrapper yielder which can be used with functions which expect an IO writer.
-  class ProcIO
-    def initialize(block)
-      @block = block
-    end
-
-    # Implementation the IO write protocol, which yield the given chunk to +@block+.
-    def write(data)
-      @block.call(data.dup)
-      data.bytesize
-    end
-  end
 end