httpx 1.4.0 → 1.4.2

data/lib/httpx/connection.rb

@@ -41,15 +41,17 @@ module HTTPX
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session
+    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
 
-    attr_writer :current_selector, :coalesced_connection
+    attr_writer :current_selector
 
     attr_accessor :current_session, :family
 
+    protected :sibling
+
     def initialize(uri, options)
-      @current_session = @current_selector = @coalesced_connection = nil
-      @exhausted = @cloned = false
+      @current_session = @current_selector = @sibling = @coalesced_connection = nil
+      @exhausted = @cloned = @main_sibling = false
 
       @options = Options.new(options)
       @type = initialize_type(uri, @options)
@@ -70,9 +72,6 @@ module HTTPX
       else
         transition(:idle)
       end
-      on(:activate) do
-        @current_session.select_connection(self, @current_selector)
-      end
       on(:close) do
         next if @exhausted # it'll reset
 
@@ -86,10 +85,13 @@ module HTTPX
       on(:terminate) do
         next if @exhausted # it'll reset
 
+        current_session = @current_session
+        current_selector = @current_selector
+
         # may be called after ":close" above, so after the connection has been checked back in.
-        next unless @current_session
+        next unless current_session && current_selector
 
-        @current_session.deselect_connection(self, @current_selector)
+        current_session.deselect_connection(self, current_selector)
       end
 
       on(:altsvc) do |alt_origin, origin, alt_params|
@@ -99,8 +101,6 @@ module HTTPX
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
 
-      @intervals = []
-
       self.addresses = @options.addresses if @options.addresses
     end
 
@@ -194,6 +194,12 @@ module HTTPX
       end
     end
 
+    def io_connected?
+      return @coalesced_connection.io_connected? if @coalesced_connection
+
+      @io && @io.state == :connected
+    end
+
     def connecting?
       @state == :idle
     end
@@ -329,31 +335,54 @@ module HTTPX
     end
 
     def handle_socket_timeout(interval)
-      @intervals.delete_if(&:elapsed?)
+      error = OperationTimeoutError.new(interval, "timed out while waiting on select")
+      error.set_backtrace(caller)
+      on_error(error)
+    end
 
-      unless @intervals.empty?
-        # remove the intervals which will elapse
+    def coalesced_connection=(connection)
+      @coalesced_connection = connection
 
-        return
-      end
+      close_sibling
+      connection.merge(self)
+    end
 
-      error = HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
-      error.set_backtrace(caller)
-      on_error(error)
+    def sibling=(connection)
+      @sibling = connection
+
+      return unless connection
+
+      @main_sibling = connection.sibling.nil?
+
+      return unless @main_sibling
+
+      connection.sibling = self
     end
 
-    private
+    def handle_connect_error(error)
+      @connect_error = error
 
-    def connect
-      transition(:open)
+      return handle_error(error) unless @sibling && @sibling.connecting?
+
+      @sibling.merge(self)
+
+      force_reset(true)
     end
 
     def disconnect
+      return unless @current_session && @current_selector
+
       emit(:close)
       @current_session = nil
       @current_selector = nil
     end
 
+    private
+
+    def connect
+      transition(:open)
+    end
+
     def consume
       return unless @io
 
@@ -448,6 +477,8 @@ module HTTPX
             end
             log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
             unless siz
+              @write_buffer.clear
+
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
               on_error(ex)
@@ -531,20 +562,22 @@ module HTTPX
         @exhausted = true
         current_session = @current_session
         current_selector = @current_selector
-        parser.close
-        @pending.concat(parser.pending)
+        begin
+          parser.close
+          @pending.concat(parser.pending)
+        ensure
+          @current_session = current_session
+          @current_selector = current_selector
+        end
+
         case @state
         when :closed
           idling
           @exhausted = false
-          @current_session = current_session
-          @current_selector = current_selector
         when :closing
-          once(:close) do
+          once(:closed) do
             idling
             @exhausted = false
-            @current_session = current_session
-            @current_selector = current_selector
           end
         end
       end
@@ -589,11 +622,15 @@ module HTTPX
           other_connection.merge(self)
           request.transition(:idle)
           other_connection.send(request)
-        else
-          response = ErrorResponse.new(request, ex)
-          request.response = response
-          request.emit(:response, response)
+          next
+        when OperationTimeoutError
+          # request level timeouts should take precedence
+          next unless request.active_timeouts.empty?
         end
+
+        response = ErrorResponse.new(request, ex)
+        request.response = response
+        request.emit(:response, response)
       end
     end
 
@@ -613,14 +650,16 @@ module HTTPX
       # connect errors, exit gracefully
       error = ConnectionError.new(e.message)
       error.set_backtrace(e.backtrace)
-      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, error) : handle_error(error)
+      handle_connect_error(error) if connecting?
       @state = :closed
+      purge_after_closed
       disconnect
     rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
-      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, e) : handle_error(e)
+      handle_connect_error(e) if connecting?
       @state = :closed
+      purge_after_closed
       disconnect
     end
 
@@ -634,7 +673,7 @@ module HTTPX
         return if @state == :closed
 
         @io.connect
-        emit(:tcp_open, self) if @io.state == :connected
+        close_sibling if @io.state == :connected
 
         return unless @io.connected?
 
@@ -667,6 +706,7 @@ module HTTPX
 
         purge_after_closed
         disconnect if @pending.empty?
+
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -677,11 +717,29 @@ module HTTPX
         return unless @state == :inactive
 
         nextstate = :open
-        emit(:activate)
+
+        # activate
+        @current_session.select_connection(self, @current_selector)
       end
       @state = nextstate
     end
 
+    def close_sibling
+      return unless @sibling
+
+      if @sibling.io_connected?
+        reset
+        # TODO: transition connection to closed
+      end
+
+      unless @sibling.state == :closed
+        merge(@sibling) unless @main_sibling
+        @sibling.force_reset(true)
+      end
+
+      @sibling = nil
+    end
+
     def purge_after_closed
       @io.close if @io
       @read_buffer.clear
@@ -754,7 +812,7 @@ module HTTPX
     end
 
     def on_error(error, request = nil)
-      if error.instance_of?(TimeoutError)
+      if error.is_a?(OperationTimeoutError)
 
         # inactive connections do not contribute to the select loop, therefore
         # they should not fail due to such errors.
@@ -799,7 +857,7 @@ module HTTPX
 
       return if read_timeout.nil? || read_timeout.infinite?
 
-      set_request_timeout(request, read_timeout, :done, :response) do
+      set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
         read_timeout_callback(request, read_timeout)
       end
     end
@@ -809,7 +867,7 @@ module HTTPX
 
       return if write_timeout.nil? || write_timeout.infinite?
 
-      set_request_timeout(request, write_timeout, :headers, %i[done response]) do
+      set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
         write_timeout_callback(request, write_timeout)
       end
     end
@@ -819,7 +877,7 @@ module HTTPX
 
       return if request_timeout.nil? || request_timeout.infinite?
 
-      set_request_timeout(request, request_timeout, :headers, :complete) do
+      set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
         read_timeout_callback(request, request_timeout, RequestTimeoutError)
       end
     end
@@ -844,21 +902,18 @@ module HTTPX
       on_error(error, request)
     end
 
-    def set_request_timeout(request, timeout, start_event, finish_events, &callback)
-      request.once(start_event) do
-        interval = @current_selector.after(timeout, callback)
+    def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
+      request.set_timeout_callback(start_event) do
+        timer = @current_selector.after(timeout, callback)
+        request.active_timeouts << label
 
         Array(finish_events).each do |event|
           # clean up request timeouts if the connection errors out
-          request.once(event) do
-            if @intervals.include?(interval)
-              interval.delete(callback)
-              @intervals.delete(interval) if interval.no_callbacks?
-            end
+          request.set_timeout_callback(event) do
+            timer.cancel
+            request.active_timeouts.delete(label)
           end
         end
-
-        @intervals << interval
       end
     end
 

data/lib/httpx/errors.rb

@@ -77,6 +77,9 @@ module HTTPX
   # Error raised when there was a timeout while resolving a domain to an IP.
   class ResolveTimeoutError < TimeoutError; end
 
+  # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
+  class OperationTimeoutError < TimeoutError; end
+
   # Error raised when there was an error while resolving a domain to an IP.
   class ResolveError < Error; end
 

data/lib/httpx/loggable.rb

@@ -22,7 +22,14 @@ module HTTPX
 
       return unless debug_stream
 
-      message = (+"" << msg.call << "\n")
+      klass = self.class
+
+      until (class_name = klass.name)
+        klass = klass.superclass
+      end
+
+      message = +"(pid:#{Process.pid} tid:#{Thread.current.object_id}, self:#{class_name}##{object_id}) "
+      message << msg.call << "\n"
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
       debug_stream << message
     end

data/lib/httpx/plugins/callbacks.rb

@@ -25,6 +25,7 @@ module HTTPX
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
             def on_#{meth}(&blk)   # def on_connection_opened(&blk)
               on(:#{meth}, &blk)   #   on(:connection_opened, &blk)
+              self                 #   self
             end                    # end
           MOD
         end

data/lib/httpx/plugins/circuit_breaker.rb

@@ -36,6 +36,7 @@ module HTTPX
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
         def on_#{meth}(&blk)   # def on_circuit_open(&blk)
           on(:#{meth}, &blk)   #   on(:circuit_open, &blk)
+          self                 #   self
         end                    # end
           MOD
         end

data/lib/httpx/plugins/expect.rb

@@ -84,7 +84,7 @@ module HTTPX
 
           return if expect_timeout.nil? || expect_timeout.infinite?
 
-          set_request_timeout(request, expect_timeout, :expect, %i[body response]) do
+          set_request_timeout(:expect_timeout, request, expect_timeout, :expect, %i[body response]) do
             # expect timeout expired
             if request.state == :expect && !request.expects?
               Expect.no_expect_store << request.origin

data/lib/httpx/plugins/grpc/grpc_encoding.rb

@@ -29,6 +29,8 @@ module HTTPX
 
           buf = outbuf if outbuf
 
+          buf = buf.b if buf.frozen?
+
           buf.prepend([compressed_flag, buf.bytesize].pack("CL>"))
           buf
         end

data/lib/httpx/request/body.rb

@@ -52,7 +52,11 @@ module HTTPX
 
       body = stream(@body)
       if body.respond_to?(:read)
-        ::IO.copy_stream(body, ProcIO.new(block))
+        while (chunk = body.read(16_384))
+          block.call(chunk)
+        end
+        # TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
+        # ::IO.copy_stream(body, ProcIO.new(block))
       elsif body.respond_to?(:each)
         body.each(&block)
       else
@@ -60,6 +64,10 @@ module HTTPX
       end
     end
 
+    def close
+      @body.close if @body.respond_to?(:close)
+    end
+
     # if the +@body+ is rewindable, it rewinnds it.
     def rewind
       return if empty?
@@ -142,17 +150,4 @@ module HTTPX
       end
     end
   end
-
-  # Wrapper yielder which can be used with functions which expect an IO writer.
-  class ProcIO
-    def initialize(block)
-      @block = block
-    end
-
-    # Implementation the IO write protocol, which yield the given chunk to +@block+.
-    def write(data)
-      @block.call(data.dup)
-      data.bytesize
-    end
-  end
 end

data/lib/httpx/request.rb

@@ -11,6 +11,8 @@ module HTTPX
     include Callbacks
     using URIExtensions
 
+    ALLOWED_URI_SCHEMES = %w[https http].freeze
+
     # default value used for "user-agent" header, when not overridden.
     USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
 
@@ -43,6 +45,8 @@ module HTTPX
 
     attr_writer :persistent
 
+    attr_reader :active_timeouts
+
     # will be +true+ when request body has been completely flushed.
     def_delegator :@body, :empty?
 
@@ -92,10 +96,13 @@ module HTTPX
         @uri = origin.merge("#{base_path}#{@uri}")
       end
 
+      raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
+
       @state = :idle
       @response = nil
       @peer_address = nil
       @persistent = @options.persistent
+      @active_timeouts = []
     end
 
     # the read timeout defined for this requet.
@@ -241,8 +248,10 @@ module HTTPX
         @body.rewind
         @response = nil
         @drainer = nil
+        @active_timeouts.clear
       when :headers
         return unless @state == :idle
+
       when :body
         return unless @state == :headers ||
                       @state == :expect
@@ -263,6 +272,8 @@ module HTTPX
         return unless @state == :body
       when :done
         return if @state == :expect
+
+        @body.close
       end
       @state = nextstate
       emit(@state, self)
@@ -273,6 +284,15 @@ module HTTPX
     def expects?
       @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end
+
+    def set_timeout_callback(event, &callback)
+      clb = once(event, &callback)
+
+      # reset timeout callbacks when requests get rerouted to a different connection
+      once(:idle) do
+        callbacks(event).delete(clb)
+      end
+    end
   end
 end
 

data/lib/httpx/resolver/https.rb

@@ -82,7 +82,9 @@ module HTTPX
 
       if hostname.nil?
         hostname = connection.peer.host
-        log { "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" } if connection.peer.non_ascii_hostname
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+        end if connection.peer.non_ascii_hostname
 
         hostname = @resolver.generate_candidates(hostname).each do |name|
           @queries[name.to_s] = connection
@@ -90,7 +92,7 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
-      log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
+      log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
 
       begin
         request = build_request(hostname)