httpx 0.10.2 → 0.12.0

data/lib/httpx/adapters/webmock.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module WebMock
+  module HttpLibAdapters
+    if RUBY_VERSION < "2.5"
+      require "webrick/httpstatus"
+      HTTP_REASONS = WEBrick::HTTPStatus::StatusMessage
+    else
+      require "net/http/status"
+      HTTP_REASONS = Net::HTTP::STATUS_CODES
+    end
+
+    #
+    # HTTPX plugin for webmock.
+    #
+    # Requests are "hijacked" at the session, before they're distributed to a connection.
+    #
+    module Plugin
+      module InstanceMethods
+        private
+
+        def send_requests(*requests, options)
+          request_signatures = requests.map do |request|
+            request_signature = _build_webmock_request_signature(request)
+            WebMock::RequestRegistry.instance.requested_signatures.put(request_signature)
+            request_signature
+          end
+
+          responses = request_signatures.map do |request_signature|
+            WebMock::StubRegistry.instance.response_for_request(request_signature)
+          end
+
+          real_requests = {}
+
+          requests.each_with_index.each_with_object([request_signatures, responses]) do |(request, idx), (sig_reqs, mock_responses)|
+            if (webmock_response = mock_responses[idx])
+              mock_responses[idx] = _build_from_webmock_response(request, webmock_response)
+              WebMock::CallbackRegistry.invoke_callbacks({ lib: :httpx }, sig_reqs[idx], webmock_response)
+              log { "mocking #{request.uri} with #{mock_responses[idx].inspect}" }
+            elsif WebMock.net_connect_allowed?(sig_reqs[idx].uri)
+              log { "performing #{request.uri}" }
+              real_requests[request] = idx
+            else
+              raise WebMock::NetConnectNotAllowedError, sig_reqs[idx]
+            end
+          end
+
+          unless real_requests.empty?
+            reqs = real_requests.keys
+            reqs.zip(super(*reqs, options)).each do |req, res|
+              idx = real_requests[req]
+
+              if WebMock::CallbackRegistry.any_callbacks?
+                webmock_response = _build_webmock_response(req, res)
+                WebMock::CallbackRegistry.invoke_callbacks(
+                  { lib: :httpx, real_request: true }, request_signatures[idx],
+                  webmock_response
+                )
+              end
+
+              responses[idx] = res
+            end
+          end
+
+          responses
+        end
+
+        def _build_webmock_request_signature(request)
+          uri = WebMock::Util::URI.heuristic_parse(request.uri)
+          uri.path = uri.normalized_path.gsub("[^:]//", "/")
+
+          WebMock::RequestSignature.new(
+            request.verb,
+            uri.to_s,
+            body: request.body.each.to_a.join,
+            headers: request.headers.to_h
+          )
+        end
+
+        def _build_webmock_response(_request, response)
+          webmock_response = WebMock::Response.new
+          webmock_response.status = [response.status, HTTP_REASONS[response.status]]
+          webmock_response.body = response.body.to_s
+          webmock_response.headers = response.headers.to_h
+          webmock_response
+        end
+
+        def _build_from_webmock_response(request, webmock_response)
+          return ErrorResponse.new(request, webmock_response.exception, request.options) if webmock_response.exception
+
+          response = request.options.response_class.new(request,
+                                                        webmock_response.status[0],
+                                                        "2.0",
+                                                        webmock_response.headers)
+          response << webmock_response.body.dup
+          response
+        end
+      end
+    end
+
+    class HttpxAdapter < HttpLibAdapter
+      adapter_for :httpx
+
+      class << self
+        def enable!
+          @original_session = ::HTTPX::Session
+
+          webmock_session = ::HTTPX.plugin(Plugin)
+
+          ::HTTPX.send(:remove_const, :Session)
+          ::HTTPX.send(:const_set, :Session, webmock_session.class)
+        end
+
+        def disable!
+          return unless @original_session
+
+          HTTPX.send(:remove_const, :Session)
+          HTTPX.send(:const_set, :Session, @original_session)
+        end
+      end
+    end
+  end
+end

data/lib/httpx/altsvc.rb

@@ -37,6 +37,7 @@ module HTTPX
     end
 
     def emit(request, response)
+      return unless response.respond_to?(:headers)
       # Alt-Svc
       return unless response.headers.key?("alt-svc")
 

data/lib/httpx/chainable.rb

@@ -59,7 +59,7 @@ module HTTPX
     private
 
     def default_options
-      @options || Options.new
+      @options || Session.default_options
     end
 
     def branch(options, &blk)

data/lib/httpx/connection.rb

@@ -170,7 +170,7 @@ module HTTPX
       end
 
       # if the write buffer is full, we drain it
-      return :w if @write_buffer.full?
+      return :w unless @write_buffer.empty?
 
       return @parser.interests if @parser
 
@@ -251,11 +251,18 @@ module HTTPX
 
     def consume
       catch(:called) do
+        epiped = false
         loop do
           parser.consume
 
-          # we exit if there's no more data to process
-          if @pending.size.zero? && @inflight.zero?
+          # we exit if there's no more requests to process
+          #
+          # this condition takes into account:
+          #
+          # * the number of inflight requests
+          # * the number of pending requests
+          # * whether the write buffer has bytes (i.e. for close handshake)
+          if @pending.size.zero? && @inflight.zero? && @write_buffer.empty?
             log(level: 3) { "NO MORE REQUESTS..." }
             return
           end
@@ -265,9 +272,17 @@ module HTTPX
           read_drained = false
           write_drained = nil
 
-          # dread
+          #
+          # tight read loop.
+          #
+          # read as much of the socket as possible.
+          #
+          # this tight loop reads all the data it can from the socket and pipes it to
+          # its parser.
+          #
           loop do
             siz = @io.read(@window_size, @read_buffer)
+            log(level: 3, color: :cyan) { "IO READ: #{siz} bytes..." }
             unless siz
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
@@ -275,27 +290,53 @@ module HTTPX
               return
             end
 
+            # socket has been drained. mark and exit the read loop.
             if siz.zero?
               read_drained = @read_buffer.empty?
+              epiped = false
               break
             end
 
             parser << @read_buffer.to_s
 
+            # continue reading if possible.
+            break if interests == :w && !epiped
+
+            # exit the read loop if connection is preparing to be closed
             break if @state == :closing || @state == :closed
 
-            # for HTTP/2, we just want to write goaway frame
-          end unless @state == :closing
+            # exit #consume altogether if all outstanding requests have been dealt with
+            return if @pending.size.zero? && @inflight.zero?
+          end unless (interests == :w || @state == :closing) && !epiped
 
-          # dwrite
+          #
+          # tight write loop.
+          #
+          # flush as many bytes as the sockets allow.
+          #
           loop do
+            # buffer has been drainned, mark and exit the write loop.
             if @write_buffer.empty?
               # we only mark as drained on the first loop
               write_drained = write_drained.nil? && @inflight.positive?
+
               break
             end
 
-            siz = @io.write(@write_buffer)
+            begin
+              siz = @io.write(@write_buffer)
+            rescue Errno::EPIPE
+              # this can happen if we still have bytes in the buffer to send to the server, but
+              # the server wants to respond immediately with some message, or an error. An example is
+              # when one's uploading a big file to an unintended endpoint, and the server stops the
+              # consumption, and responds immediately with an authorization of even method not allowed error.
+              # at this point, we have to let the connection switch to read-mode.
+              log(level: 2) { "pipe broken, could not flush buffer..." }
+              epiped = true
+              read_drained = false
+              break
+            end
+            log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
             unless siz
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
@@ -303,21 +344,28 @@ module HTTPX
               return
             end
 
+            # socket closed for writing. mark and exit the write loop.
             if siz.zero?
               write_drained = !@write_buffer.empty?
               break
             end
 
-            break if @state == :closing || @state == :closed
+            # exit write loop if marked to consume from peer, or is closing.
+            break if interests == :r || @state == :closing || @state == :closed
 
             write_drained = false
-          end
+          end unless interests == :r
 
           # return if socket is drained
-          if read_drained && write_drained
-            log(level: 3) { "WAITING FOR EVENTS..." }
-            return
-          end
+          next unless (interests != :r || read_drained) &&
+                      (interests != :w || write_drained)
+
+          # gotta go back to the event loop. It happens when:
+          #
+          # * the socket is drained of bytes or it's not the interest of the conn to read;
+          # * theres nothing more to write, or it's not in the interest of the conn to write;
+          log(level: 3) { "(#{interests}): WAITING FOR EVENTS..." }
+          return
         end
       end
     end
@@ -444,7 +492,7 @@ module HTTPX
       throw(:jump_tick)
     rescue Errno::ECONNREFUSED,
            Errno::EADDRNOTAVAIL,
-           OpenSSL::SSL::SSLError => e
+           TLSError => e
       # connect errors, exit gracefully
       handle_error(e)
       @state = :closed

data/lib/httpx/connection/http1.rb

@@ -69,14 +69,16 @@ module HTTPX
 
       return if @requests.include?(request)
 
+      request.once(:headers, &method(:set_protocol_headers))
       @requests << request
       @pipelining = true if @requests.size > 1
     end
 
     def consume
-      requests_limit = [@max_concurrent_requests, @max_requests, @requests.size].min
+      requests_limit = [@max_requests, @requests.size].min
+      concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
       @requests.each_with_index do |request, idx|
-        break if idx >= requests_limit
+        break if idx >= concurrent_requests_limit
         next if request.state == :done
 
         request.headers["connection"] ||= request.options.persistent || idx < requests_limit - 1 ? "keep-alive" : "close"
@@ -115,7 +117,7 @@ module HTTPX
       response = @request.response
       log(level: 2) { "trailer headers received" }
 
-      log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
+      log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v.join(", ")}" }.join("\n") }
       response.merge_headers(h)
     end
 
@@ -161,6 +163,16 @@ module HTTPX
     end
 
     def handle_error(ex)
+      if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request && @request.response &&
+         !@request.response.headers.key?("content-length") &&
+         !@request.response.headers.key?("transfer-encoding")
+        # if the response does not contain a content-length header, the server closing the
+        # connnection is the indicator of response consumed.
+        # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
+        catch(:called) { on_complete }
+        return
+      end
+
       if @pipelining
         disable
       else
@@ -240,7 +252,7 @@ module HTTPX
       @pipelining = false
     end
 
-    def set_request_headers(request)
+    def set_protocol_headers(request)
       request.headers["host"] ||= request.authority
       request.headers["connection"] ||= request.options.persistent ? "keep-alive" : "close"
       if !request.headers.key?("content-length") &&
@@ -254,7 +266,6 @@ module HTTPX
     end
 
     def handle(request)
-      set_request_headers(request)
       catch(:buffer_full) do
         request.transition(:headers)
         join_headers(request) if request.state == :headers

data/lib/httpx/connection/http2.rb

@@ -42,11 +42,15 @@ module HTTPX
         return @buffer.empty? ? :r : :rw
       end
 
-      return :w unless @pending.empty?
+      return :w if !@pending.empty? && can_buffer_more_requests?
 
       return :w if @streams.each_key.any? { |r| r.interests == :w }
 
-      return :r if @buffer.empty?
+      if @buffer.empty?
+        return if @streams.empty? && @pings.empty?
+
+        return :r
+      end
 
       :rw
     end
@@ -70,10 +74,14 @@ module HTTPX
       @connection << data
     end
 
+    def can_buffer_more_requests?
+      @handshake_completed &&
+        @streams.size < @max_concurrent_requests &&
+        @streams.size < @max_requests
+    end
+
     def send(request)
-      if !@handshake_completed ||
-         @streams.size >= @max_concurrent_requests ||
-         @streams.size >= @max_requests
+      unless can_buffer_more_requests?
         @pending << request
         return
       end
@@ -83,6 +91,7 @@ module HTTPX
         @streams[request] = stream
         @max_requests -= 1
       end
+      request.once(:headers, &method(:set_protocol_headers))
       handle(request, stream)
       true
     rescue HTTP2Next::Error::StreamLimitExceeded
@@ -126,8 +135,6 @@ module HTTPX
       request.path
     end
 
-    def set_request_headers(request); end
-
     def handle(request, stream)
       catch(:buffer_full) do
         request.transition(:headers)
@@ -163,27 +170,27 @@ module HTTPX
     public :reset
 
     def handle_stream(stream, request)
-      stream.on(:close, &method(:on_stream_close).curry[stream, request])
+      stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
       stream.on(:half_close) do
         log(level: 2) { "#{stream.id}: waiting for response..." }
       end
-      stream.on(:altsvc, &method(:on_altsvc).curry[request.origin])
-      stream.on(:headers, &method(:on_stream_headers).curry[stream, request])
-      stream.on(:data, &method(:on_stream_data).curry[stream, request])
+      stream.on(:altsvc, &method(:on_altsvc).curry(2)[request.origin])
+      stream.on(:headers, &method(:on_stream_headers).curry(3)[stream, request])
+      stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
+    end
+
+    def set_protocol_headers(request)
+      request.headers[":scheme"]    = request.scheme
+      request.headers[":method"]    = request.verb.to_s.upcase
+      request.headers[":path"]      = headline_uri(request)
+      request.headers[":authority"] = request.authority
     end
 
     def join_headers(stream, request)
-      set_request_headers(request)
-      headers = {}
-      headers[":scheme"]    = request.scheme
-      headers[":method"]    = request.verb.to_s.upcase
-      headers[":path"]      = headline_uri(request)
-      headers[":authority"] = request.authority
-      headers = headers.merge(request.headers)
       log(level: 1, color: :yellow) do
-        headers.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
+        request.headers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
       end
-      stream.headers(headers, end_stream: request.empty?)
+      stream.headers(request.headers.each, end_stream: request.empty?)
     end
 
     def join_body(stream, request)
@@ -227,10 +234,15 @@ module HTTPX
     end
 
     def on_stream_close(stream, request, error)
+      log(level: 2) { "#{stream.id}: closing stream" }
+      @drains.delete(request)
+      @streams.delete(request)
+
       if error && error != :no_error
         ex = Error.new(stream.id, error)
         ex.set_backtrace(caller)
-        emit(:error, request, ex)
+        response = ErrorResponse.new(request, ex, request.options)
+        emit(:response, request, response)
       else
         response = request.response
         if response.status == 421
@@ -241,9 +253,6 @@ module HTTPX
           emit(:response, request, response)
         end
       end
-      log(level: 2) { "#{stream.id}: closing stream" }
-
-      @streams.delete(request)
       send(@pending.shift) unless @pending.empty?
       return unless @streams.empty? && exhausted?
 
@@ -328,11 +337,9 @@ module HTTPX
     end
 
     def method_missing(meth, *args, &blk)
-      if @connection.respond_to?(meth)
-        @connection.__send__(meth, *args, &blk)
-      else
-        super
-      end
+      return super unless @connection.respond_to?(meth)
+
+      @connection.__send__(meth, *args, &blk)
     end
   end
   Connection.register "h2", Connection::HTTP2