httpx 0.6.7 → 0.9.0

data/doc/release_notes/0_9_0.md

@@ -0,0 +1,38 @@
+# 0.9.0
+
+## Features
+
+### Multiple requests with specific options
+
+You can now pass a third element to the "request element" of an array to `.request`.
+
+```ruby
+requests = [
+  [:post, "https://url/post", { form: { foo: "bar" } }],
+  [:post, "https://url/post", { form: { foo: "bar2" } }]
+]
+HTTPX.request(requests)
+# or, if you want to pass options common to all requests
+HTTPX.request(requests, max_concurrent_requests: 1)
+```
+
+
+### HTTPX::Session#build_request
+
+`HTTPX::Session::build_request` is now public API from a session. You can now build requests before you send them. These request objects are still considered somewhat "internal", so consider them immutable and **do not rely on its API**. Just pass them forward.
+
+Note: this API is only available for instantiated session, so there is no `HTTPX.build_request`.
+
+
+```ruby
+
+HTTPX.wrap do |http|
+  requests = [
+    http.build_request(:post, "https://url/post", { form: { foo: "bar" } }),
+    http.build_request(:post, "https://url/post", { form: { foo: "bar2" } })
+  ]
+  http.request(requests)
+  # or, if you want to pass options common to all requests
+  http.request(requests, max_concurrent_requests: 1)
+end
+```

data/lib/httpx/adapters/faraday.rb

@@ -153,7 +153,7 @@ module Faraday
           proxy_options = { uri: env.request.proxy }
 
           session = @session.with(options_from_env(env))
-          session = session.plugin(:proxy).with_proxy(proxy_options) if env.request.proxy
+          session = session.plugin(:proxy).with(proxy: proxy_options) if env.request.proxy
 
           responses = session.request(requests)
           Array(responses).each_with_index do |response, index|
@@ -192,7 +192,7 @@ module Faraday
         meth, uri, request_options = build_request(env)
 
         session = @session.with(options_from_env(env))
-        session = session.plugin(:proxy).with_proxy(proxy_options) if env.request.proxy
+        session = session.plugin(:proxy).with(proxy: proxy_options) if env.request.proxy
         response = session.__send__(meth, uri, **request_options)
         response.raise_for_status unless response.is_a?(::HTTPX::Response)
         save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|

data/lib/httpx/altsvc.rb

@@ -42,7 +42,23 @@ module HTTPX
 
       origin = request.origin
       host = request.uri.host
-      parse(response.headers["alt-svc"]) do |alt_origin, alt_params|
+
+      altsvc = response.headers["alt-svc"]
+
+      # https://tools.ietf.org/html/rfc7838#section-3
+      # A field value containing the special value "clear" indicates that the
+      # origin requests all alternatives for that origin to be invalidated
+      # (including those specified in the same response, in case of an
+      # invalid reply containing both "clear" and alternative services).
+      if altsvc == "clear"
+        @altsvc_mutex.synchronize do
+          @altsvcs[origin].clear
+        end
+
+        return
+      end
+
+      parse(altsvc) do |alt_origin, alt_params|
         alt_origin.host ||= host
         yield(alt_origin, origin, alt_params)
       end
@@ -73,7 +89,7 @@ module HTTPX
         alt_proto, alt_origin = alt_origin.split("=")
         alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
         if alt_origin.start_with?(":")
-          alt_origin = "dummy#{alt_origin}"
+          alt_origin = "#{alt_proto}://dummy#{alt_origin}"
           uri = URI.parse(alt_origin)
           uri.host = nil
           uri

data/lib/httpx/chainable.rb

@@ -3,25 +3,31 @@
 module HTTPX
   module Chainable
     %i[head get post put delete trace options connect patch].each do |meth|
-      define_method meth do |*uri, **options|
-        request(meth, uri, **options)
-      end
+      class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+        def #{meth}(*uri, **options)
+          request(:#{meth}, uri, **options)
+        end
+      MOD
     end
 
-    def request(verb, uri, **options)
-      branch(default_options).request(verb, uri, **options)
+    def request(*args, **options)
+      branch(default_options).request(*args, **options)
     end
 
+    # :nocov:
     def timeout(**args)
-      branch(default_options.with_timeout(args))
+      warn ":#{__method__} is deprecated, use :with_timeout instead"
+      branch(default_options.with(timeout: args))
     end
 
     def headers(headers)
-      branch(default_options.with_headers(headers))
+      warn ":#{__method__} is deprecated, use :with_headers instead"
+      branch(default_options.with(headers: headers))
     end
+    # :nocov:
 
     def accept(type)
-      headers("accept" => String(type))
+      with(headers: { "accept" => String(type) })
     end
 
     def wrap(&blk)
@@ -53,11 +59,23 @@ module HTTPX
       @options || Options.new
     end
 
-    # :nodoc:
     def branch(options, &blk)
       return self.class.new(options, &blk) if is_a?(Session)
 
       Session.new(options, &blk)
     end
+
+    def method_missing(meth, *args, **options)
+      if meth =~ /\Awith_(.+)/
+        option = Regexp.last_match(1).to_sym
+        with(option => (args.first || options))
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(meth, *args)
+      default_options.respond_to?(meth, *args) || super
+    end
   end
 end

data/lib/httpx/connection.rb

@@ -46,6 +46,8 @@ module HTTPX
 
     attr_reader :origin, :state, :pending, :options
 
+    attr_writer :timers
+
     def initialize(type, uri, options)
       @type = type
       @origins = [uri.origin]
@@ -65,6 +67,10 @@ module HTTPX
       else
         transition(:idle)
       end
+
+      @inflight = 0
+      @keep_alive_timeout = options.timeout.keep_alive_timeout
+      @keep_alive_timer = nil
     end
 
     # this is a semi-private method, to be used by the resolver
@@ -80,6 +86,8 @@ module HTTPX
     def match?(uri, options)
       return false if @state == :closing || @state == :closed
 
+      return false if exhausted?
+
       (
         (
           @origins.include?(uri.origin) &&
@@ -95,6 +103,8 @@ module HTTPX
     def mergeable?(connection)
       return false if @state == :closing || @state == :closed || !@io
 
+      return false if exhausted?
+
       !(@io.addresses & connection.addresses).empty? && @options == connection.options
     end
 
@@ -110,10 +120,13 @@ module HTTPX
       end
     end
 
+    def create_idle
+      self.class.new(@type, @origin, @options)
+    end
+
     def merge(connection)
       @origins += connection.instance_variable_get(:@origins)
-      pending = connection.instance_variable_get(:@pending)
-      pending.each do |req|
+      connection.purge_pending do |req|
         send(req)
       end
     end
@@ -130,7 +143,13 @@ module HTTPX
     end
 
     def purge_pending
-      [*@parser.pending, *@pending].each do |pending|
+      pendings = []
+      if @parser
+        @inflight -= @parser.pending.size
+        pendings << @parser.pending
+      end
+      pendings << @pending
+      pendings.each do |pending|
         pending.reject! do |request|
           yield request
         end
@@ -156,22 +175,45 @@ module HTTPX
     end
 
     def interests
-      return :w if @state == :idle
+      # connecting
+      if connecting?
+        connect
+
+        return @io.interests if connecting?
+      end
+
+      # if the write buffer is full, we drain it
+      return :w if @write_buffer.full?
+
+      return @parser.interests if @parser
 
-      :rw
+      nil
     end
 
     def to_io
+      @io.to_io
+    end
+
+    def call
       case @state
-      when :idle
-        transition(:open)
+      when :closed
+        return
+      when :closing
+        consume
+        transition(:closed)
+        emit(:close)
+      when :open
+        consume
       end
-      @io.to_io
+      nil
     end
 
     def close
       @parser.close if @parser
-      transition(:closing)
+      return unless @keep_alive_timer
+
+      @keep_alive_timer.cancel
+      remove_instance_variable(:@keep_alive_timer)
     end
 
     def reset
@@ -183,26 +225,25 @@ module HTTPX
     def send(request)
       if @parser && !@write_buffer.full?
         request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
+        if @keep_alive_timer
+          # when pushing a request into an existing connection, we have to check whether there
+          # is the possibility that the connection might have extended the keep alive timeout.
+          # for such cases, we want to ping for availability before deciding to shovel requests.
+          if @keep_alive_timer.fires_in.negative?
+            @pending << request
+            parser.ping
+            return
+          end
+
+          @keep_alive_timer.pause
+        end
+        @inflight += 1
         parser.send(request)
       else
         @pending << request
       end
     end
 
-    def call
-      case @state
-      when :closed
-        return
-      when :closing
-        dwrite
-        transition(:closed)
-        emit(:close)
-      when :open
-        consume
-      end
-      nil
-    end
-
     def timeout
       return @timeout if defined?(@timeout)
 
@@ -213,50 +254,94 @@ module HTTPX
 
     private
 
-    def consume
-      catch(:called) do
-        dread
-        dwrite
-        parser.consume
-      end
+    def connect
+      transition(:open)
     end
 
-    def dread(wsize = @window_size)
-      loop do
-        siz = @io.read(wsize, @read_buffer)
-        unless siz
-          ex = EOFError.new("descriptor closed")
-          ex.set_backtrace(caller)
-          on_error(ex)
-          return
-        end
-        return if siz.zero?
-
-        log { "READ: #{siz} bytes..." }
-        parser << @read_buffer.to_s
-        return if @state == :closing || @state == :closed
-      end
+    def exhausted?
+      @parser && parser.exhausted?
     end
 
-    def dwrite
-      loop do
-        return if @write_buffer.empty?
-
-        siz = @io.write(@write_buffer)
-        unless siz
-          ex = EOFError.new("descriptor closed")
-          ex.set_backtrace(caller)
-          on_error(ex)
-          return
+    def consume
+      catch(:called) do
+        loop do
+          parser.consume
+
+          # we exit if there's no more data to process
+          if @pending.size.zero? && @inflight.zero?
+            log(level: 3) { "NO MORE REQUESTS..." }
+            return
+          end
+
+          @timeout = @current_timeout
+
+          read_drained = false
+          write_drained = nil
+
+          # dread
+          loop do
+            siz = @io.read(@window_size, @read_buffer)
+            unless siz
+              ex = EOFError.new("descriptor closed")
+              ex.set_backtrace(caller)
+              on_error(ex)
+              return
+            end
+
+            if siz.zero?
+              read_drained = @read_buffer.empty?
+              break
+            end
+
+            log { "READ: #{siz} bytes..." }
+
+            parser << @read_buffer.to_s
+
+            break if @state == :closing || @state == :closed
+
+            # for HTTP/2, we just want to write goaway frame
+          end unless @state == :closing
+
+          # dwrite
+          loop do
+            if @write_buffer.empty?
+              # we only mark as drained on the first loop
+              write_drained = write_drained.nil? && @inflight.positive?
+              break
+            end
+
+            siz = @io.write(@write_buffer)
+            unless siz
+              ex = EOFError.new("descriptor closed")
+              ex.set_backtrace(caller)
+              on_error(ex)
+              return
+            end
+            log { "WRITE: #{siz} bytes..." }
+
+            if siz.zero?
+              write_drained = !@write_buffer.empty?
+              break
+            end
+
+            break if @state == :closing || @state == :closed
+
+            write_drained = false
+          end
+
+          # return if socket is drained
+          if read_drained && write_drained
+            log(level: 3) { "WAITING FOR EVENTS..." }
+            return
+          end
         end
-        log { "WRITE: #{siz} bytes..." }
-        return if siz.zero?
-        return if @state == :closing || @state == :closed
       end
     end
 
     def send_pending
       while !@write_buffer.full? && (request = @pending.shift)
+        @inflight += 1
+        @keep_alive_timer.pause if @keep_alive_timer
         parser.send(request)
       end
     end
@@ -276,26 +361,39 @@ module HTTPX
         AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
           emit(:altsvc, alt_origin, origin, alt_params)
         end
+        handle_response
         request.emit(:response, response)
       end
       parser.on(:altsvc) do |alt_origin, origin, alt_params|
         emit(:altsvc, alt_origin, origin, alt_params)
       end
 
+      parser.on(:pong, &method(:send_pending))
+
       parser.on(:promise) do |request, stream|
         request.emit(:promise, parser, stream)
       end
+      parser.on(:exhausted) do
+        emit(:exhausted)
+      end
       parser.on(:origin) do |origin|
         @origins << origin
       end
-      parser.on(:close) do
+      parser.on(:close) do |force|
         transition(:closing)
+        if force
+          transition(:closed)
+          emit(:close)
+        end
       end
       parser.on(:reset) do
-        transition(:closing)
-        unless parser.empty?
+        if parser.empty?
+          reset
+        else
+          transition(:closing)
           transition(:closed)
           emit(:reset)
+          @parser.reset if @parser
           transition(:idle)
           transition(:open)
         end
@@ -316,13 +414,20 @@ module HTTPX
 
     def transition(nextstate)
       case nextstate
+      when :idle
+        @timeout = @current_timeout = @options.timeout.connect_timeout
+
       when :open
         return if @state == :closed
 
+        total_timeout
+
         @io.connect
         return unless @io.connected?
 
         send_pending
+
+        @timeout = @current_timeout = @options.timeout.operation_timeout
         emit(:open)
       when :closing
         return unless @state == :open
@@ -330,8 +435,18 @@ module HTTPX
         return unless @state == :closing
         return unless @write_buffer.empty?
 
+        if @total_timeout
+          @total_timeout.cancel
+          remove_instance_variable(:@total_timeout)
+        end
+
         @io.close
         @read_buffer.clear
+        if @keep_alive_timer
+          @keep_alive_timer.cancel
+          remove_instance_variable(:@keep_alive_timer)
+        end
+
         remove_instance_variable(:@timeout) if defined?(@timeout)
       when :already_open
         nextstate = :open
@@ -353,25 +468,60 @@ module HTTPX
       emit(:close)
     end
 
-    def on_error(ex)
-      handle_error(ex)
-      reset
+    def handle_response
+      @inflight -= 1
+      return unless @inflight.zero?
+
+      if @keep_alive_timer
+        @keep_alive_timer.resume
+        @keep_alive_timer.reset
+      else
+        @keep_alive_timer = @timers.after(@keep_alive_timeout) do
+          unless @inflight.zero?
+            log { "(#{@origin}): keep alive timeout expired" }
+            parser.ping
+          end
+        end
+      end
     end
 
-    def handle_error(error)
+    def on_error(error)
       if error.instance_of?(TimeoutError)
         if @timeout
           @timeout -= error.timeout
           return unless @timeout <= 0
         end
 
-        error = error.to_connection_error if connecting?
+        if @total_timeout && @total_timeout.fires_in.negative?
+          ex = TotalTimeoutError.new(@total_timeout.interval, "Timed out after #{@total_timeout.interval} seconds")
+          ex.set_backtrace(error.backtrace)
+          error = ex
+        elsif connecting?
+          error = error.to_connection_error
+        end
       end
+      handle_error(error)
+      reset
+    end
 
+    def handle_error(error)
       parser.handle_error(error) if @parser && parser.respond_to?(:handle_error)
       while (request = @pending.shift)
         request.emit(:response, ErrorResponse.new(request, error, @options))
       end
     end
+
+    def total_timeout
+      total = @options.timeout.total_timeout
+
+      return unless total
+
+      @total_timeout ||= @timers.after(total) do
+        ex = TotalTimeoutError.new(total, "Timed out after #{total} seconds")
+        ex.set_backtrace(caller)
+        on_error(ex)
+        @parser.close if @parser
+      end
+    end
   end
 end