httpx 1.2.6 → 1.4.4

data/lib/httpx/session.rb

@@ -9,8 +9,6 @@ module HTTPX
     include Loggable
     include Chainable
 
-    EMPTY_HASH = {}.freeze
-
     # initializes the session with a set of +options+, which will be shared by all
     # requests sent from it.
     #
@@ -19,6 +17,9 @@ module HTTPX
       @options = self.class.default_options.merge(options)
       @responses = {}
       @persistent = @options.persistent
+      @pool = @options.pool_class.new(@options.pool_options)
+      @wrapped = false
+      @closing = false
       wrap(&blk) if blk
     end
 
@@ -28,21 +29,54 @@ module HTTPX
     #     http.get("https://wikipedia.com")
     #   end # wikipedia connection closes here
     def wrap
-      prev_persistent = @persistent
-      @persistent = true
-      pool.wrap do
-        begin
-          yield self
-        ensure
-          @persistent = prev_persistent
-          close unless @persistent
+      prev_wrapped = @wrapped
+      @wrapped = true
+      was_initialized = false
+      current_selector = get_current_selector do
+        selector = Selector.new
+
+        set_current_selector(selector)
+
+        was_initialized = true
+
+        selector
+      end
+      begin
+        yield self
+      ensure
+        unless prev_wrapped
+          if @persistent
+            deactivate(current_selector)
+          else
+            close(current_selector)
+          end
         end
+        @wrapped = prev_wrapped
+        set_current_selector(nil) if was_initialized
       end
     end
 
-    # closes all the active connections from the session
-    def close(*args)
-      pool.close(*args)
+    # closes all the active connections from the session.
+    #
+    # when called directly without specifying +selector+, all available connections
+    # will be picked up from the connection pool and closed. Connections in use
+    # by other sessions, or same session in a different thread, will not be reaped.
+    def close(selector = Selector.new)
+      # throw resolvers away from the pool
+      @pool.reset_resolvers
+
+      # preparing to throw away connections
+      while (connection = @pool.pop_connection)
+        next if connection.state == :closed
+
+        select_connection(connection, selector)
+      end
+      begin
+        @closing = true
+        selector.terminate
+      ensure
+        @closing = false
+      end
     end
 
     # performs one, or multple requests; it accepts:
@@ -65,10 +99,10 @@ module HTTPX
     #  resp1, resp2 = session.request(["POST", "https://server.org/a", form: { "foo" => "bar" }], ["GET", "https://server.org/b"])
     #  resp1, resp2 = session.request("GET", ["https://server.org/a", "https://server.org/b"], headers: { "x-api-token" => "TOKEN" })
     #
-    def request(*args, **options)
+    def request(*args, **params)
       raise ArgumentError, "must perform at least one request" if args.empty?
 
-      requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+      requests = args.first.is_a?(Request) ? args : build_requests(*args, params)
       responses = send_requests(*requests)
       return responses.first if responses.size == 1
 
@@ -81,133 +115,150 @@ module HTTPX
     #
     #   req = session.build_request("GET", "https://server.com")
     #   resp = session.request(req)
-    def build_request(verb, uri, options = EMPTY_HASH)
-      rklass = @options.request_class
-      options = @options.merge(options) unless options.is_a?(Options)
-      request = rklass.new(verb, uri, options)
+    def build_request(verb, uri, params = EMPTY_HASH, options = @options)
+      rklass = options.request_class
+      request = rklass.new(verb, uri, options, params)
       request.persistent = @persistent
       set_request_callbacks(request)
       request
     end
 
-    private
-
-    # returns the HTTPX::Pool object which manages the networking required to
-    # perform requests.
-    def pool
-      Thread.current[:httpx_connection_pool] ||= Pool.new
+    def select_connection(connection, selector)
+      pin_connection(connection, selector)
+      selector.register(connection)
     end
 
-    # callback executed when a response for a given request has been received.
-    def on_response(request, response)
-      @responses[request] = response
+    def pin_connection(connection, selector)
+      connection.current_session = self
+      connection.current_selector = selector
     end
 
-    # callback executed when an HTTP/2 promise frame has been received.
-    def on_promise(_, stream)
-      log(level: 2) { "#{stream.id}: refusing stream!" }
-      stream.refuse
+    alias_method :select_resolver, :select_connection
+
+    def deselect_connection(connection, selector, cloned = false)
+      selector.deregister(connection)
+
+      # when connections coalesce
+      return if connection.state == :idle
+
+      return if cloned
+
+      return if @closing && connection.state == :closed
+
+      @pool.checkin_connection(connection)
     end
 
-    # returns the corresponding HTTP::Response to the given +request+ if it has been received.
-    def fetch_response(request, _, _)
-      @responses.delete(request)
+    def deselect_resolver(resolver, selector)
+      selector.deregister(resolver)
+
+      return if @closing && resolver.closed?
+
+      @pool.checkin_resolver(resolver)
     end
 
-    # returns the HTTPX::Connection through which the +request+ should be sent through.
-    def find_connection(request, connections, options)
-      uri = request.uri
+    def try_clone_connection(connection, selector, family)
+      connection.family ||= family
 
-      connection = pool.find_connection(uri, options) || init_connection(uri, options)
-      unless connections.nil? || connections.include?(connection)
-        connections << connection
-        set_connection_callbacks(connection, connections, options)
-      end
-      connection
+      return connection if connection.family == family
+
+      new_connection = connection.class.new(connection.origin, connection.options)
+
+      new_connection.family = family
+
+      connection.sibling = new_connection
+
+      do_init_connection(new_connection, selector)
+      new_connection
     end
 
-    def send_request(request, connections, options = request.options)
-      error = catch(:resolve_error) do
-        connection = find_connection(request, connections, options)
-        connection.send(request)
+    # returns the HTTPX::Connection through which the +request+ should be sent through.
+    def find_connection(request_uri, selector, options)
+      if (connection = selector.find_connection(request_uri, options))
+        return connection
       end
-      return unless error.is_a?(Error)
 
-      request.emit(:response, ErrorResponse.new(request, error, options))
-    end
+      connection = @pool.checkout_connection(request_uri, options)
 
-    # sets the callbacks on the +connection+ required to process certain specific
-    # connection lifecycle events which deal with request rerouting.
-    def set_connection_callbacks(connection, connections, options, cloned: false)
-      connection.only(:misdirected) do |misdirected_request|
-        other_connection = connection.create_idle(ssl: { alpn_protocols: %w[http/1.1] })
-        other_connection.merge(connection)
-        catch(:coalesced) do
-          pool.init_connection(other_connection, options)
+      case connection.state
+      when :idle
+        do_init_connection(connection, selector)
+      when :open
+        if options.io
+          select_connection(connection, selector)
+        else
+          pin_connection(connection, selector)
         end
-        set_connection_callbacks(other_connection, connections, options)
-        connections << other_connection
-        misdirected_request.transition(:idle)
-        other_connection.send(misdirected_request)
-      end
-      connection.only(:altsvc) do |alt_origin, origin, alt_params|
-        other_connection = build_altsvc_connection(connection, connections, alt_origin, origin, alt_params, options)
-        connections << other_connection if other_connection
+      when :closing, :closed
+        connection.idling
+        select_connection(connection, selector)
+      else
+        pin_connection(connection, selector)
       end
-      connection.only(:cloned) do |cloned_conn|
-        set_connection_callbacks(cloned_conn, connections, options, cloned: true)
-        connections << cloned_conn
-      end unless cloned
-    end
-
-    # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
-    def build_altsvc_connection(existing_connection, connections, alt_origin, origin, alt_params, options)
-      # do not allow security downgrades on altsvc negotiation
-      return if existing_connection.origin.scheme == "https" && alt_origin.scheme != "https"
 
-      altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
+      connection
+    end
 
-      # altsvc already exists, somehow it wasn't advertised, probably noop
-      return unless altsvc
+    private
 
-      alt_options = options.merge(ssl: options.ssl.merge(hostname: URI(origin).host))
+    def deactivate(selector)
+      selector.each_connection do |connection|
+        connection.deactivate
+        deselect_connection(connection, selector) if connection.state == :inactive
+      end
+    end
 
-      connection = pool.find_connection(alt_origin, alt_options) || init_connection(alt_origin, alt_options)
+    # callback executed when a response for a given request has been received.
+    def on_response(request, response)
+      @responses[request] = response
+    end
 
-      # advertised altsvc is the same origin being used, ignore
-      return if connection == existing_connection
+    # callback executed when an HTTP/2 promise frame has been received.
+    def on_promise(_, stream)
+      log(level: 2) { "#{stream.id}: refusing stream!" }
+      stream.refuse
+    end
 
-      connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
+    # returns the corresponding HTTP::Response to the given +request+ if it has been received.
+    def fetch_response(request, _selector, _options)
+      @responses.delete(request)
+    end
 
-      set_connection_callbacks(connection, connections, alt_options)
+    # sends the +request+ to the corresponding HTTPX::Connection
+    def send_request(request, selector, options = request.options)
+      error = begin
+        catch(:resolve_error) do
+          connection = find_connection(request.uri, selector, options)
+          connection.send(request)
+        end
+      rescue StandardError => e
+        e
+      end
+      return unless error && error.is_a?(Exception)
 
-      log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
+      raise error unless error.is_a?(Error)
 
-      connection.merge(existing_connection)
-      existing_connection.terminate
-      connection
-    rescue UnsupportedSchemeError
-      altsvc["noop"] = true
-      nil
+      request.emit(:response, ErrorResponse.new(request, error))
     end
 
     # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
-    def build_requests(*args, options)
-      request_options = @options.merge(options)
-
+    def build_requests(*args, params)
       requests = if args.size == 1
         reqs = args.first
-        reqs.map do |verb, uri, opts = EMPTY_HASH|
-          build_request(verb, uri, request_options.merge(opts))
+        reqs.map do |verb, uri, ps = EMPTY_HASH|
+          request_params = params
+          request_params = request_params.merge(ps) unless ps.empty?
+          build_request(verb, uri, request_params)
         end
       else
         verb, uris = args
         if uris.respond_to?(:each)
-          uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
-            build_request(verb, uri, request_options.merge(opts))
+          uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
+            request_params = params
+            request_params = request_params.merge(ps) unless ps.empty?
+            build_request(verb, uri, request_params)
           end
         else
-          [build_request(verb, uris, request_options)]
+          [build_request(verb, uris, params)]
         end
       end
       raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
@@ -220,67 +271,159 @@ module HTTPX
       request.on(:promise, &method(:on_promise))
     end
 
-    def init_connection(uri, options)
-      connection = options.connection_class.new(uri, options)
-      catch(:coalesced) do
-        pool.init_connection(connection, options)
-        connection
-      end
+    def do_init_connection(connection, selector)
+      resolve_connection(connection, selector) unless connection.family
     end
 
     # sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
     def send_requests(*requests)
-      connections = _send_requests(requests)
-      receive_requests(requests, connections)
+      selector = get_current_selector { Selector.new }
+      begin
+        _send_requests(requests, selector)
+        receive_requests(requests, selector)
+      ensure
+        unless @wrapped
+          if @persistent
+            deactivate(selector)
+          else
+            close(selector)
+          end
+        end
+      end
     end
 
     # sends an array of HTTPX::Request objects
-    def _send_requests(requests)
-      connections = []
-
+    def _send_requests(requests, selector)
       requests.each do |request|
-        send_request(request, connections)
+        send_request(request, selector)
       end
-
-      connections
     end
 
     # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
-    def receive_requests(requests, connections)
+    def receive_requests(requests, selector)
       # @type var responses: Array[response]
       responses = []
 
-      begin
-        # guarantee ordered responses
-        loop do
-          request = requests.first
+      # guarantee ordered responses
+      loop do
+        request = requests.first
+
+        return responses unless request
 
-          return responses unless request
+        catch(:coalesced) { selector.next_tick } until (response = fetch_response(request, selector, request.options))
+        request.emit(:complete, response)
 
-          catch(:coalesced) { pool.next_tick } until (response = fetch_response(request, connections, request.options))
+        responses << response
+        requests.shift
 
+        break if requests.empty?
+
+        next unless selector.empty?
+
+        # in some cases, the pool of connections might have been drained because there was some
+        # handshake error, and the error responses have already been emitted, but there was no
+        # opportunity to traverse the requests, hence we're returning only a fraction of the errors
+        # we were supposed to. This effectively fetches the existing responses and return them.
+        while (request = requests.shift)
+          response = fetch_response(request, selector, request.options)
+          request.emit(:complete, response) if response
           responses << response
-          requests.shift
+        end
+        break
+      end
+      responses
+    end
 
-          break if requests.empty?
+    def resolve_connection(connection, selector)
+      if connection.addresses || connection.open?
+        #
+        # there are two cases in which we want to activate initialization of
+        # connection immediately:
+        #
+        # 1. when the connection already has addresses, i.e. it doesn't need to
+        #    resolve a name (not the same as name being an IP, yet)
+        # 2. when the connection is initialized with an external already open IO.
+        #
+        on_resolver_connection(connection, selector)
+        return
+      end
 
-          next unless pool.empty?
+      resolver = find_resolver_for(connection, selector)
 
-          # in some cases, the pool of connections might have been drained because there was some
-          # handshake error, and the error responses have already been emitted, but there was no
-          # opportunity to traverse the requests, hence we're returning only a fraction of the errors
-          # we were supposed to. This effectively fetches the existing responses and return them.
-          while (request = requests.shift)
-            responses << fetch_response(request, connections, request.options)
-          end
-          break
+      resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
+    end
+
+    def on_resolver_connection(connection, selector)
+      from_pool = false
+      found_connection = selector.find_mergeable_connection(connection) || begin
+        from_pool = true
+        @pool.checkout_mergeable_connection(connection)
+      end
+
+      return select_connection(connection, selector) unless found_connection
+
+      if found_connection.open?
+        coalesce_connections(found_connection, connection, selector, from_pool)
+      else
+        found_connection.once(:open) do
+          next unless found_connection.current_session == self
+
+          coalesce_connections(found_connection, connection, selector, from_pool)
         end
-        responses
-      ensure
-        if @persistent
-          pool.deactivate(connections)
-        else
-          close(connections)
+      end
+    end
+
+    def on_resolver_close(resolver, selector)
+      return if resolver.closed?
+
+      deselect_resolver(resolver, selector)
+      resolver.close unless resolver.closed?
+    end
+
+    def find_resolver_for(connection, selector)
+      resolver = selector.find_resolver(connection.options)
+
+      unless resolver
+        resolver = @pool.checkout_resolver(connection.options)
+        resolver.current_session = self
+        resolver.current_selector = selector
+      end
+
+      resolver
+    end
+
+    # coalesces +conn2+ into +conn1+. if +conn1+ was loaded from the connection pool
+    # (it is known via +from_pool+), then it adds its to the +selector+.
+    def coalesce_connections(conn1, conn2, selector, from_pool)
+      unless conn1.coalescable?(conn2)
+        select_connection(conn2, selector)
+        @pool.checkin_connection(conn1) if from_pool
+        return false
+      end
+
+      conn2.coalesced_connection = conn1
+      select_connection(conn1, selector) if from_pool
+      deselect_connection(conn2, selector)
+      true
+    end
+
+    def get_current_selector
+      selector_store[self] || (yield if block_given?)
+    end
+
+    def set_current_selector(selector)
+      if selector
+        selector_store[self] = selector
+      else
+        selector_store.delete(self)
+      end
+    end
+
+    def selector_store
+      th_current = Thread.current
+      th_current.thread_variable_get(:httpx_persistent_selector_store) || begin
+        {}.compare_by_identity.tap do |store|
+          th_current.thread_variable_set(:httpx_persistent_selector_store, store)
         end
       end
     end

data/lib/httpx/timers.rb

@@ -26,7 +26,7 @@ module HTTPX
 
       @next_interval_at = nil
 
-      interval
+      Timer.new(interval, callback)
     end
 
     def wait_interval
@@ -43,11 +43,22 @@ module HTTPX
 
       elapsed_time = Utils.elapsed_time(@next_interval_at)
 
-      @intervals.delete_if { |interval| interval.elapse(elapsed_time) <= 0 }
+      @intervals = @intervals.drop_while { |interval| interval.elapse(elapsed_time) <= 0 }
 
       @next_interval_at = nil if @intervals.empty?
     end
 
+    class Timer
+      def initialize(interval, callback)
+        @interval = interval
+        @callback = callback
+      end
+
+      def cancel
+        @interval.delete(@callback)
+      end
+    end
+
     class Interval
       include Comparable
 
@@ -63,6 +74,10 @@ module HTTPX
         @on_empty = blk
       end
 
+      def cancel
+        @on_empty.call
+      end
+
       def <=>(other)
         @interval <=> other.interval
       end

data/lib/httpx/transcoder/body.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "forwardable"
+require "delegate"
 
 module HTTPX::Transcoder
   module Body
@@ -8,46 +8,32 @@ module HTTPX::Transcoder
 
     module_function
 
-    class Encoder
-      extend Forwardable
-
-      def_delegator :@raw, :to_s
-
+    class Encoder < SimpleDelegator
       def initialize(body)
-        @raw = body
+        body = body.open(File::RDONLY, encoding: Encoding::BINARY) if Object.const_defined?(:Pathname) && body.is_a?(Pathname)
+        @body = body
+        super(body)
       end
 
       def bytesize
-        if @raw.respond_to?(:bytesize)
-          @raw.bytesize
-        elsif @raw.respond_to?(:to_ary)
-          @raw.sum(&:bytesize)
-        elsif @raw.respond_to?(:size)
-          @raw.size || Float::INFINITY
-        elsif @raw.respond_to?(:length)
-          @raw.length || Float::INFINITY
-        elsif @raw.respond_to?(:each)
+        if @body.respond_to?(:bytesize)
+          @body.bytesize
+        elsif @body.respond_to?(:to_ary)
+          @body.sum(&:bytesize)
+        elsif @body.respond_to?(:size)
+          @body.size || Float::INFINITY
+        elsif @body.respond_to?(:length)
+          @body.length || Float::INFINITY
+        elsif @body.respond_to?(:each)
           Float::INFINITY
         else
-          raise Error, "cannot determine size of body: #{@raw.inspect}"
+          raise Error, "cannot determine size of body: #{@body.inspect}"
         end
       end
 
       def content_type
         "application/octet-stream"
       end
-
-      private
-
-      def respond_to_missing?(meth, *args)
-        @raw.respond_to?(meth, *args) || super
-      end
-
-      def method_missing(meth, *args, &block)
-        return super unless @raw.respond_to?(meth)
-
-        @raw.__send__(meth, *args, &block)
-      end
     end
 
     def encode(body)

data/lib/httpx/transcoder/form.rb

@@ -20,6 +20,8 @@ module HTTPX
 
         def_delegator :@raw, :bytesize
 
+        def_delegator :@raw, :==
+
         def initialize(form)
           @raw = form.each_with_object("".b) do |(key, val), buf|
             HTTPX::Transcoder.normalize_keys(key, val) do |k, v|

data/lib/httpx/transcoder/gzip.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "forwardable"
-require "uri"
-require "stringio"
 require "zlib"
 
 module HTTPX