httpx 1.3.3 → 1.4.0

data/lib/httpx/session.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 module HTTPX
-  EMPTY_HASH = {}.freeze
-
   # Class implementing the APIs being used publicly.
   #
   #   HTTPX.get(..) #=> delegating to an internal HTTPX::Session object.
@@ -19,6 +17,9 @@ module HTTPX
       @options = self.class.default_options.merge(options)
       @responses = {}
       @persistent = @options.persistent
+      @pool = @options.pool_class.new(@options.pool_options)
+      @wrapped = false
+      @closing = false
       wrap(&blk) if blk
     end
 
@@ -28,21 +29,56 @@ module HTTPX
     #     http.get("https://wikipedia.com")
     #   end # wikipedia connection closes here
     def wrap
-      prev_persistent = @persistent
-      @persistent = true
-      pool.wrap do
-        begin
-          yield self
-        ensure
-          @persistent = prev_persistent
-          close unless @persistent
+      prev_wrapped = @wrapped
+      @wrapped = true
+      was_initialized = false
+      current_selector = get_current_selector do
+        selector = Selector.new
+
+        set_current_selector(selector)
+
+        was_initialized = true
+
+        selector
+      end
+      begin
+        yield self
+      ensure
+        unless prev_wrapped
+          if @persistent
+            deactivate(current_selector)
+          else
+            close(current_selector)
+          end
         end
+        @wrapped = prev_wrapped
+        set_current_selector(nil) if was_initialized
       end
     end
 
-    # closes all the active connections from the session
-    def close(*args)
-      pool.close(*args)
+    # closes all the active connections from the session.
+    #
+    # when called directly without specifying +selector+, all available connections
+    # will be picked up from the connection pool and closed. Connections in use
+    # by other sessions, or same session in a different thread, will not be reaped.
+    def close(selector = Selector.new)
+      # throw resolvers away from the pool
+      @pool.reset_resolvers
+
+      # preparing to throw away connections
+      while (connection = @pool.pop_connection)
+        next if connection.state == :closed
+
+        connection.current_session = self
+        connection.current_selector = selector
+        select_connection(connection, selector)
+      end
+      begin
+        @closing = true
+        selector.terminate
+      ensure
+        @closing = false
+      end
     end
 
     # performs one, or multple requests; it accepts:
@@ -89,113 +125,153 @@ module HTTPX
       request
     end
 
-    private
-
-    # returns the HTTPX::Pool object which manages the networking required to
-    # perform requests.
-    def pool
-      Thread.current[:httpx_connection_pool] ||= Pool.new
+    def select_connection(connection, selector)
+      selector.register(connection)
     end
 
-    # callback executed when a response for a given request has been received.
-    def on_response(request, response)
-      @responses[request] = response
-    end
+    alias_method :select_resolver, :select_connection
 
-    # callback executed when an HTTP/2 promise frame has been received.
-    def on_promise(_, stream)
-      log(level: 2) { "#{stream.id}: refusing stream!" }
-      stream.refuse
-    end
+    def deselect_connection(connection, selector, cloned = false)
+      selector.deregister(connection)
 
-    # returns the corresponding HTTP::Response to the given +request+ if it has been received.
-    def fetch_response(request, _, _)
-      @responses.delete(request)
-    end
+      # when connections coalesce
+      return if connection.state == :idle
 
-    # returns the HTTPX::Connection through which the +request+ should be sent through.
-    def find_connection(request, connections, options)
-      uri = request.uri
+      return if cloned
 
-      connection = pool.find_connection(uri, options) || init_connection(uri, options)
-      unless connections.nil? || connections.include?(connection)
-        connections << connection
-        set_connection_callbacks(connection, connections, options)
-      end
-      connection
+      return if @closing && connection.state == :closed
+
+      @pool.checkin_connection(connection)
     end
 
-    # sends the +request+ to the corresponding HTTPX::Connection
-    def send_request(request, connections, options = request.options)
-      error = catch(:resolve_error) do
-        connection = find_connection(request, connections, options)
-        connection.send(request)
-      end
-      return unless error.is_a?(Error)
+    def deselect_resolver(resolver, selector)
+      selector.deregister(resolver)
 
-      request.emit(:response, ErrorResponse.new(request, error))
+      return if @closing && resolver.closed?
+
+      @pool.checkin_resolver(resolver)
     end
 
-    # sets the callbacks on the +connection+ required to process certain specific
-    # connection lifecycle events which deal with request rerouting.
-    def set_connection_callbacks(connection, connections, options, cloned: false)
-      connection.only(:misdirected) do |misdirected_request|
-        other_connection = connection.create_idle(ssl: { alpn_protocols: %w[http/1.1] })
-        other_connection.merge(connection)
-        catch(:coalesced) do
-          pool.init_connection(other_connection, options)
+    def try_clone_connection(connection, selector, family)
+      connection.family ||= family
+
+      return connection if connection.family == family
+
+      new_connection = connection.class.new(connection.origin, connection.options)
+
+      new_connection.family = family
+      new_connection.current_session = self
+      new_connection.current_selector = selector
+
+      connection.once(:tcp_open) { new_connection.force_reset(true) }
+      connection.once(:connect_error) do |err|
+        if new_connection.connecting?
+          new_connection.merge(connection)
+          connection.emit(:cloned, new_connection)
+          connection.force_reset(true)
+        else
+          connection.__send__(:handle_error, err)
+        end
+      end
+
+      new_connection.once(:tcp_open) do |new_conn|
+        if new_conn != connection
+          new_conn.merge(connection)
+          connection.force_reset(true)
         end
-        set_connection_callbacks(other_connection, connections, options)
-        connections << other_connection
-        misdirected_request.transition(:idle)
-        other_connection.send(misdirected_request)
       end
-      connection.only(:altsvc) do |alt_origin, origin, alt_params|
-        other_connection = build_altsvc_connection(connection, connections, alt_origin, origin, alt_params, options)
-        connections << other_connection if other_connection
+      new_connection.once(:connect_error) do |err|
+        if connection.connecting?
+          # main connection has the requests
+          connection.merge(new_connection)
+          new_connection.emit(:cloned, connection)
+          new_connection.force_reset(true)
+        else
+          new_connection.__send__(:handle_error, err)
+        end
       end
-      connection.only(:cloned) do |cloned_conn|
-        set_connection_callbacks(cloned_conn, connections, options, cloned: true)
-        connections << cloned_conn
-      end unless cloned
+
+      do_init_connection(new_connection, selector)
+      new_connection
     end
 
-    # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
-    def build_altsvc_connection(existing_connection, connections, alt_origin, origin, alt_params, options)
-      # do not allow security downgrades on altsvc negotiation
-      return if existing_connection.origin.scheme == "https" && alt_origin.scheme != "https"
+    # returns the HTTPX::Connection through which the +request+ should be sent through.
+    def find_connection(request_uri, selector, options)
+      if (connection = selector.find_connection(request_uri, options))
+        return connection
+      end
 
-      altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
+      connection = @pool.checkout_connection(request_uri, options)
+
+      connection.current_session = self
+      connection.current_selector = selector
+
+      case connection.state
+      when :idle
+        do_init_connection(connection, selector)
+      when :open
+        select_connection(connection, selector) if options.io
+      when :closed
+        connection.idling
+        select_connection(connection, selector)
+      when :closing
+        connection.once(:close) do
+          connection.idling
+          select_connection(connection, selector)
+        end
+      end
 
-      # altsvc already exists, somehow it wasn't advertised, probably noop
-      return unless altsvc
+      connection
+    end
 
-      alt_options = options.merge(ssl: options.ssl.merge(hostname: URI(origin).host))
+    private
 
-      connection = pool.find_connection(alt_origin, alt_options) || init_connection(alt_origin, alt_options)
+    def deactivate(selector)
+      selector.each_connection do |connection|
+        connection.deactivate
+        deselect_connection(connection, selector) if connection.state == :inactive
+      end
+    end
 
-      # advertised altsvc is the same origin being used, ignore
-      return if connection == existing_connection
+    # callback executed when a response for a given request has been received.
+    def on_response(request, response)
+      @responses[request] = response
+    end
 
-      connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
+    # callback executed when an HTTP/2 promise frame has been received.
+    def on_promise(_, stream)
+      log(level: 2) { "#{stream.id}: refusing stream!" }
+      stream.refuse
+    end
 
-      set_connection_callbacks(connection, connections, alt_options)
+    # returns the corresponding HTTP::Response to the given +request+ if it has been received.
+    def fetch_response(request, _selector, _options)
+      @responses.delete(request)
+    end
 
-      log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
+    # sends the +request+ to the corresponding HTTPX::Connection
+    def send_request(request, selector, options = request.options)
+      error = begin
+        catch(:resolve_error) do
+          connection = find_connection(request.uri, selector, options)
+          connection.send(request)
+        end
+      rescue StandardError => e
+        e
+      end
+      return unless error && error.is_a?(Exception)
 
-      connection.merge(existing_connection)
-      existing_connection.terminate
-      connection
-    rescue UnsupportedSchemeError
-      altsvc["noop"] = true
-      nil
+      if error.is_a?(Error)
+        request.emit(:response, ErrorResponse.new(request, error))
+      else
+        raise error if selector.empty?
+      end
     end
 
     # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
     def build_requests(*args, params)
       requests = if args.size == 1
         reqs = args.first
-        # TODO: find a way to make requests share same options object
         reqs.map do |verb, uri, ps = EMPTY_HASH|
           request_params = params
           request_params = request_params.merge(ps) unless ps.empty?
@@ -204,7 +280,6 @@ module HTTPX
       else
         verb, uris = args
         if uris.respond_to?(:each)
-          # TODO: find a way to make requests share same options object
           uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
             request_params = params
             request_params = request_params.merge(ps) unless ps.empty?
@@ -224,78 +299,160 @@ module HTTPX
       request.on(:promise, &method(:on_promise))
     end
 
-    def init_connection(uri, options)
-      connection = options.connection_class.new(uri, options)
-      catch(:coalesced) do
-        pool.init_connection(connection, options)
-        connection
-      end
-    end
-
-    def deactivate_connection(request, connections, options)
-      conn = connections.find do |c|
-        c.match?(request.uri, options)
-      end
-
-      pool.deactivate(conn) if conn
+    def do_init_connection(connection, selector)
+      resolve_connection(connection, selector) unless connection.family
     end
 
     # sends an array of HTTPX::Request +requests+, returns the respective array of HTTPX::Response objects.
     def send_requests(*requests)
-      connections = _send_requests(requests)
-      receive_requests(requests, connections)
+      selector = get_current_selector { Selector.new }
+      begin
+        _send_requests(requests, selector)
+        receive_requests(requests, selector)
+      ensure
+        unless @wrapped
+          if @persistent
+            deactivate(selector)
+          else
+            close(selector)
+          end
+        end
+      end
     end
 
     # sends an array of HTTPX::Request objects
-    def _send_requests(requests)
-      connections = []
-
+    def _send_requests(requests, selector)
       requests.each do |request|
-        send_request(request, connections)
+        send_request(request, selector)
       end
-
-      connections
     end
 
     # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
-    def receive_requests(requests, connections)
+    def receive_requests(requests, selector)
       # @type var responses: Array[response]
       responses = []
 
-      begin
-        # guarantee ordered responses
-        loop do
-          request = requests.first
+      # guarantee ordered responses
+      loop do
+        request = requests.first
+
+        return responses unless request
 
-          return responses unless request
+        catch(:coalesced) { selector.next_tick } until (response = fetch_response(request, selector, request.options))
+        request.emit(:complete, response)
 
-          catch(:coalesced) { pool.next_tick } until (response = fetch_response(request, connections, request.options))
-          request.emit(:complete, response)
+        responses << response
+        requests.shift
 
+        break if requests.empty?
+
+        next unless selector.empty?
+
+        # in some cases, the pool of connections might have been drained because there was some
+        # handshake error, and the error responses have already been emitted, but there was no
+        # opportunity to traverse the requests, hence we're returning only a fraction of the errors
+        # we were supposed to. This effectively fetches the existing responses and return them.
+        while (request = requests.shift)
+          response = fetch_response(request, selector, request.options)
+          request.emit(:complete, response) if response
           responses << response
-          requests.shift
+        end
+        break
+      end
+      responses
+    end
 
-          break if requests.empty?
+    def resolve_connection(connection, selector)
+      if connection.addresses || connection.open?
+        #
+        # there are two cases in which we want to activate initialization of
+        # connection immediately:
+        #
+        # 1. when the connection already has addresses, i.e. it doesn't need to
+        #    resolve a name (not the same as name being an IP, yet)
+        # 2. when the connection is initialized with an external already open IO.
+        #
+        connection.once(:connect_error, &connection.method(:handle_error))
+        on_resolver_connection(connection, selector)
+        return
+      end
 
-          next unless pool.empty?
+      resolver = find_resolver_for(connection, selector)
 
-          # in some cases, the pool of connections might have been drained because there was some
-          # handshake error, and the error responses have already been emitted, but there was no
-          # opportunity to traverse the requests, hence we're returning only a fraction of the errors
-          # we were supposed to. This effectively fetches the existing responses and return them.
-          while (request = requests.shift)
-            response = fetch_response(request, connections, request.options)
-            request.emit(:complete, response) if response
-            responses << response
-          end
-          break
+      resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
+    end
+
+    def on_resolver_connection(connection, selector)
+      from_pool = false
+      found_connection = selector.find_mergeable_connection(connection) || begin
+        from_pool = true
+        @pool.checkout_mergeable_connection(connection)
+      end
+
+      return select_connection(connection, selector) unless found_connection
+
+      if found_connection.open?
+        coalesce_connections(found_connection, connection, selector, from_pool)
+      else
+        found_connection.once(:open) do
+          coalesce_connections(found_connection, connection, selector, from_pool)
         end
-        responses
-      ensure
-        if @persistent
-          pool.deactivate(*connections)
-        else
-          close(connections)
+      end
+    end
+
+    def on_resolver_close(resolver, selector)
+      return if resolver.closed?
+
+      deselect_resolver(resolver, selector)
+      resolver.close unless resolver.closed?
+    end
+
+    def find_resolver_for(connection, selector)
+      resolver = selector.find_resolver(connection.options)
+
+      unless resolver
+        resolver = @pool.checkout_resolver(connection.options)
+        resolver.current_session = self
+        resolver.current_selector = selector
+      end
+
+      resolver
+    end
+
+    # coalesces +conn2+ into +conn1+. if +conn1+ was loaded from the connection pool
+    # (it is known via +from_pool+), then it adds its to the +selector+.
+    def coalesce_connections(conn1, conn2, selector, from_pool)
+      unless conn1.coalescable?(conn2)
+        select_connection(conn2, selector)
+        @pool.checkin_connection(conn1) if from_pool
+        return false
+      end
+
+      conn2.emit(:tcp_open, conn1)
+      conn1.merge(conn2)
+      conn2.coalesced_connection = conn1
+      select_connection(conn1, selector) if from_pool
+      deselect_connection(conn2, selector)
+      true
+    end
+
+    def get_current_selector
+      selector_store[self] || (yield if block_given?)
+    end
+
+    def set_current_selector(selector)
+      if selector
+        selector_store[self] = selector
+      else
+        selector_store.delete(self)
+      end
+    end
+
+    def selector_store
+      th_current = Thread.current
+      th_current.thread_variable_get(:httpx_persistent_selector_store) || begin
+        {}.compare_by_identity.tap do |store|
+          th_current.thread_variable_set(:httpx_persistent_selector_store, store)
         end
       end
     end

data/lib/httpx/transcoder/gzip.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "forwardable"
-require "uri"
-require "stringio"
 require "zlib"
 
 module HTTPX

data/lib/httpx/transcoder/json.rb

@@ -6,7 +6,19 @@ module HTTPX::Transcoder
   module JSON
     module_function
 
-    JSON_REGEX = %r{\bapplication/(?:vnd\.api\+|hal\+)?json\b}i.freeze
+    JSON_REGEX = %r{
+      \b
+      application/
+      # optional vendor specific type
+      (?:
+        # token as per https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6
+        [!#$%&'*+\-.^_`|~0-9a-z]+
+        # literal plus sign
+        \+
+      )?
+      json
+      \b
+    }ix.freeze
 
     class Encoder
       extend Forwardable
@@ -45,7 +57,7 @@ module HTTPX::Transcoder
       def json_dump(*args); MultiJson.dump(*args); end
     elsif defined?(Oj)
       def json_load(response, *args); Oj.load(response.to_s, *args); end
-      def json_dump(*args); Oj.dump(*args); end
+      def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
     elsif defined?(Yajl)
       def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end
       def json_dump(*args); Yajl::Encoder.encode(*args); end

data/lib/httpx/transcoder/multipart/encoder.rb

@@ -35,7 +35,9 @@ module HTTPX
 
       def rewind
         form = @form.each_with_object([]) do |(key, val), aux|
-          val = val.reopen(val.path, File::RDONLY) if val.is_a?(File) && val.closed?
+          if val.respond_to?(:path) && val.respond_to?(:reopen) && val.respond_to?(:closed?) && val.closed?
+            val = val.reopen(val.path, File::RDONLY)
+          end
           val.rewind if val.respond_to?(:rewind)
           aux << [key, val]
         end

data/lib/httpx/transcoder/utils/deflater.rb

@@ -1,13 +1,10 @@
 # frozen_string_literal: true
 
-require "forwardable"
 require_relative "body_reader"
 
 module HTTPX
   module Transcoder
     class Deflater
-      extend Forwardable
-
       attr_reader :content_type
 
       def initialize(body)
@@ -51,6 +48,12 @@ module HTTPX
         @closed = true
       end
 
+      def rewind
+        return unless @buffer
+
+        @buffer.rewind
+      end
+
       private
 
       # rubocop:disable Naming/MemoizedInstanceVariableName
@@ -62,7 +65,7 @@ module HTTPX
         )
         ::IO.copy_stream(self, buffer)
 
-        buffer.rewind
+        buffer.rewind if buffer.respond_to?(:rewind)
 
         @buffer = buffer
       end

data/lib/httpx/transcoder/utils/inflater.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTPX
   module Transcoder
     class Inflater

data/lib/httpx/transcoder.rb

@@ -86,7 +86,6 @@ end
 require "httpx/transcoder/body"
 require "httpx/transcoder/form"
 require "httpx/transcoder/json"
-require "httpx/transcoder/xml"
 require "httpx/transcoder/chunker"
 require "httpx/transcoder/deflate"
 require "httpx/transcoder/gzip"

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.3.3"
+  VERSION = "1.4.0"
 end