httpx 1.5.1 → 1.6.0

data/lib/httpx/plugins/fiber_concurrency.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    # This plugin makes a session reuse the same selector across all fibers in a given thread.
+    #
+    # This enables integration with fiber scheduler implementations such as [async](https://github.com/async).
+    #
+    # # https://gitlab.com/os85/httpx/wikis/FiberConcurrency
+    #
+    module FiberConcurrency
+      def self.subplugins
+        {
+          h2c: FiberConcurrencyH2C,
+        }
+      end
+
+      module InstanceMethods
+        private
+
+        def send_request(request, *)
+          request.set_context!
+
+          super
+        end
+
+        def get_current_selector
+          super(&nil) || begin
+            return unless block_given?
+
+            default = yield
+
+            set_current_selector(default)
+
+            default
+          end
+        end
+      end
+
+      module RequestMethods
+        # the execution context (fiber) this request was sent on.
+        attr_reader :context
+
+        def initialize(*)
+          super
+          @context = nil
+        end
+
+        # sets the execution context for this request. the default is the current fiber.
+        def set_context!
+          @context ||= Fiber.current # rubocop:disable Naming/MemoizedInstanceVariableName
+        end
+
+        # checks whether the current execution context is the one where the request was created.
+        def current_context?
+          @context == Fiber.current
+        end
+
+        def complete!(response = @response)
+          @context = nil
+          super
+        end
+      end
+
+      module ConnectionMethods
+        def current_context?
+          @pending.any?(&:current_context?) || (
+            @sibling && @sibling.pending.any?(&:current_context?)
+          )
+        end
+
+        def interests
+          return if connecting? && @pending.none?(&:current_context?)
+
+          super
+        end
+
+        def send(request)
+          # DoH requests bypass the session, so context needs to be set here.
+          request.set_context!
+
+          super
+        end
+      end
+
+      module HTTP1Methods
+        def interests
+          request = @request || @requests.first
+
+          return unless request
+
+          return unless request.current_context? || @requests.any?(&:current_context?) || @pending.any?(&:current_context?)
+
+          super
+        end
+      end
+
+      module HTTP2Methods
+        def initialize(*)
+          super
+          @contexts = Hash.new { |hs, k| hs[k] = Set.new }
+        end
+
+        def interests
+          if @connection.state == :connected && @handshake_completed && !@contexts.key?(Fiber.current)
+            return :w unless @pings.empty?
+
+            return
+          end
+
+          super
+        end
+
+        def send(request, *)
+          add_to_context(request)
+
+          super
+        end
+
+        private
+
+        def on_close(_, error, _)
+          if error == :http_1_1_required
+            # remove all pending requests context
+            @pending.each do |req|
+              clear_from_context(req)
+            end
+          end
+
+          super
+        end
+
+        def on_stream_close(_, request, error)
+          clear_from_context(request) if error != :stream_closed && @streams.key?(request)
+
+          super
+        end
+
+        def teardown(request = nil)
+          super
+
+          if request
+            clear_from_context(request)
+          else
+            @contexts.clear
+          end
+        end
+
+        def add_to_context(request)
+          @contexts[request.context] << request
+        end
+
+        def clear_from_context(request)
+          requests = @contexts[request.context]
+
+          requests.delete(request)
+
+          @contexts.delete(request.context) if requests.empty?
+        end
+      end
+
+      module NativeResolverMethods
+        private
+
+        def calculate_interests
+          return if @queries.empty?
+
+          return unless @queries.values.any?(&:current_context?) || @connections.any?(&:current_context?)
+
+          super
+        end
+      end
+
+      module SystemResolverMethods
+        def interests
+          return unless @queries.any? { |_, conn| conn.current_context? }
+
+          super
+        end
+      end
+
+      module FiberConcurrencyH2C
+        module HTTP2Methods
+          def upgrade(request, *)
+            @contexts[request.context] << request
+
+            super
+          end
+        end
+      end
+    end
+
+    register_plugin :fiber_concurrency, FiberConcurrency
+  end
+end

data/lib/httpx/plugins/follow_redirects.rb

@@ -34,6 +34,8 @@ module HTTPX
       #                                 (defaults to <tt>false</tt>).
       # :redirect_on :: optional callback which receives the redirect location and can halt the redirect chain if it returns <tt>false</tt>.
       module OptionsMethods
+        private
+
         def option_max_redirects(value)
           num = Integer(value)
           raise TypeError, ":max_redirects must be positive" if num.negative?

data/lib/httpx/plugins/grpc.rb

@@ -70,6 +70,8 @@ module HTTPX
       end
 
       module OptionsMethods
+        private
+
         def option_grpc_service(value)
           String(value)
         end

data/lib/httpx/plugins/h2c.rb

@@ -21,24 +21,17 @@ module HTTPX
         end
 
         def extra_options(options)
-          options.merge(max_concurrent_requests: 1, upgrade_handlers: options.upgrade_handlers.merge("h2c" => self))
+          options.merge(
+            h2c_class: Class.new(options.http2_class) { include(H2CParser) },
+            max_concurrent_requests: 1,
+            upgrade_handlers: options.upgrade_handlers.merge("h2c" => self),
+          )
         end
       end
 
-      class H2CParser < Connection::HTTP2
-        def upgrade(request, response)
-          # skip checks, it is assumed that this is the first
-          # request in the connection
-          stream = @connection.upgrade
-
-          # on_settings
-          handle_stream(stream, request)
-          @streams[request] = stream
-
-          # clean up data left behind in the buffer, if the server started
-          # sending frames
-          data = response.read
-          @connection << data
+      module OptionsMethods
+        def option_h2c_class(value)
+          value
         end
       end
 
@@ -82,7 +75,7 @@ module HTTPX
             @inflight -= prev_parser.requests.size
           end
 
-          @parser = H2CParser.new(@write_buffer, @options)
+          @parser = request.options.h2c_class.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
           @inflight += 1
           @parser.upgrade(request, response)
@@ -111,6 +104,23 @@ module HTTPX
           end
         end
       end
+
+      module H2CParser
+        def upgrade(request, response)
+          # skip checks, it is assumed that this is the first
+          # request in the connection
+          stream = @connection.upgrade
+
+          # on_settings
+          handle_stream(stream, request)
+          @streams[request] = stream
+
+          # clean up data left behind in the buffer, if the server started
+          # sending frames
+          data = response.read
+          @connection << data
+        end
+      end
     end
     register_plugin(:h2c, H2C)
   end

data/lib/httpx/plugins/internal_telemetry.rb

@@ -50,15 +50,6 @@ module HTTPX
         end
       end
 
-      module NativeResolverMethods
-        def transition(nextstate)
-          state = @state
-          val = super
-          meter_elapsed_time("Resolver::Native: #{state} -> #{nextstate}")
-          val
-        end
-      end
-
       module InstanceMethods
         def self.included(klass)
           klass.prepend TrackTimeMethods
@@ -69,13 +60,6 @@ module HTTPX
           meter_elapsed_time("Session: initializing...")
           super
           meter_elapsed_time("Session: initialized!!!")
-          resolver_type = @options.resolver_class
-          resolver_type = Resolver.resolver_for(resolver_type)
-          return unless resolver_type <= Resolver::Native
-
-          resolver_type.prepend TrackTimeMethods
-          resolver_type.prepend NativeResolverMethods
-          @options = @options.merge(resolver_class: resolver_type)
         end
 
         def close(*)
@@ -104,33 +88,6 @@ module HTTPX
         end
       end
 
-      module RequestMethods
-        def self.included(klass)
-          klass.prepend Loggable
-          klass.prepend TrackTimeMethods
-          super
-        end
-
-        def transition(nextstate)
-          prev_state = @state
-          super
-          meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{prev_state}] -> #{@state}") if prev_state != @state
-        end
-      end
-
-      module ConnectionMethods
-        def self.included(klass)
-          klass.prepend TrackTimeMethods
-          super
-        end
-
-        def handle_transition(nextstate)
-          state = @state
-          super
-          meter_elapsed_time("Connection##{object_id}[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
-        end
-      end
-
       module PoolMethods
         def self.included(klass)
           klass.prepend Loggable
@@ -138,12 +95,6 @@ module HTTPX
           super
         end
 
-        def checkout_connection(request_uri, options)
-          super.tap do |connection|
-            meter_elapsed_time("Pool##{object_id}: checked out connection for Connection##{connection.object_id}[#{connection.origin}]}")
-          end
-        end
-
         def checkin_connection(connection)
           super.tap do
             meter_elapsed_time("Pool##{object_id}: checked in connection for Connection##{connection.object_id}[#{connection.origin}]}")

data/lib/httpx/plugins/ntlm_auth.rb

@@ -17,6 +17,8 @@ module HTTPX
       end
 
       module OptionsMethods
+        private
+
         def option_ntlm(value)
           raise TypeError, ":ntlm must be a #{Authentication::Ntlm}" unless value.is_a?(Authentication::Ntlm)
 

data/lib/httpx/plugins/oauth.rb

@@ -98,6 +98,8 @@ module HTTPX
       end
 
       module OptionsMethods
+        private
+
         def option_oauth_session(value)
           case value
           when Hash

data/lib/httpx/plugins/persistent.rb

@@ -18,13 +18,17 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Persistent
     #
     module Persistent
-      def self.load_dependencies(klass)
-        max_retries = if klass.default_options.respond_to?(:max_retries)
-          [klass.default_options.max_retries, 1].max
-        else
-          1
+      class << self
+        def load_dependencies(klass)
+          klass.plugin(:fiber_concurrency)
+
+          max_retries = if klass.default_options.respond_to?(:max_retries)
+            [klass.default_options.max_retries, 1].max
+          else
+            1
+          end
+          klass.plugin(:retries, max_retries: max_retries)
         end
-        klass.plugin(:retries, max_retries: max_retries)
       end
 
       def self.extra_options(options)
@@ -32,6 +36,23 @@ module HTTPX
       end
 
       module InstanceMethods
+        def close(*)
+          super
+
+          # traverse other threads and unlink respective selector
+          # WARNING: this is not thread safe, make sure that the session isn't being
+          # used anymore, or all non-main threads are stopped.
+          Thread.list.each do |th|
+            store = thread_selector_store(th)
+
+            next unless store && store.key?(self)
+
+            selector = store.delete(self)
+
+            selector_close(selector)
+          end
+        end
+
         private
 
         def repeatable_request?(request, _)
@@ -54,18 +75,6 @@ module HTTPX
             # consequences.
             (!ex.is_a?(RequestTimeoutError) || @options.max_retries != 1)
         end
-
-        def get_current_selector
-          super(&nil) || begin
-            return unless block_given?
-
-            default = yield
-
-            set_current_selector(default)
-
-            default
-          end
-        end
       end
     end
     register_plugin :persistent, Persistent

data/lib/httpx/plugins/proxy/socks4.rb

@@ -4,7 +4,7 @@ require "resolv"
 require "ipaddr"
 
 module HTTPX
-  class Socks4Error < HTTPProxyError; end
+  class Socks4Error < ProxyError; end
 
   module Plugins
     module Proxy

data/lib/httpx/plugins/proxy/socks5.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class Socks5Error < HTTPProxyError; end
+  class Socks5Error < ProxyError; end
 
   module Plugins
     module Proxy

data/lib/httpx/plugins/proxy/ssh.rb

@@ -13,6 +13,8 @@ module HTTPX
         end
 
         module OptionsMethods
+          private
+
           def option_proxy(value)
             Hash[value]
           end

data/lib/httpx/plugins/proxy.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class HTTPProxyError < ConnectionError; end
+  class ProxyError < ConnectionError; end
 
   module Plugins
     #
@@ -15,7 +15,8 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Proxy
     #
     module Proxy
-      Error = HTTPProxyError
+      class ProxyConnectionError < ProxyError; end
+
       PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze
 
       class << self
@@ -28,6 +29,12 @@ module HTTPX
         def extra_options(options)
           options.merge(supported_proxy_protocols: [])
         end
+
+        def subplugins
+          {
+            retries: ProxyRetries,
+          }
+        end
       end
 
       class Parameters
@@ -135,6 +142,8 @@ module HTTPX
       # :proxy :: proxy options defining *:uri*, *:username*, *:password* or
       #           *:scheme* (i.e. <tt>{ uri: "http://proxy" }</tt>)
       module OptionsMethods
+        private
+
         def option_proxy(value)
           value.is_a?(Parameters) ? value : Parameters.new(**Hash[value])
         end
@@ -160,9 +169,9 @@ module HTTPX
 
           next_proxy = proxy.uri
 
-          raise Error, "Failed to connect to proxy" unless next_proxy
+          raise ProxyError, "Failed to connect to proxy" unless next_proxy
 
-          raise Error,
+          raise ProxyError,
                 "#{next_proxy.scheme}: unsupported proxy protocol" unless options.supported_proxy_protocols.include?(next_proxy.scheme)
 
           if (no_proxy = proxy.no_proxy)
@@ -179,20 +188,28 @@ module HTTPX
         private
 
         def fetch_response(request, selector, options)
-          response = super
-
-          if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
-            options.proxy.shift
-
-            # return last error response if no more proxies to try
-            return response if options.proxy.uri.nil?
-
-            log { "failed connecting to proxy, trying next..." }
-            request.transition(:idle)
-            send_request(request, selector, options)
-            return
+          response = request.response # in case it goes wrong later
+
+          begin
+            response = super
+
+            if response.is_a?(ErrorResponse) && proxy_error?(request, response, options)
+              options.proxy.shift
+
+              # return last error response if no more proxies to try
+              return response if options.proxy.uri.nil?
+
+              log { "failed connecting to proxy, trying next..." }
+              request.transition(:idle)
+              send_request(request, selector, options)
+              return
+            end
+            response
+          rescue ProxyError
+            # may happen if coupled with retries, and there are no more proxies to try, in which case
+            # it'll end up here
+            response
           end
-          response
         end
 
         def proxy_error?(_request, response, options)
@@ -203,15 +220,15 @@ module HTTPX
           when NativeResolveError
             proxy_uri = URI(options.proxy.uri)
 
-            peer = error.connection.peer
+            unresolved_host = error.host
 
             # failed resolving proxy domain
-            peer.host == proxy_uri.host && peer.port == proxy_uri.port
+            unresolved_host == proxy_uri.host
           when ResolveError
             proxy_uri = URI(options.proxy.uri)
 
             error.message.end_with?(proxy_uri.to_s)
-          when *PROXY_ERRORS
+          when ProxyConnectionError
             # timeout errors connecting to proxy
             true
           else
@@ -251,6 +268,14 @@ module HTTPX
           when :connecting
             consume
           end
+        rescue *PROXY_ERRORS => e
+          if connecting?
+            error = ProxyConnectionError.new(e.message)
+            error.set_backtrace(e.backtrace)
+            raise error
+          end
+
+          raise e
         end
 
         def reset
@@ -292,13 +317,29 @@ module HTTPX
           end
           super
         end
+
+        def purge_after_closed
+          super
+          @io = @io.proxy_io if @io.respond_to?(:proxy_io)
+        end
+      end
+
+      module ProxyRetries
+        module InstanceMethods
+          def retryable_error?(ex)
+            super || ex.is_a?(ProxyConnectionError)
+          end
+        end
       end
     end
     register_plugin :proxy, Proxy
   end
 
   class ProxySSL < SSL
+    attr_reader :proxy_io
+
     def initialize(tcp, request_uri, options)
+      @proxy_io = tcp
       @io = tcp.to_io
       super(request_uri, tcp.addresses, options)
       @hostname = request_uri.host

data/lib/httpx/plugins/response_cache/file_store.rb

@@ -81,7 +81,7 @@ module HTTPX::Plugins
 
           response.body.rewind
 
-          ::IO.copy_stream(response.body, f)
+          IO.copy_stream(response.body, f)
         end
       end
 
@@ -131,7 +131,7 @@ module HTTPX::Plugins
         response.original_request = original_request
         response.finish!
 
-        ::IO.copy_stream(f, response.body)
+        IO.copy_stream(f, response.body)
 
         response
       end

data/lib/httpx/plugins/response_cache.rb

@@ -67,6 +67,8 @@ module HTTPX
       # * +#clear() -> void+)
       #
       module OptionsMethods
+        private
+
         def option_response_cache_store(value)
           case value
           when :store

data/lib/httpx/plugins/retries.rb

@@ -57,6 +57,8 @@ module HTTPX
       # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
       #              (i.e. <tt>->(res) { ... }</tt>).
       module OptionsMethods
+        private
+
         def option_retry_after(value)
           # return early if callable
           unless value.respond_to?(:call)

data/lib/httpx/plugins/ssrf_filter.rb

@@ -91,6 +91,8 @@ module HTTPX
       #
       # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
       module OptionsMethods
+        private
+
         def option_allowed_schemes(value)
           Array(value)
         end
@@ -129,8 +131,6 @@ module HTTPX
         end
 
         def addresses=(addrs)
-          addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
-
           addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
 
           raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?