httpx 0.20.0 → 1.3.1

data/lib/httpx/adapters/faraday.rb

@@ -7,69 +7,110 @@ require "faraday"
 module Faraday
   class Adapter
     class HTTPX < Faraday::Adapter
-      # :nocov:
-      SSL_ERROR = if defined?(Faraday::SSLError)
-        Faraday::SSLError
-      else
-        Faraday::Error::SSLError
-      end
+      module RequestMixin
+        def build_connection(env)
+          return @connection if defined?(@connection)
 
-      CONNECTION_FAILED_ERROR = if defined?(Faraday::ConnectionFailed)
-        Faraday::ConnectionFailed
-      else
-        Faraday::Error::ConnectionFailed
-      end
-      # :nocov:
+          @connection = ::HTTPX.plugin(:persistent).plugin(ReasonPlugin)
+          @connection = @connection.with(@connection_options) unless @connection_options.empty?
+          connection_opts = options_from_env(env)
 
-      unless Faraday::RequestOptions.method_defined?(:stream_response?)
-        module RequestOptionsExtensions
-          refine Faraday::RequestOptions do
-            def stream_response?
-              false
-            end
+          if (bind = env.request.bind)
+            @bind = TCPSocket.new(bind[:host], bind[:port])
+            connection_opts[:io] = @bind
+          end
+          @connection = @connection.with(connection_opts)
+
+          if (proxy = env.request.proxy)
+            proxy_options = { uri: proxy.uri }
+            proxy_options[:username] = proxy.user if proxy.user
+            proxy_options[:password] = proxy.password if proxy.password
+
+            @connection = @connection.plugin(:proxy).with(proxy: proxy_options)
           end
+          @connection = @connection.plugin(OnDataPlugin) if env.request.stream_response?
+
+          @connection = @config_block.call(@connection) || @connection if @config_block
+          @connection
         end
-        using RequestOptionsExtensions
-      end
 
-      module RequestMixin
-        using ::HTTPX::HashExtensions
+        def close
+          @connection.close if @connection
+          @bind.close if @bind
+        end
 
         private
 
+        def connect(env, &blk)
+          connection(env, &blk)
+        rescue ::HTTPX::TLSError => e
+          raise Faraday::SSLError, e
+        rescue Errno::ECONNABORTED,
+               Errno::ECONNREFUSED,
+               Errno::ECONNRESET,
+               Errno::EHOSTUNREACH,
+               Errno::EINVAL,
+               Errno::ENETUNREACH,
+               Errno::EPIPE,
+               ::HTTPX::ConnectionError => e
+          raise Faraday::ConnectionFailed, e
+        end
+
         def build_request(env)
           meth = env[:method]
 
           request_options = {
             headers: env.request_headers,
             body: env.body,
+            **options_from_env(env),
           }
-          [meth, env.url, request_options]
+          [meth.to_s.upcase, env.url, request_options]
         end
 
         def options_from_env(env)
-          timeout_options = {
-            connect_timeout: env.request.open_timeout,
-            operation_timeout: env.request.timeout,
-          }.compact
+          timeout_options = {}
+          req_opts = env.request
+          if (sec = request_timeout(:read, req_opts))
+            timeout_options[:read_timeout] = sec
+          end
 
-          options = {
-            ssl: {},
+          if (sec = request_timeout(:write, req_opts))
+            timeout_options[:write_timeout] = sec
+          end
+
+          if (sec = request_timeout(:open, req_opts))
+            timeout_options[:connect_timeout] = sec
+          end
+
+          {
+            ssl: ssl_options_from_env(env),
             timeout: timeout_options,
           }
+        end
+
+        if defined?(::OpenSSL)
+          def ssl_options_from_env(env)
+            ssl_options = {}
+
+            unless env.ssl.verify.nil?
+              ssl_options[:verify_mode] = env.ssl.verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+            end
 
-          options[:ssl][:verify_mode] = OpenSSL::SSL::VERIFY_PEER if env.ssl.verify
-          options[:ssl][:ca_file] = env.ssl.ca_file if env.ssl.ca_file
-          options[:ssl][:ca_path] = env.ssl.ca_path if env.ssl.ca_path
-          options[:ssl][:cert_store] = env.ssl.cert_store if env.ssl.cert_store
-          options[:ssl][:cert] = env.ssl.client_cert if env.ssl.client_cert
-          options[:ssl][:key] = env.ssl.client_key if env.ssl.client_key
-          options[:ssl][:ssl_version] = env.ssl.version if env.ssl.version
-          options[:ssl][:verify_depth] = env.ssl.verify_depth if env.ssl.verify_depth
-          options[:ssl][:min_version] = env.ssl.min_version if env.ssl.min_version
-          options[:ssl][:max_version] = env.ssl.max_version if env.ssl.max_version
-
-          options
+            ssl_options[:ca_file] = env.ssl.ca_file if env.ssl.ca_file
+            ssl_options[:ca_path] = env.ssl.ca_path if env.ssl.ca_path
+            ssl_options[:cert_store] = env.ssl.cert_store if env.ssl.cert_store
+            ssl_options[:cert] = env.ssl.client_cert if env.ssl.client_cert
+            ssl_options[:key] = env.ssl.client_key if env.ssl.client_key
+            ssl_options[:ssl_version] = env.ssl.version if env.ssl.version
+            ssl_options[:verify_depth] = env.ssl.verify_depth if env.ssl.verify_depth
+            ssl_options[:min_version] = env.ssl.min_version if env.ssl.min_version
+            ssl_options[:max_version] = env.ssl.max_version if env.ssl.max_version
+            ssl_options
+          end
+        else
+          def ssl_options_from_env(*)
+            {}
+          end
         end
       end
 
@@ -100,32 +141,17 @@ module Faraday
       end
 
       module ReasonPlugin
-        if RUBY_VERSION < "2.5"
-          def self.load_dependencies(*)
-            require "webrick"
-          end
-        else
-          def self.load_dependencies(*)
-            require "net/http/status"
-          end
+        def self.load_dependencies(*)
+          require "net/http/status"
         end
+
         module ResponseMethods
-          if RUBY_VERSION < "2.5"
-            def reason
-              WEBrick::HTTPStatus::StatusMessage.fetch(@status)
-            end
-          else
-            def reason
-              Net::HTTP::STATUS_CODES.fetch(@status)
-            end
+          def reason
+            Net::HTTP::STATUS_CODES.fetch(@status)
           end
         end
       end
 
-      def self.session
-        @session ||= ::HTTPX.plugin(:compression).plugin(:persistent).plugin(ReasonPlugin)
-      end
-
       class ParallelManager
         class ResponseHandler < SimpleDelegator
           attr_reader :env
@@ -158,8 +184,9 @@ module Faraday
 
         include RequestMixin
 
-        def initialize
+        def initialize(options)
           @handlers = []
+          @connection_options = options
         end
 
         def enqueue(request)
@@ -169,85 +196,96 @@ module Faraday
         end
 
         def run
+          return unless @handlers.last
+
           env = @handlers.last.env
 
-          session = HTTPX.session.with(options_from_env(env))
-          session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
-          session = session.plugin(OnDataPlugin) if env.request.stream_response?
+          connect(env) do |session|
+            requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
 
-          requests = @handlers.map { |handler| session.build_request(*build_request(handler.env)) }
+            if env.request.stream_response?
+              requests.each do |request|
+                request.response_on_data = env.request.on_data
+              end
+            end
 
-          if env.request.stream_response?
-            requests.each do |request|
-              request.response_on_data = env.request.on_data
+            responses = session.request(*requests)
+            Array(responses).each_with_index do |response, index|
+              handler = @handlers[index]
+              handler.on_response.call(response)
+              handler.on_complete.call(handler.env) if handler.on_complete
             end
           end
+        rescue ::HTTPX::TimeoutError => e
+          raise Faraday::TimeoutError, e
+        end
 
-          responses = session.request(*requests)
-          Array(responses).each_with_index do |response, index|
-            handler = @handlers[index]
-            handler.on_response.call(response)
-            handler.on_complete.call(handler.env)
-          end
+        # from Faraday::Adapter#connection
+        def connection(env)
+          conn = build_connection(env)
+          return conn unless block_given?
+
+          yield conn
+        end
+
+        private
+
+        # from Faraday::Adapter#request_timeout
+        def request_timeout(type, options)
+          key = Faraday::Adapter::TIMEOUT_KEYS[type]
+          options[key] || options[:timeout]
         end
       end
 
       self.supports_parallel = true
 
       class << self
-        def setup_parallel_manager
-          ParallelManager.new
+        def setup_parallel_manager(options = {})
+          ParallelManager.new(options)
         end
       end
 
-      def initialize(app, options = {})
-        super(app)
-        @session_options = options
-      end
-
       def call(env)
         super
         if parallel?(env)
           handler = env[:parallel_manager].enqueue(env)
           handler.on_response do |response|
-            response.raise_for_status
-            save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
-              response_headers.merge!(response.headers)
+            if response.is_a?(::HTTPX::Response)
+              save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
+                response_headers.merge!(response.headers)
+              end
+            else
+              env[:error] = response.error
+              save_response(env, 0, "", {}, nil)
             end
           end
           return handler
         end
 
-        session = HTTPX.session
-        session = session.with(@session_options) unless @session_options.empty?
-        session = session.with(options_from_env(env))
-        session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
-        session = session.plugin(OnDataPlugin) if env.request.stream_response?
-
-        request = session.build_request(*build_request(env))
-
-        request.response_on_data = env.request.on_data if env.request.stream_response?
-
-        response = session.request(request)
-        response.raise_for_status unless response.is_a?(::HTTPX::Response)
+        response = connect_and_request(env)
         save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
           response_headers.merge!(response.headers)
         end
         @app.call(env)
-      rescue ::HTTPX::TLSError => e
-        raise SSL_ERROR, e
-      rescue Errno::ECONNABORTED,
-             Errno::ECONNREFUSED,
-             Errno::ECONNRESET,
-             Errno::EHOSTUNREACH,
-             Errno::EINVAL,
-             Errno::ENETUNREACH,
-             Errno::EPIPE => e
-        raise CONNECTION_FAILED_ERROR, e
       end
 
       private
 
+      def connect_and_request(env)
+        connect(env) do |session|
+          request = session.build_request(*build_request(env))
+
+          request.response_on_data = env.request.on_data if env.request.stream_response?
+
+          response = session.request(request)
+          # do not call #raise_for_status for HTTP 4xx or 5xx, as faraday has a middleware for that.
+          response.raise_for_status unless response.is_a?(::HTTPX::Response)
+          response
+        end
+      rescue ::HTTPX::TimeoutError => e
+        raise Faraday::TimeoutError, e
+      end
+
       def parallel?(env)
         env[:parallel_manager]
       end

data/lib/httpx/adapters/sentry.rb

@@ -27,6 +27,11 @@ module HTTPX::Plugins
       def set_sentry_trace_header(request, sentry_span)
         return unless sentry_span
 
+        config = ::Sentry.configuration
+        url = request.uri.to_s
+
+        return unless config.propagate_traces && config.trace_propagation_targets.any? { |target| url.match?(target) }
+
         trace = ::Sentry.get_current_client.generate_sentry_trace(sentry_span)
         request.headers[::Sentry::SENTRY_TRACE_HEADER_NAME] = trace if trace
       end
@@ -43,8 +48,8 @@ module HTTPX::Plugins
 
         request_info = extract_request_info(req)
 
-        data = if response.is_a?(HTTPX::ErrorResponse)
-          { error: res.message, **request_info }
+        data = if res.is_a?(HTTPX::ErrorResponse)
+          { error: res.error.message, **request_info }
         else
           { status: res.status, **request_info }
         end
@@ -63,7 +68,11 @@ module HTTPX::Plugins
 
         request_info = extract_request_info(req)
         sentry_span.set_description("#{request_info[:method]} #{request_info[:url]}")
-        sentry_span.set_data(:status, res.status)
+        if res.is_a?(HTTPX::ErrorResponse)
+          sentry_span.set_data(:error, res.error.message)
+        else
+          sentry_span.set_data(:status, res.status)
+        end
         sentry_span.set_timestamp(::Sentry.utc_now.to_f)
       end
 
@@ -71,7 +80,7 @@ module HTTPX::Plugins
         uri = req.uri
 
         result = {
-          method: req.verb.to_s.upcase,
+          method: req.verb,
         }
 
         if ::Sentry.configuration.send_default_pii
@@ -85,17 +94,27 @@ module HTTPX::Plugins
       end
     end
 
+    module RequestMethods
+      def __sentry_enable_trace!
+        return if @__sentry_enable_trace
+
+        Tracer.call(self)
+        @__sentry_enable_trace = true
+      end
+    end
+
     module ConnectionMethods
       def send(request)
-        Tracer.call(request)
+        request.__sentry_enable_trace!
+
         super
       end
     end
   end
 end
 
-Sentry.register_patch do
-  sentry_session = ::HTTPX.plugin(HTTPX::Plugins::Sentry)
+Sentry.register_patch(:httpx) do
+  sentry_session = HTTPX.plugin(HTTPX::Plugins::Sentry)
 
   HTTPX.send(:remove_const, :Session)
   HTTPX.send(:const_set, :Session, sentry_session.class)

data/lib/httpx/adapters/webmock.rb

@@ -2,13 +2,8 @@
 
 module WebMock
   module HttpLibAdapters
-    if RUBY_VERSION < "2.5"
-      require "webrick/httpstatus"
-      HTTP_REASONS = WEBrick::HTTPStatus::StatusMessage
-    else
-      require "net/http/status"
-      HTTP_REASONS = Net::HTTP::STATUS_CODES
-    end
+    require "net/http/status"
+    HTTP_REASONS = Net::HTTP::STATUS_CODES
 
     #
     # HTTPX plugin for webmock.
@@ -23,7 +18,7 @@ module WebMock
           uri.path = uri.normalized_path.gsub("[^:]//", "/")
 
           WebMock::RequestSignature.new(
-            request.verb,
+            request.verb.downcase.to_sym,
             uri.to_s,
             body: request.body.each.to_a.join,
             headers: request.headers.to_h
@@ -43,30 +38,50 @@ module WebMock
 
           return build_error_response(request, webmock_response.exception) if webmock_response.exception
 
-          response = request.options.response_class.new(request,
-                                                        webmock_response.status[0],
-                                                        "2.0",
-                                                        webmock_response.headers)
-          response << webmock_response.body.dup
-          response
+          request.options.response_class.new(request,
+                                             webmock_response.status[0],
+                                             "2.0",
+                                             webmock_response.headers).tap do |res|
+            res.mocked = true
+          end
         end
 
         def build_error_response(request, exception)
-          HTTPX::ErrorResponse.new(request, exception, request.options)
+          HTTPX::ErrorResponse.new(request, exception)
         end
       end
 
       module InstanceMethods
-        def build_connection(*)
+        def init_connection(*)
           connection = super
           connection.once(:unmock_connection) do
+            unless connection.addresses
+              connection.__send__(:callbacks)[:connect_error].clear
+              pool.__send__(:unregister_connection, connection)
+            end
             pool.__send__(:resolve_connection, connection)
-            pool.__send__(:unregister_connection, connection) unless connection.addresses
           end
           connection
         end
       end
 
+      module ResponseMethods
+        attr_accessor :mocked
+
+        def initialize(*)
+          super
+          @mocked = false
+        end
+      end
+
+      module ResponseBodyMethods
+        def decode_chunk(chunk)
+          return chunk if @response.mocked
+
+          super
+        end
+      end
+
       module ConnectionMethods
         def initialize(*)
           super
@@ -95,6 +110,7 @@ module WebMock
             log { "mocking #{request.uri} with #{mock_response.inspect}" }
             request.response = response
             request.emit(:response, response)
+            response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)
           elsif WebMock.net_connect_allowed?(request_signature.uri)
             if WebMock::CallbackRegistry.any_callbacks?
               request.on(:response) do |resp|
@@ -122,7 +138,7 @@ module WebMock
 
       class << self
         def enable!
-          @original_session = HTTPX::Session
+          @original_session ||= HTTPX::Session
 
           webmock_session = HTTPX.plugin(Plugin)
 

data/lib/httpx/altsvc.rb

@@ -4,7 +4,59 @@ require "strscan"
 
 module HTTPX
   module AltSvc
-    @altsvc_mutex = Mutex.new
+    # makes connections able to accept requests destined to primary service.
+    module ConnectionMixin
+      using URIExtensions
+
+      def send(request)
+        request.headers["alt-used"] = @origin.authority if @parser && !@write_buffer.full? && match_altsvcs?(request.uri)
+
+        super
+      end
+
+      def match?(uri, options)
+        return false if !used? && (@state == :closing || @state == :closed)
+
+        match_altsvcs?(uri) && match_altsvc_options?(uri, options)
+      end
+
+      private
+
+      # checks if this is connection is an alternative service of
+      # +uri+
+      def match_altsvcs?(uri)
+        @origins.any? { |origin| altsvc_match?(uri, origin) } ||
+          AltSvc.cached_altsvc(@origin).any? do |altsvc|
+            origin = altsvc["origin"]
+            altsvc_match?(origin, uri.origin)
+          end
+      end
+
+      def match_altsvc_options?(uri, options)
+        return @options == options unless @options.ssl.all? do |k, v|
+          v == (k == :hostname ? uri.host : options.ssl[k])
+        end
+
+        @options.options_equals?(options, Options::REQUEST_BODY_IVARS + %i[@ssl])
+      end
+
+      def altsvc_match?(uri, other_uri)
+        other_uri = URI(other_uri)
+
+        uri.origin == other_uri.origin || begin
+          case uri.scheme
+          when "h2"
+            (other_uri.scheme == "https" || other_uri.scheme == "h2") &&
+              uri.host == other_uri.host &&
+              uri.port == other_uri.port
+          else
+            false
+          end
+        end
+      end
+    end
+
+    @altsvc_mutex = Thread::Mutex.new
     @altsvcs = Hash.new { |h, k| h[k] = [] }
 
     module_function
@@ -46,7 +98,7 @@ module HTTPX
 
       altsvc = response.headers["alt-svc"]
 
-      # https://tools.ietf.org/html/rfc7838#section-3
+      # https://datatracker.ietf.org/doc/html/rfc7838#section-3
       # A field value containing the special value "clear" indicates that the
       # origin requests all alternatives for that origin to be invalidated
       # (including those specified in the same response, in case of an
@@ -79,9 +131,9 @@ module HTTPX
           scanner.skip(/;/)
           break if scanner.eos? || scanner.scan(/ *, */)
         end
-        alt_params = Hash[alt_params.map { |field| field.split("=") }]
+        alt_params = Hash[alt_params.map { |field| field.split("=", 2) }]
 
-        alt_proto, alt_authority = alt_service.split("=")
+        alt_proto, alt_authority = alt_service.split("=", 2)
         alt_origin = parse_altsvc_origin(alt_proto, alt_authority)
         return unless alt_origin
 
@@ -98,29 +150,14 @@ module HTTPX
       end
     end
 
-    # :nocov:
-    if RUBY_VERSION < "2.2"
-      def parse_altsvc_origin(alt_proto, alt_origin)
-        alt_scheme = parse_altsvc_scheme(alt_proto) or return
-
-        alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
-        if alt_origin.start_with?(":")
-          alt_origin = "#{alt_scheme}://dummy#{alt_origin}"
-          uri = URI.parse(alt_origin)
-          uri.host = nil
-          uri
-        else
-          URI.parse("#{alt_scheme}://#{alt_origin}")
-        end
-      end
-    else
-      def parse_altsvc_origin(alt_proto, alt_origin)
-        alt_scheme = parse_altsvc_scheme(alt_proto) or return
-        alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
+    def parse_altsvc_origin(alt_proto, alt_origin)
+      alt_scheme = parse_altsvc_scheme(alt_proto)
 
-        URI.parse("#{alt_scheme}://#{alt_origin}")
-      end
+      return unless alt_scheme
+
+      alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
+
+      URI.parse("#{alt_scheme}://#{alt_origin}")
     end
-    # :nocov:
   end
 end

data/lib/httpx/base64.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+if RUBY_VERSION < "3.3.0"
+  require "base64"
+elsif !defined?(Base64)
+  module HTTPX
+    # require "base64" will not be a default gem after ruby 3.4.0
+    module Base64
+      module_function
+
+      def decode64(str)
+        str.unpack1("m")
+      end
+
+      def strict_encode64(bin)
+        [bin].pack("m0")
+      end
+
+      def urlsafe_encode64(bin, padding: true)
+        str = strict_encode64(bin)
+        str.chomp!("==") or str.chomp!("=") unless padding
+        str.tr!("+/", "-_")
+        str
+      end
+    end
+  end
+end

data/lib/httpx/buffer.rb

@@ -3,6 +3,14 @@
 require "forwardable"
 
 module HTTPX
+  # Internal class to abstract a string buffer, by wrapping a string and providing the
+  # minimum possible API and functionality required.
+  #
+  #     buffer = Buffer.new(640)
+  #     buffer.full? #=> false
+  #     buffer << "aa"
+  #     buffer.capacity #=> 638
+  #
   class Buffer
     extend Forwardable
 
@@ -31,6 +39,10 @@ module HTTPX
       @buffer.bytesize >= @limit
     end
 
+    def capacity
+      @limit - @buffer.bytesize
+    end
+
     def shift!(fin)
       @buffer = @buffer.byteslice(fin..-1) || "".b
     end