httpx 1.6.2 → 1.7.0

data/lib/httpx/plugins/proxy/http.rb

@@ -47,6 +47,17 @@ module HTTPX
             super || @state == :connecting || @state == :connected
           end
 
+          def force_close(*)
+            if @state == :connecting
+              # proxy connect related requests should not be reenqueed
+              @parser.reset!
+              @inflight -= @parser.pending.size
+              @parser.pending.clear
+            end
+
+            super
+          end
+
           private
 
           def handle_transition(nextstate)
@@ -64,23 +75,40 @@ module HTTPX
                 parser = @parser
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
-                parser.on(:close) do |force|
+                parser.on(:close) do
                   next unless @parser
 
-                  if force
-                    reset
-                    emit(:terminate)
-                  end
+                  reset
+                  disconnect
                 end
                 parser.on(:reset) do
                   if parser.empty?
                     reset
                   else
-                    transition(:closing)
-                    transition(:closed)
+                    enqueue_pending_requests_from_parser(parser)
+
+                    initial_state = @state
+
+                    reset
+
+                    if @pending.empty?
+                      @parser = nil
+                      next
+                    end
+                    # keep parser state around due to proxy auth protocol;
+                    # intermediate authenticated request is already inside
+                    # the parser
+                    parser = nil
+
+                    if initial_state == :connecting
+                      parser = @parser
+                      @parser.reset
+                    end
+
+                    idling
+
+                    @parser = parser
 
-                    parser.reset if @parser
-                    transition(:idle)
                     transition(:connecting)
                   end
                 end

data/lib/httpx/plugins/rate_limiter.rb

@@ -18,11 +18,11 @@ module HTTPX
         def configure(klass)
           klass.plugin(:retries,
                        retry_change_requests: true,
-                       retry_on: method(:retry_on_rate_limited_response),
+                       retry_on: method(:retry_on_rate_limited_response?),
                        retry_after: method(:retry_after_rate_limit))
         end
 
-        def retry_on_rate_limited_response(response)
+        def retry_on_rate_limited_response?(response)
           return false unless response.is_a?(Response)
 
           status = response.status

data/lib/httpx/plugins/response_cache/file_store.rb

@@ -130,6 +130,7 @@ module HTTPX::Plugins
         response = request.options.response_class.new(request, status, version, response_headers)
         response.original_request = original_request
         response.finish!
+        response.mark_as_cached!
 
         IO.copy_stream(f, response.body)
 

data/lib/httpx/plugins/response_cache.rb

@@ -118,7 +118,10 @@ module HTTPX
 
             response.copy_from_cached!
           elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
-            request.options.response_cache_store.set(request, response) unless response.cached?
+            unless response.cached?
+              log { "caching response for #{request.uri}..." }
+              request.options.response_cache_store.set(request, response)
+            end
           end
 
           response
@@ -147,7 +150,7 @@ module HTTPX
             request.headers.add("if-modified-since", last_modified)
           end
 
-          if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
+          if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"])
             request.headers.add("if-none-match", etag)
           end
         end
@@ -204,7 +207,7 @@ module HTTPX
         # returns a unique cache key as a String identifying this request
         def response_cache_key
           @response_cache_key ||= begin
-            keys = [@verb, @uri]
+            keys = [@verb, @uri.merge(path)]
 
             @options.supported_vary_headers.each do |field|
               value = @headers[field]
@@ -293,9 +296,7 @@ module HTTPX
           return @cache_control if defined?(@cache_control)
 
           @cache_control = begin
-            return unless @headers.key?("cache-control")
-
-            @headers["cache-control"].split(/ *, */)
+            @headers["cache-control"].split(/ *, */) if @headers.key?("cache-control")
           end
         end
 
@@ -304,9 +305,7 @@ module HTTPX
           return @vary if defined?(@vary)
 
           @vary = begin
-            return unless @headers.key?("vary")
-
-            @headers["vary"].split(/ *, */).map(&:downcase)
+            @headers["vary"].split(/ *, */).map(&:downcase) if @headers.key?("vary")
           end
         end
 
@@ -327,6 +326,14 @@ module HTTPX
           Time.now
         end
       end
+
+      module ResponseBodyMethods
+        def decode_chunk(chunk)
+          return chunk if @response.cached?
+
+          super
+        end
+      end
     end
     register_plugin :response_cache, ResponseCache
   end

data/lib/httpx/plugins/retries.rb

@@ -36,15 +36,33 @@ module HTTPX
         Parser::Error,
         TimeoutError,
       ]).freeze
-      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
 
-      if ENV.key?("HTTPX_NO_JITTER")
-        def self.extra_options(options)
-          options.merge(max_retries: MAX_RETRIES)
+      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }.freeze
+
+      # list of supported backoff algorithms
+      BACKOFF_ALGORITHMS = %i[exponential_backoff polynomial_backoff].freeze
+
+      class << self
+        if ENV.key?("HTTPX_NO_JITTER")
+          def extra_options(options)
+            options.merge(max_retries: MAX_RETRIES)
+          end
+        else
+          def extra_options(options)
+            options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+          end
+        end
+
+        # returns the time to wait before resending +request+ as per the polynomial backoff retry strategy.
+        def retry_after_polynomial_backoff(request, _)
+          offset = request.options.max_retries - request.retries
+          2 * (offset - 1)
         end
-      else
-        def self.extra_options(options)
-          options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+
+        # returns the time to wait before resending +request+ as per the exponential backoff retry strategy.
+        def retry_after_exponential_backoff(request, _)
+          offset = request.options.max_retries - request.retries
+          (offset - 1) * 2
         end
       end
 
@@ -53,6 +71,7 @@ module HTTPX
       # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
       # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
       # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
+      #                or the name of a supported backoff algorithm (i.e. <tt>:exponential_backoff</tt>).
       # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
       # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
       #              (i.e. <tt>->(res) { ... }</tt>).
@@ -60,10 +79,26 @@ module HTTPX
         private
 
         def option_retry_after(value)
-          # return early if callable
-          unless value.respond_to?(:call)
-            value = Float(value)
-            raise TypeError, ":retry_after must be positive" unless value.positive?
+          if value.respond_to?(:call)
+            value1 = value
+            value1 = value1.method(:call) unless value1.respond_to?(:arity)
+
+            # allow ->(*) arity as well, which is < 0
+            raise TypeError, "`:retry_after` proc has invalid number of parameters" unless value1.arity.negative? || value1.arity.between?(
+              1, 2
+            )
+
+          else
+            case value
+            when Symbol
+              raise TypeError, "`retry_after`: `#{value}` is not a supported backoff algorithm" unless BACKOFF_ALGORITHMS.include?(value)
+
+              value = Retries.method(:"retry_after_#{value}")
+
+            else
+              value = Float(value)
+              raise TypeError, "`:retry_after` must be positive" unless value.positive?
+            end
           end
 
           value
@@ -112,14 +147,13 @@ module HTTPX
                (
                  response.is_a?(ErrorResponse) && retryable_error?(response.error)
                ) ||
-               (
-                 options.retry_on && options.retry_on.call(response)
-               )
+
+                 options.retry_on&.call(response)
+
              )
             try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
-            request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
-            request.transition(:idle)
+            prepare_to_retry(request, response)
 
             retry_after = options.retry_after
             retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
@@ -165,6 +199,11 @@ module HTTPX
           super && !request.retries.positive?
         end
 
+        def prepare_to_retry(request, _response)
+          request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
+          request.transition(:idle)
+        end
+
         #
         # Attempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and

data/lib/httpx/plugins/ssrf_filter.rb

@@ -16,7 +16,7 @@ module HTTPX
             mask_addr = @mask_addr
             raise "Invalid mask" if mask_addr.zero?
 
-            mask_addr >>= 1 while (mask_addr & 0x1).zero?
+            mask_addr >>= 1 while mask_addr.nobits?(0x1)
 
             length = 0
             while mask_addr & 0x1 == 0x1

data/lib/httpx/plugins/stream.rb

@@ -55,9 +55,9 @@ module HTTPX
         line << chunk
 
         while (idx = line.index("\n"))
-          yield line.byteslice(0..idx - 1)
+          yield line.byteslice(0..(idx - 1))
 
-          line = line.byteslice(idx + 1..-1)
+          line = line.byteslice((idx + 1)..-1)
         end
       end
 
@@ -121,20 +121,71 @@ module HTTPX
     # https://gitlab.com/os85/httpx/wikis/Stream
     #
     module Stream
+      STREAM_REQUEST_OPTIONS = { timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 }.freeze }.freeze
+
       def self.extra_options(options)
-        options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
+        options.merge(
+          stream: false,
+          timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 },
+          stream_response_class: Class.new(StreamResponse, &Options::SET_TEMPORARY_NAME).freeze
+        )
+      end
+
+      # adds support for the following options:
+      #
+      # :stream :: whether the request to process should be handled as a stream (defaults to <tt>false</tt>).
+      # :stream_response_class :: Class used to build the stream response object.
+      module OptionsMethods
+        def option_stream(val)
+          val
+        end
+
+        def option_stream_response_class(value)
+          value
+        end
+
+        def extend_with_plugin_classes(pl)
+          return super unless defined?(pl::StreamResponseMethods)
+
+          @stream_response_class = @stream_response_class.dup
+          Options::SET_TEMPORARY_NAME[@stream_response_class, pl]
+          @stream_response_class.__send__(:include, pl::StreamResponseMethods) if defined?(pl::StreamResponseMethods)
+
+          super
+        end
       end
 
       module InstanceMethods
-        def request(*args, stream: false, **options)
-          return super(*args, **options) unless stream
+        def request(*args, **options)
+          if args.first.is_a?(Request)
+            requests = args
+
+            request = requests.first
+
+            unless request.options.stream && !request.stream
+              if options[:stream]
+                warn "passing `stream: true` with a request obkect is not supported anymore. " \
+                     "You can instead build the request object with `stream :true`"
+              end
+              return super
+            end
+          else
+            return super unless options[:stream]
+
+            requests = build_requests(*args, options)
+
+            request = requests.first
+          end
 
-          requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
           raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
 
-          request = requests.first
+          @options.stream_response_class.new(request, self)
+        end
+
+        def build_request(verb, uri, params = EMPTY_HASH, options = @options)
+          return super unless params[:stream]
 
-          StreamResponse.new(request, self)
+          super(verb, uri, params, options.merge(STREAM_REQUEST_OPTIONS.merge(stream: true)))
         end
       end
 

data/lib/httpx/plugins/stream_bidi.rb

@@ -16,7 +16,7 @@ module HTTPX
       # The streams keeps send DATA frames while there's data; when they're ain't,
       # the stream is kept open; it must be explicitly closed by the end user.
       #
-      class HTTP2Bidi < Connection::HTTP2
+      module HTTP2Methods
         def initialize(*)
           super
           @lock = Thread::Mutex.new
@@ -26,6 +26,8 @@ module HTTPX
           class_eval(<<-METH, __FILE__, __LINE__ + 1)
             # lock.aware version of +#{lock_meth}+
             def #{lock_meth}(*)                # def close(*)
+              return super unless @options.stream
+
               return super if @lock.owned?
 
               # small race condition between
@@ -43,6 +45,8 @@ module HTTPX
           class_eval(<<-METH, __FILE__, __LINE__ + 1)
             # lock.aware version of +#{lock_meth}+
             private def #{lock_meth}(*)                # private def join_headers(*)
+              return super unless @options.stream
+
               return super if @lock.owned?
 
               # small race condition between
@@ -55,6 +59,8 @@ module HTTPX
         end
 
         def handle_stream(stream, request)
+          return super unless @options.stream
+
           request.on(:body) do
             next unless request.headers_sent
 
@@ -67,6 +73,8 @@ module HTTPX
 
         # when there ain't more chunks, it makes the buffer as full.
         def send_chunk(request, stream, chunk, next_chunk)
+          return super unless @options.stream
+
           super
 
           return if next_chunk
@@ -77,39 +85,60 @@ module HTTPX
 
         # sets end-stream flag when the request is closed.
         def end_stream?(request, next_chunk)
+          return super unless @options.stream
+
           request.closed? && next_chunk.nil?
         end
       end
 
-      # BidiBuffer is a Buffer which can be receive data from threads othr
-      # than the thread of the corresponding Connection/Session.
+      # BidiBuffer is a thread-safe Buffer which can receive data from any thread.
       #
-      # It synchronizes access to a secondary internal +@oob_buffer+, which periodically
-      # is reconciled to the main internal +@buffer+.
+      # It uses a dual-buffer strategy with mutex protection:
+      # - +@buffer+ is the main buffer, protected by +@buffer_mutex+
+      # - +@oob_buffer+ receives data when +@buffer_mutex+ is contended
+      #
+      # This allows non-blocking writes from any thread while maintaining thread safety.
       class BidiBuffer < Buffer
         def initialize(*)
           super
-          @parent_thread = Thread.current
+          @buffer_mutex = Thread::Mutex.new
           @oob_mutex = Thread::Mutex.new
           @oob_buffer = "".b
         end
 
-        # buffers the +chunk+ to be sent
+        # buffers the +chunk+ to be sent (thread-safe, non-blocking)
         def <<(chunk)
-          return super if Thread.current == @parent_thread
-
-          @oob_mutex.synchronize { @oob_buffer << chunk }
+          if @buffer_mutex.try_lock
+            begin
+              super
+            ensure
+              @buffer_mutex.unlock
+            end
+          else
+            # another thread holds the lock, use OOB buffer to avoid blocking
+            @oob_mutex.synchronize { @oob_buffer << chunk }
+          end
         end
 
-        # reconciles the main and secondary buffer (which receives data from other threads).
+        # reconciles the main and secondary buffer (thread-safe, callable from any thread).
         def rebuffer
-          raise Error, "can only rebuffer while waiting on a response" unless Thread.current == @parent_thread
+          @buffer_mutex.synchronize do
+            @oob_mutex.synchronize do
+              return if @oob_buffer.empty?
 
-          @oob_mutex.synchronize do
-            @buffer << @oob_buffer
-            @oob_buffer.clear
+              @buffer << @oob_buffer
+              @oob_buffer.clear
+            end
           end
         end
+
+        Buffer.instance_methods - Object.instance_methods - %i[<<].each do |meth|
+          class_eval(<<-MOD, __FILE__, __LINE__ + 1)
+            def #{meth}(*) # def empty?
+              @buffer_mutex.synchronize { super }
+            end
+          MOD
+        end
       end
 
       # Proxy to wake up the session main loop when one
@@ -117,8 +146,11 @@ module HTTPX
       # which allows it to be registered in the selector alongside actual HTTP-based
       # HTTPX::Connection objects.
       class Signal
+        attr_reader :error
+
         def initialize
           @closed = false
+          @error = nil
           @pipe_read, @pipe_write = IO.pipe
         end
 
@@ -153,12 +185,23 @@ module HTTPX
 
         def timeout; end
 
+        def inflight?
+          !@closed
+        end
+
         def terminate
+          return if @closed
+
           @pipe_write.close
           @pipe_read.close
           @closed = true
         end
 
+        def on_error(error)
+          @error = error
+          terminate
+        end
+
         # noop (the owner connection will take of it)
         def handle_socket_timeout(interval); end
       end
@@ -182,17 +225,22 @@ module HTTPX
         def close(selector = Selector.new)
           @signal.terminate
           selector.deregister(@signal)
-          super(selector)
+          super
         end
 
         def select_connection(connection, selector)
+          return super unless connection.options.stream
+
           super
           selector.register(@signal)
           connection.signal = @signal
         end
 
         def deselect_connection(connection, *)
+          return super unless connection.options.stream
+
           super
+
           connection.signal = nil
         end
       end
@@ -210,10 +258,14 @@ module HTTPX
         end
 
         def closed?
+          return super unless @options.stream
+
           @closed
         end
 
         def can_buffer?
+          return super unless @options.stream
+
           super && @state != :waiting_for_chunk
         end
 
@@ -221,6 +273,8 @@ module HTTPX
         # +:waiting_for_chunk+ state, which the request transitions to once payload
         # is buffered.
         def transition(nextstate)
+          return super unless @options.stream
+
           headers_sent = @headers_sent
 
           case nextstate
@@ -255,6 +309,8 @@ module HTTPX
         end
 
         def close
+          return super unless @options.stream
+
           @mutex.synchronize do
             return if @closed
 
@@ -269,10 +325,12 @@ module HTTPX
       module RequestBodyMethods
         def initialize(*, **)
           super
-          @headers.delete("content-length")
+          @headers.delete("content-length") if @options.stream
         end
 
         def empty?
+          return super unless @options.stream
+
           false
         end
       end
@@ -284,25 +342,32 @@ module HTTPX
 
         def initialize(*)
           super
+
+          return unless @options.stream
+
           @write_buffer = BidiBuffer.new(@options.buffer_size)
         end
 
         # rebuffers the +@write_buffer+ before calculating interests.
         def interests
+          return super unless @options.stream
+
           @write_buffer.rebuffer
 
           super
         end
 
-        private
-
-        def parser_type(protocol)
-          return HTTP2Bidi if protocol == "h2"
+        def call
+          return super unless @options.stream && (error = @signal.error)
 
-          super
+          on_error(error)
         end
 
+        private
+
         def set_parser_callbacks(parser)
+          return super unless @options.stream
+
           super
           parser.on(:flush_buffer) do
             @signal.wakeup if @signal