httpx 1.7.7 → 1.8.0

data/lib/httpx/plugins/response_cache.rb

@@ -3,7 +3,8 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds support for retrying requests when certain errors happen.
+    # This plugin caches and reuses responses based on HTTP caching directives defined by
+    # the [HTTP Caching RFC](https://www.rfc-editor.org/rfc/rfc9111.html)
     #
     # https://gitlab.com/os85/httpx/wikis/Response-Cache
     #
@@ -15,9 +16,8 @@ module HTTPX
       private_constant :CACHEABLE_STATUS_CODES
 
       class << self
-        def load_dependencies(*)
-          require_relative "response_cache/store"
-          require_relative "response_cache/file_store"
+        def load_dependencies(klass)
+          klass.plugin(:cache)
         end
 
         # whether the +response+ can be stored in the response cache.
@@ -47,7 +47,6 @@ module HTTPX
         def extra_options(options)
           options.merge(
             supported_vary_headers: SUPPORTED_VARY_HEADERS,
-            response_cache_store: :store,
           )
         end
       end
@@ -56,58 +55,18 @@ module HTTPX
       #
       # :supported_vary_headers :: array of header values that will be considered for a "vary" header based cache validation
       #                            (defaults to {SUPPORTED_VARY_HEADERS}).
-      # :response_cache_store :: object where cached responses are fetch from or stored in; defaults to <tt>:store</tt> (in-memory
-      #                          cache), can be set to <tt>:file_store</tt> (file system cache store) as well, or any object which
-      #                          abides by the Cache Store Interface
-      #
-      # The Cache Store Interface requires implementation of the following methods:
-      #
-      # * +#get(request) -> response or nil+
-      # * +#set(request, response) -> void+
-      # * +#clear() -> void+)
       #
       module OptionsMethods
         private
 
-        def option_response_cache_store(value)
-          case value
-          when :store
-            Store.new
-          when :file_store
-            FileStore.new
-          else
-            value
-          end
-        end
-
         def option_supported_vary_headers(value)
           Array(value).sort
         end
       end
 
       module InstanceMethods
-        # wipes out all cached responses from the cache store.
-        def clear_response_cache
-          @options.response_cache_store.clear
-        end
-
-        def build_request(*)
-          request = super
-          return request unless cacheable_request?(request)
-
-          prepare_cache(request)
-
-          request
-        end
-
         private
 
-        def send_request(request, *)
-          return request if request.response
-
-          super
-        end
-
         def fetch_response(request, *)
           response = super
 
@@ -117,11 +76,7 @@ module HTTPX
             log { "returning cached response for #{request.uri}" }
 
             response.copy_from_cached!
-          elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
-            unless response.cached?
-              log { "caching response for #{request.uri}..." }
-              request.options.response_cache_store.set(request, response)
-            end
+
           end
 
           response
@@ -130,21 +85,13 @@ module HTTPX
         # will either assign a still-fresh cached response to +request+, or set up its HTTP
         # cache invalidation headers in case it's not fresh anymore.
         def prepare_cache(request)
-          cached_response = request.options.response_cache_store.get(request)
+          super
 
-          return unless cached_response && match_by_vary?(request, cached_response)
+          return if request.response # already cached
 
-          cached_response.body.rewind
+          cached_response = retrieve_cached_response(request)
 
-          if cached_response.fresh?
-            cached_response = cached_response.dup
-            cached_response.mark_as_cached!
-            request.response = cached_response
-            request.emit_response(cached_response)
-            return
-          end
-
-          request.cached_response = cached_response
+          return unless cached_response && match_by_vary?(request, cached_response)
 
           if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
             request.headers.add("if-modified-since", last_modified)
@@ -156,20 +103,31 @@ module HTTPX
         end
 
         def cacheable_request?(request)
-          request.cacheable_verb? &&
+          (
+            request.cacheable_verb? &&
             (
               !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
             )
+          ) || super
         end
 
-        # whether the +response+ complies with the directives set by the +request+ "vary" header
+        def cacheable_response?(_, response)
+          ResponseCache.cacheable_response?(response) || super
+        end
+
+        # +cached_response+ is still valid if it's still fresh
+        def valid_cached_response?(_, cached_response)
+          cached_response.fresh?
+        end
+
+        # whether the +cached_response+ complies with the directives set by the +request+ "vary" header
         # (true when none is available).
-        def match_by_vary?(request, response)
-          vary = response.vary
+        def match_by_vary?(request, cached_response)
+          vary = cached_response.vary
 
           return true unless vary
 
-          original_request = response.original_request
+          original_request = cached_response.original_request
 
           if vary == %w[*]
             request.options.supported_vary_headers.each do |field|
@@ -186,19 +144,6 @@ module HTTPX
       end
 
       module RequestMethods
-        # points to a previously cached Response corresponding to this request.
-        attr_accessor :cached_response
-
-        def initialize(*)
-          super
-          @cached_response = nil
-        end
-
-        def merge_headers(*)
-          super
-          @response_cache_key = nil
-        end
-
         # returns whether this request is cacheable as per HTTP caching rules.
         def cacheable_verb?
           CACHEABLE_VERBS.include?(@verb)
@@ -220,29 +165,13 @@ module HTTPX
       end
 
       module ResponseMethods
-        attr_writer :original_request, :revalidated_at
+        attr_writer :revalidated_at
 
         def initialize(*)
           super
-          @cached = false
           @revalidated_at = nil
         end
 
-        # a copy of the request this response was originally cached from
-        def original_request
-          @original_request || @request
-        end
-
-        # whether this Response was duplicated from a previously {RequestMethods#cached_response}.
-        def cached?
-          @cached
-        end
-
-        # sets this Response as being duplicated from a previously cached response.
-        def mark_as_cached!
-          @cached = true
-        end
-
         # eager-copies the response headers and body from {RequestMethods#cached_response}.
         def copy_from_cached!
           cached_response = @request.cached_response
@@ -333,14 +262,6 @@ module HTTPX
           Time.now
         end
       end
-
-      module ResponseBodyMethods
-        def decode_chunk(chunk)
-          return chunk if @response.cached?
-
-          super
-        end
-      end
     end
     register_plugin :response_cache, ResponseCache
   end

data/lib/httpx/plugins/retries.rb

@@ -53,16 +53,18 @@ module HTTPX
           end
         end
 
-        # returns the time to wait before resending +request+ as per the polynomial backoff retry strategy.
+        # returns the time to wait before resending +request+ as per the polynomial backoff retry strategy,
+        # where base is 1 and exponent is 2.
         def retry_after_polynomial_backoff(request, _)
           offset = request.options.max_retries - request.retries
-          2 * (offset - 1)
+          1 * ((offset - 1)**2)
         end
 
-        # returns the time to wait before resending +request+ as per the exponential backoff retry strategy.
+        # returns the time to wait before resending +request+ as per the exponential backoff retry strategy,
+        # where base is 2
         def retry_after_exponential_backoff(request, _)
           offset = request.options.max_retries - request.retries
-          (offset - 1) * 2
+          2**(offset - 1)
         end
       end
 
@@ -186,7 +188,7 @@ module HTTPX
 
         # returns whether the +ex+ exception happend for a retriable request.
         def retryable_error?(ex, _)
-          RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
+          RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) } && !ex.is_a?(TotalRequestTimeoutError)
         end
 
         def proxy_error?(request, response, _)
@@ -200,7 +202,13 @@ module HTTPX
 
         def when_to_retry(request, response, options)
           retry_after = options.retry_after
+
+          return unless retry_after
+
           retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+
+          return unless retry_after
+
           # apply jitter
           if (jitter = request.options.retry_jitter)
             retry_after = jitter.call(retry_after)

data/lib/httpx/plugins/server_sent_events.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements convenience methods for Server Sent Events streams.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Server-Sent-Events
+    #
+    module ServerSentEvents
+      Message = if RUBY_VERSION >= "3.2.0" # rubocop:disable Naming/ConstantName
+        Data.define(:data, :event, :id, :retry_after) do
+          def initialize(event: nil, id: nil, retry_after: nil, **kwargs)
+            super
+          end
+        end
+      else
+        Struct.new(:data, :event, :id, :retry_after, keyword_init: true)
+      end
+
+      class << self
+        def subplugins
+          {
+            retries: ServerSentEventsRetries,
+          }
+        end
+
+        def load_dependencies(klass)
+          klass.plugin(:stream)
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :event_stream :: whether the request is a server-sent events text event stream (defaults to <tt>false</tt>).
+      module OptionsMethods
+        def option_event_stream(val)
+          val
+        end
+      end
+
+      module InstanceMethods
+        def request(*args, **options)
+          options[:stream] = true if options[:event_stream]
+
+          super
+        end
+
+        def build_request(*)
+          super.tap do |request|
+            if request.options.event_stream
+              request.headers["accept"] = "text/event-stream"
+              request.headers["cache-control"] = "no-cache"
+            end
+          end
+        end
+      end
+
+      module RequestMethods
+        attr_accessor :last_server_sent_message
+
+        def initialize(*)
+          super
+
+          @last_server_sent_message = nil
+        end
+      end
+
+      module StreamResponseMethods
+        # yields each event Message as the server emits them.
+        def each_message(&block)
+          return enum_for(__method__) unless block
+
+          payload = {}
+
+          each_line do |line|
+            if line.empty?
+              if payload[:comment]
+                payload.clear
+                next
+              end
+
+              next if payload.empty?
+
+              message = Message.new(**payload)
+
+              payload.clear
+
+              @request.last_server_sent_message = message
+
+              yield message
+            else
+              type, value = line.split(": ", 2)
+
+              case type
+              when "data"
+                type = type.to_sym
+                if payload.key?(type)
+                  payload[type] << "\n" << value
+                else
+                  payload[type] = value
+                end
+              when "id", "event", "retry"
+                type = type.to_sym
+                raise_format_error(line) if payload.key?(type) || value.empty?
+
+                type = :retry_after if type == :retry # avoid using keyword
+
+                payload[type] = value
+              else
+                # skip if it's a comment
+                if line.start_with?(":")
+                  payload[:comment] = true
+                  next
+                end
+
+                raise_format_error(line)
+              end
+            end
+          end
+        end
+
+        private
+
+        def raise_format_error(line)
+          raise Error, "'#{line}': invalid or unsupported event stream format"
+        end
+      end
+
+      module ServerSentEventsRetries
+        module InstanceMethods
+          private
+
+          def prepare_to_retry(request, *)
+            super
+
+            last_message = request.last_server_sent_message
+
+            return unless last_message && last_message.id
+
+            request.headers["last-event-id"] = last_message.id
+          ensure
+            request.last_server_sent_message = nil
+          end
+
+          def when_to_retry(request, *)
+            retry_after = request.last_server_sent_message&.retry_after
+
+            retry_after / 1_000.0 if retry_after # original in milliseconds
+
+            request.last_server_sent_message&.retry_after && super
+          end
+        end
+      end
+    end
+    register_plugin(:server_sent_events, ServerSentEvents)
+  end
+end

data/lib/httpx/plugins/ssrf_filter.rb

@@ -90,12 +90,22 @@ module HTTPX
       # adds support for the following options:
       #
       # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
+      # :extra_unsafe_ranges :: A list of IP ranges (or addresses) that will be filtered, in addition to the defaults
+      # :safe_private_ranges :: A list of IP ranges (or addresses) that will not be filtered, even if they'd be filtered by default
       module OptionsMethods
         private
 
         def option_allowed_schemes(value)
           Array(value)
         end
+
+        def option_extra_unsafe_ranges(value)
+          Array(value).map { |v| v.is_a?(IPAddr) ? v : IPAddr.new(v) }
+        end
+
+        def option_safe_private_ranges(value)
+          Array(value).map { |v| v.is_a?(IPAddr) ? v : IPAddr.new(v) }
+        end
       end
 
       module InstanceMethods
@@ -132,7 +142,12 @@ module HTTPX
         end
 
         def addresses=(addrs)
-          addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
+          addrs.reject! do |ipaddr|
+            ipaddr = ipaddr.address
+            next false if @options.safe_private_ranges&.any? { |r| r.include?(ipaddr) }
+
+            SsrfFilter.unsafe_ip_address?(ipaddr) || @options.extra_unsafe_ranges&.any? { |r| r.include?(ipaddr) }
+          end
 
           raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
 

data/lib/httpx/plugins/stream.rb

@@ -46,8 +46,8 @@ module HTTPX
       end
     end
 
-    def each_line
-      return enum_for(__method__) unless block_given?
+    def each_line(&block)
+      return enum_for(__method__) unless block
 
       line = "".b
 
@@ -55,7 +55,11 @@ module HTTPX
         line << chunk
 
         while (idx = line.index("\n"))
-          yield line.byteslice(0..(idx - 1))
+          if idx.zero?
+            yield ""
+          else
+            yield line.byteslice(0..(idx - 1))
+          end
 
           line = line.byteslice((idx + 1)..-1)
         end

data/lib/httpx/plugins/tracing.rb

@@ -111,10 +111,11 @@ module HTTPX::Plugins
         @init_time = ::Time.now.utc
       end
 
-      def send(request)
-        # request init time is only the same as the connection init time
-        # if the connection is going through the connection handshake.
-        request.init_time ||= @init_time unless open?
+      def send_request_to_parser(request)
+        if connecting?
+          # request span timeframe should include the time it took to connect.
+          request.init_time ||= @init_time
+        end
 
         super
       end
@@ -126,6 +127,16 @@ module HTTPX::Plugins
         # the connection is back to :idle, and ready to connect again.
         @init_time = ::Time.now.utc
       end
+
+      private
+
+      def ping(request)
+        # if a connection is probed for liveness, the request timeframe should include
+        # it too.
+        request.init_time ||= ::Time.now.utc
+
+        super
+      end
     end
   end
   register_plugin :tracing, Tracing

data/lib/httpx/request.rb

@@ -39,6 +39,10 @@ module HTTPX
     # Exception raised during enumerable body writes.
     attr_reader :drain_error
 
+    # when this request is sent via HTTP/2, it'll use this hash of options to set the priority of the
+    # respective HTTP/2 frame.
+    attr_reader :http2_stream_options
+
     # The IP address from the peer server.
     attr_accessor :peer_address
 
@@ -70,6 +74,7 @@ module HTTPX
     # :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
     # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
     # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
+    # :http2_stream_options :: hash of options to be used to set the HTTP/2 priority by sending an initial PRIORITY frame.
     #
     # :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
     def initialize(verb, uri, options, params = EMPTY_HASH)
@@ -81,6 +86,8 @@ module HTTPX
 
       @query_params = params.delete(:params) if params.key?(:params)
 
+      @http2_stream_options = params.key?(:http2_stream_options) ? params.delete(:http2_stream_options) : EMPTY_HASH
+
       @body = options.request_body_class.new(@headers, options, **params)
 
       @options = @body.options
@@ -100,7 +107,7 @@ module HTTPX
       @connection = @response =
         @drainer = @peer_address =
           @informational_status = @on_response_arrived = nil
-      @ping = false
+      @ping = @started = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
@@ -142,6 +149,11 @@ module HTTPX
       @options.timeout[:request_timeout]
     end
 
+    # the total request timeout defined for this request.
+    def total_request_timeout
+      @options.timeout[:total_request_timeout]
+    end
+
     def persistent?
       @persistent
     end
@@ -167,6 +179,10 @@ module HTTPX
       @state != :done
     end
 
+    def started?
+      @started
+    end
+
     # merges +h+ into the instance of HTTPX::Headers of the request.
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -287,6 +303,7 @@ module HTTPX
       when :headers
         return unless @state == :idle
 
+        @started = true
       when :body
         return unless @state == :headers ||
                       @state == :expect

data/lib/httpx/resolver/cache/file.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "pstore"
+require "tmpdir"
+
+module HTTPX
+  module Resolver::Cache
+    # Implementation of a file resolver cache.
+    class File < Base
+      # default path where the resolver cache is stored. It's versioned, as the file may
+      # change format in-between releases, and it'd signal it as corrupted.
+      DEFAULT_PATH = ::File.join(Dir.tmpdir, "httpx-ruby-#{VERSION}.cache")
+
+      def initialize(path = DEFAULT_PATH)
+        super()
+        @store = PStore.new(path, true)
+      end
+
+      def get(hostname)
+        now = Utils.now
+        @store.transaction do
+          lookups = @store[:lookups] || EMPTY_HASH
+          hostnames = @store[:hostnames] || EMPTY
+
+          _get(hostname, lookups, hostnames, now)
+        end
+      end
+
+      def set(hostname, family, entries)
+        @store.transaction do
+          lookups = @store[:lookups] || {}
+          hostnames = @store[:hostnames] || []
+
+          _set(hostname, family, entries, lookups, hostnames)
+
+          @store[:lookups] = lookups
+          @store[:hostnames] = hostnames
+        end
+      end
+
+      def evict(hostname, ip)
+        ip = ip.to_s
+
+        @store.transaction do
+          lookups = @store[:lookups] || EMPTY_HASH
+          hostnames = @store[:hostnames] || EMPTY
+
+          _evict(hostname, ip, lookups, hostnames)
+
+          @store[:lookups] = lookups
+          @store[:hostnames] = hostnames
+        end
+      end
+    end
+  end
+end

data/lib/httpx/resolver/native.rb

@@ -512,12 +512,16 @@ module HTTPX
       transition(:open)
     end
 
+    # moves the resolver state machine to the +nextstate+ state (if all conditions are met)-
     def transition(nextstate)
       case nextstate
       when :idle
-        if @io
-          @io.close
+        if (io = @io)
           @io = nil
+          io.close
+
+          # @fiber-switch-guard
+          return if @io
         end
       when :open
         return unless @state == :idle
@@ -531,7 +535,14 @@ module HTTPX
       when :closed
         return if @state == :closed
 
-        @io.close if @io
+        if (io = @io)
+          @io = nil
+          io.close
+
+          # @fiber-switch-guard
+          return if @io
+        end
+
         @start_timeout = nil
         @write_buffer.clear
         @read_buffer.clear