httpx-patched 1.6.2.1

data/lib/httpx/plugins/retries.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin adds support for retrying requests when errors happen.
+    #
+    # It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
+    # after which it will return the last response, error or not. It will **not** raise an exception.
+    #
+    # It does not retry which are not considered idempotent (see *retry_change_requests* to override).
+    #
+    # https://gitlab.com/os85/httpx/wikis/Retries
+    #
+    module Retries
+      MAX_RETRIES = 3
+      # TODO: pass max_retries in a configure/load block
+
+      IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
+
+      # subset of retryable errors which are safe to retry when reconnecting
+      RECONNECTABLE_ERRORS = [
+        IOError,
+        EOFError,
+        Errno::ECONNRESET,
+        Errno::ECONNABORTED,
+        Errno::EPIPE,
+        Errno::EINVAL,
+        Errno::ETIMEDOUT,
+        ConnectionError,
+        TLSError,
+        Connection::HTTP2::Error,
+      ].freeze
+
+      RETRYABLE_ERRORS = (RECONNECTABLE_ERRORS + [
+        Parser::Error,
+        TimeoutError,
+      ]).freeze
+      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
+
+      if ENV.key?("HTTPX_NO_JITTER")
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES)
+        end
+      else
+        def self.extra_options(options)
+          options.merge(max_retries: MAX_RETRIES, retry_jitter: DEFAULT_JITTER)
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
+      # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
+      # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
+      # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
+      # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
+      #              (i.e. <tt>->(res) { ... }</tt>).
+      module OptionsMethods
+        private
+
+        def option_retry_after(value)
+          # return early if callable
+          unless value.respond_to?(:call)
+            value = Float(value)
+            raise TypeError, ":retry_after must be positive" unless value.positive?
+          end
+
+          value
+        end
+
+        def option_retry_jitter(value)
+          # return early if callable
+          raise TypeError, ":retry_jitter must be callable" unless value.respond_to?(:call)
+
+          value
+        end
+
+        def option_max_retries(value)
+          num = Integer(value)
+          raise TypeError, ":max_retries must be positive" unless num >= 0
+
+          num
+        end
+
+        def option_retry_change_requests(v)
+          v
+        end
+
+        def option_retry_on(value)
+          raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
+
+          value
+        end
+      end
+
+      module InstanceMethods
+        # returns a `:retries` plugin enabled session with +n+ maximum retries per request setting.
+        def max_retries(n)
+          with(max_retries: n)
+        end
+
+        private
+
+        def fetch_response(request, selector, options)
+          response = super
+
+          if response &&
+             request.retries.positive? &&
+             repeatable_request?(request, options) &&
+             (
+               (
+                 response.is_a?(ErrorResponse) && retryable_error?(response.error)
+               ) ||
+               (
+                 options.retry_on && options.retry_on.call(response)
+               )
+             )
+            try_partial_retry(request, response)
+            log { "failed to get response, #{request.retries} tries to go..." }
+            request.retries -= 1 unless request.ping? # do not exhaust retries on connection liveness probes
+            request.transition(:idle)
+
+            retry_after = options.retry_after
+            retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+
+            if retry_after
+              # apply jitter
+              if (jitter = request.options.retry_jitter)
+                retry_after = jitter.call(retry_after)
+              end
+
+              retry_start = Utils.now
+              log { "retrying after #{retry_after} secs..." }
+              selector.after(retry_after) do
+                if (response = request.response)
+                  response.finish!
+                  # request has terminated abruptly meanwhile
+                  request.emit(:response, response)
+                else
+                  log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
+                  send_request(request, selector, options)
+                end
+              end
+            else
+              send_request(request, selector, options)
+            end
+
+            return
+          end
+          response
+        end
+
+        # returns whether +request+ can be retried.
+        def repeatable_request?(request, options)
+          IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
+        end
+
+        # returns whether the +ex+ exception happend for a retriable request.
+        def retryable_error?(ex)
+          RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
+        end
+
+        def proxy_error?(request, response, _)
+          super && !request.retries.positive?
+        end
+
+        #
+        # Attempt to set the request to perform a partial range request.
+        # This happens if the peer server accepts byte-range requests, and
+        # the last response contains some body payload.
+        #
+        def try_partial_retry(request, response)
+          response = response.response if response.is_a?(ErrorResponse)
+
+          return unless response
+
+          unless response.headers.key?("accept-ranges") &&
+                 response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
+                 (original_body = response.body)
+            response.body.close
+            return
+          end
+
+          request.partial_response = response
+
+          size = original_body.bytesize
+
+          request.headers["range"] = "bytes=#{size}-"
+        end
+      end
+
+      module RequestMethods
+        # number of retries left.
+        attr_accessor :retries
+
+        # a response partially received before.
+        attr_writer :partial_response
+
+        # initializes the request instance, sets the number of retries for the request.
+        def initialize(*args)
+          super
+          @retries = @options.max_retries
+        end
+
+        def response=(response)
+          if @partial_response
+            if response.is_a?(Response) && response.status == 206
+              response.from_partial_response(@partial_response)
+            else
+              @partial_response.close
+            end
+            @partial_response = nil
+          end
+
+          super
+        end
+      end
+
+      module ResponseMethods
+        def from_partial_response(response)
+          @status = response.status
+          @headers = response.headers
+          @body = response.body
+        end
+      end
+    end
+    register_plugin :retries, Retries
+  end
+end

data/lib/httpx/plugins/ssrf_filter.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class ServerSideRequestForgeryError < Error; end
+
+  module Plugins
+    #
+    # This plugin adds support for preventing Server-Side Request Forgery attacks.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
+    #
+    module SsrfFilter
+      module IPAddrExtensions
+        refine IPAddr do
+          def prefixlen
+            mask_addr = @mask_addr
+            raise "Invalid mask" if mask_addr.zero?
+
+            mask_addr >>= 1 while (mask_addr & 0x1).zero?
+
+            length = 0
+            while mask_addr & 0x1 == 0x1
+              length += 1
+              mask_addr >>= 1
+            end
+
+            length
+          end
+        end
+      end
+
+      using IPAddrExtensions
+
+      # https://en.wikipedia.org/wiki/Reserved_IP_addresses
+      IPV4_BLACKLIST = [
+        IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
+        IPAddr.new("10.0.0.0/8"), # Private network
+        IPAddr.new("100.64.0.0/10"), # Shared Address Space
+        IPAddr.new("127.0.0.0/8"), # Loopback
+        IPAddr.new("169.254.0.0/16"), # Link-local
+        IPAddr.new("172.16.0.0/12"), # Private network
+        IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
+        IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
+        IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
+        IPAddr.new("192.168.0.0/16"), # Private network
+        IPAddr.new("198.18.0.0/15"), # Network benchmark tests
+        IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
+        IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
+        IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
+        IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
+        IPAddr.new("255.255.255.255"), # Broadcast
+      ].freeze
+
+      IPV6_BLACKLIST = ([
+        IPAddr.new("::1/128"), # Loopback
+        IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
+        IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
+        IPAddr.new("2001::/32"), # Teredo tunneling
+        IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
+        IPAddr.new("2001:20::/28"), # ORCHIDv2
+        IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
+        IPAddr.new("2002::/16"), # 6to4
+        IPAddr.new("fc00::/7"), # Unique local address
+        IPAddr.new("fe80::/10"), # Link-local address
+        IPAddr.new("ff00::/8"), # Multicast
+      ] + IPV4_BLACKLIST.flat_map do |ipaddr|
+        prefixlen = ipaddr.prefixlen
+
+        ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
+        ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
+
+        [ipv4_compatible, ipv4_mapped]
+      end).freeze
+
+      class << self
+        def extra_options(options)
+          options.merge(allowed_schemes: %w[https http])
+        end
+
+        def unsafe_ip_address?(ipaddr)
+          range = ipaddr.to_range
+          return true if range.first != range.last
+
+          return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
+
+          IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
+      module OptionsMethods
+        private
+
+        def option_allowed_schemes(value)
+          Array(value)
+        end
+      end
+
+      module InstanceMethods
+        def send_requests(*requests)
+          responses = requests.map do |request|
+            next if @options.allowed_schemes.include?(request.uri.scheme)
+
+            error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
+            error.set_backtrace(caller)
+            response = ErrorResponse.new(request, error)
+            request.emit(:response, response)
+            response
+          end
+          allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
+          allowed_responses = super(*allowed_requests)
+          allowed_responses.each_with_index do |res, idx|
+            req = allowed_requests[idx]
+            responses[requests.index(req)] = res
+          end
+
+          responses
+        end
+      end
+
+      module ConnectionMethods
+        def initialize(*)
+          begin
+            super
+          rescue ServerSideRequestForgeryError => e
+            # may raise when IPs are passed as options via :addresses
+            throw(:resolve_error, e)
+          end
+        end
+
+        def addresses=(addrs)
+          addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
+
+          raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
+
+          super
+        end
+      end
+    end
+
+    register_plugin :ssrf_filter, SsrfFilter
+  end
+end

data/lib/httpx/plugins/stream.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class StreamResponse
+    attr_reader :request
+
+    def initialize(request, session)
+      @request = request
+      @options = @request.options
+      @session = session
+      @response_enum = nil
+      @buffered_chunks = []
+    end
+
+    def each(&block)
+      return enum_for(__method__) unless block
+
+      if (response_enum = @response_enum)
+        @response_enum = nil
+        # streaming already started, let's finish it
+
+        while (chunk = @buffered_chunks.shift)
+          block.call(chunk)
+        end
+
+        # consume enum til the end
+        begin
+          while (chunk = response_enum.next)
+            block.call(chunk)
+          end
+        rescue StopIteration
+          return
+        end
+      end
+
+      @request.stream = self
+
+      begin
+        @on_chunk = block
+
+        response = @session.request(@request)
+
+        response.raise_for_status
+      ensure
+        @on_chunk = nil
+      end
+    end
+
+    def each_line
+      return enum_for(__method__) unless block_given?
+
+      line = "".b
+
+      each do |chunk|
+        line << chunk
+
+        while (idx = line.index("\n"))
+          yield line.byteslice(0..idx - 1)
+
+          line = line.byteslice(idx + 1..-1)
+        end
+      end
+
+      yield line unless line.empty?
+    end
+
+    # This is a ghost method. It's to be used ONLY internally, when processing streams
+    def on_chunk(chunk)
+      raise NoMethodError unless @on_chunk
+
+      @on_chunk.call(chunk)
+    end
+
+    # :nocov:
+    def inspect
+      "#<#{self.class}:#{object_id}>"
+    end
+    # :nocov:
+
+    def to_s
+      if @request.response
+        @request.response.to_s
+      else
+        @buffered_chunks.join
+      end
+    end
+
+    private
+
+    def response
+      @request.response || begin
+        response_enum = each
+        while (chunk = response_enum.next)
+          @buffered_chunks << chunk
+          break if @request.response
+        end
+        @response_enum = response_enum
+        @request.response
+      end
+    end
+
+    def respond_to_missing?(meth, include_private)
+      if (response = @request.response)
+        response.respond_to_missing?(meth, include_private)
+      else
+        @options.response_class.method_defined?(meth) || (include_private && @options.response_class.private_method_defined?(meth))
+      end || super
+    end
+
+    def method_missing(meth, *args, **kwargs, &block)
+      return super unless response.respond_to?(meth)
+
+      response.__send__(meth, *args, **kwargs, &block)
+    end
+  end
+
+  module Plugins
+    #
+    # This plugin adds support for streaming a response (useful for i.e. "text/event-stream" payloads).
+    #
+    # https://gitlab.com/os85/httpx/wikis/Stream
+    #
+    module Stream
+      def self.extra_options(options)
+        options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
+      end
+
+      module InstanceMethods
+        def request(*args, stream: false, **options)
+          return super(*args, **options) unless stream
+
+          requests = args.first.is_a?(Request) ? args : build_requests(*args, options)
+          raise Error, "only 1 response at a time is supported for streaming requests" unless requests.size == 1
+
+          request = requests.first
+
+          StreamResponse.new(request, self)
+        end
+      end
+
+      module RequestMethods
+        attr_accessor :stream
+      end
+
+      module ResponseMethods
+        def stream
+          request = @request.root_request if @request.respond_to?(:root_request)
+          request ||= @request
+
+          request.stream
+        end
+      end
+
+      module ResponseBodyMethods
+        def initialize(*)
+          super
+          @stream = @response.stream
+        end
+
+        def write(chunk)
+          return super unless @stream
+
+          return 0 if chunk.empty?
+
+          chunk = decode_chunk(chunk)
+
+          @stream.on_chunk(chunk.dup)
+
+          chunk.size
+        end
+
+        private
+
+        def transition(*)
+          return if @stream
+
+          super
+        end
+      end
+    end
+    register_plugin :stream, Stream
+  end
+end