httpx 0.21.0 → 1.2.1

data/lib/httpx/plugins/response_cache/file_store.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "pathname"
+require_relative "store"
+
+module HTTPX::Plugins
+  module ResponseCache
+    class FileStore < Store
+      def initialize(dir = Dir.tmpdir)
+        @dir = Pathname.new(dir)
+      end
+
+      def clear
+        # delete all files
+      end
+
+      def cached?(request)
+        file_path = @dir.join(request.response_cache_key)
+
+        exist?(file_path)
+      end
+
+      private
+
+      def _get(request)
+        return unless cached?(request)
+
+        File.open(@dir.join(request.response_cache_key))
+      end
+
+      def _set(request, response)
+        file_path = @dir.join(request.response_cache_key)
+
+        response.copy_to(file_path)
+
+        response.body.rewind
+      end
+    end
+  end
+end

data/lib/httpx/plugins/response_cache/store.rb

@@ -1,28 +1,23 @@
 # frozen_string_literal: true
 
-require "forwardable"
-
 module HTTPX::Plugins
   module ResponseCache
     class Store
-      extend Forwardable
-
-      def_delegator :@store, :clear
-
       def initialize
         @store = {}
+        @store_mutex = Thread::Mutex.new
+      end
+
+      def clear
+        @store_mutex.synchronize { @store.clear }
       end
 
       def lookup(request)
-        responses = @store[request.response_cache_key]
+        responses = _get(request)
 
         return unless responses
 
-        response = responses.find(&method(:match_by_vary?).curry(2)[request])
-
-        return unless response && response.fresh?
-
-        response
+        responses.find(&method(:match_by_vary?).curry(2)[request])
       end
 
       def cached?(request)
@@ -32,11 +27,7 @@ module HTTPX::Plugins
       def cache(request, response)
         return unless ResponseCache.cacheable_request?(request) && ResponseCache.cacheable_response?(response)
 
-        responses = (@store[request.response_cache_key] ||= [])
-
-        responses.reject!(&method(:match_by_vary?).curry(2)[request])
-
-        responses << response
+        _set(request, response)
       end
 
       def prepare(request)
@@ -71,6 +62,32 @@ module HTTPX::Plugins
           !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
         end
       end
+
+      def _get(request)
+        @store_mutex.synchronize do
+          responses = @store[request.response_cache_key]
+
+          return unless responses
+
+          responses.select! do |res|
+            !res.body.closed? && res.fresh?
+          end
+
+          responses
+        end
+      end
+
+      def _set(request, response)
+        @store_mutex.synchronize do
+          responses = (@store[request.response_cache_key] ||= [])
+
+          responses.reject! do |res|
+            res.body.closed? || !res.fresh? || match_by_vary?(request, res)
+          end
+
+          responses << response
+        end
+      end
     end
   end
 end

data/lib/httpx/plugins/response_cache.rb

@@ -5,10 +5,10 @@ module HTTPX
     #
     # This plugin adds support for retrying requests when certain errors happen.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Response-Cache
+    # https://gitlab.com/os85/httpx/wikis/Response-Cache
     #
     module ResponseCache
-      CACHEABLE_VERBS = %i[get head].freeze
+      CACHEABLE_VERBS = %w[GET HEAD].freeze
       CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
       private_constant :CACHEABLE_VERBS
       private_constant :CACHEABLE_STATUS_CODES
@@ -96,15 +96,15 @@ module HTTPX
 
       module RequestMethods
         def response_cache_key
-          @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}#{@uri}")
+          @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}-#{@uri}")
         end
       end
 
       module ResponseMethods
         def copy_from_cached(other)
-          @body = other.body
+          @body = other.body.dup
 
-          @body.__send__(:rewind)
+          @body.rewind
         end
 
         # A response is fresh if its age has not yet exceeded its freshness lifetime.
@@ -169,7 +169,7 @@ module HTTPX
         def date
           @date ||= Time.httpdate(@headers["date"])
         rescue NoMethodError, ArgumentError
-          Time.now.httpdate
+          Time.now
         end
       end
     end

data/lib/httpx/plugins/retries.rb

@@ -5,13 +5,13 @@ module HTTPX
     #
     # This plugin adds support for retrying requests when certain errors happen.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Retries
+    # https://gitlab.com/os85/httpx/wikis/Retries
     #
     module Retries
       MAX_RETRIES = 3
       # TODO: pass max_retries in a configure/load block
 
-      IDEMPOTENT_METHODS = %i[get options head put delete].freeze
+      IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
       RETRYABLE_ERRORS = [
         IOError,
         EOFError,
@@ -23,9 +23,10 @@ module HTTPX
         Parser::Error,
         TLSError,
         TimeoutError,
+        ConnectionError,
         Connection::HTTP2::GoawayError,
       ].freeze
-      DEFAULT_JITTER = ->(interval) { interval * (0.5 * (1 + rand)) }
+      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
 
       if ENV.key?("HTTPX_NO_JITTER")
         def self.extra_options(options)
@@ -87,15 +88,14 @@ module HTTPX
              request.retries.positive? &&
              __repeatable_request?(request, options) &&
              (
-               # rubocop:disable Style/MultilineTernaryOperator
-               options.retry_on ?
-               options.retry_on.call(response) :
                (
                  response.is_a?(ErrorResponse) && __retryable_error?(response.error)
+               ) ||
+               (
+                 options.retry_on && options.retry_on.call(response)
                )
-               # rubocop:enable Style/MultilineTernaryOperator
              )
-            response.close if response.respond_to?(:close)
+            __try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
             request.retries -= 1
             request.transition(:idle)
@@ -113,12 +113,10 @@ module HTTPX
               log { "retrying after #{retry_after} secs..." }
               pool.after(retry_after) do
                 log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-                connection = find_connection(request, connections, options)
-                connection.send(request)
+                send_request(request, connections, options)
               end
             else
-              connection = find_connection(request, connections, options)
-              connection.send(request)
+              send_request(request, connections, options)
             end
 
             return
@@ -133,15 +131,66 @@ module HTTPX
         def __retryable_error?(ex)
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
+
+        def proxy_error?(request, response)
+          super && !request.retries.positive?
+        end
+
+        #
+        # Atttempt to set the request to perform a partial range request.
+        # This happens if the peer server accepts byte-range requests, and
+        # the last response contains some body payload.
+        #
+        def __try_partial_retry(request, response)
+          response = response.response if response.is_a?(ErrorResponse)
+
+          return unless response
+
+          unless response.headers.key?("accept-ranges") &&
+                 response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
+                 (original_body = response.body)
+            response.close if response.respond_to?(:close)
+            return
+          end
+
+          request.partial_response = response
+
+          size = original_body.bytesize
+
+          request.headers["range"] = "bytes=#{size}-"
+        end
       end
 
       module RequestMethods
         attr_accessor :retries
 
+        attr_writer :partial_response
+
         def initialize(*args)
           super
           @retries = @options.max_retries
         end
+
+        def response=(response)
+          if @partial_response
+            if response.is_a?(Response) && response.status == 206
+              response.from_partial_response(@partial_response)
+            else
+              @partial_response.close
+            end
+            @partial_response = nil
+          end
+
+          super
+        end
+      end
+
+      module ResponseMethods
+        def from_partial_response(response)
+          @status = response.status
+          @headers = response.headers
+          @body = response.body
+        end
       end
     end
     register_plugin :retries, Retries

data/lib/httpx/plugins/ssrf_filter.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class ServerSideRequestForgeryError < Error; end
+
+  module Plugins
+    #
+    # This plugin adds support for preventing Server-Side Request Forgery attacks.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
+    #
+    module SsrfFilter
+      module IPAddrExtensions
+        refine IPAddr do
+          def prefixlen
+            mask_addr = @mask_addr
+            raise "Invalid mask" if mask_addr.zero?
+
+            mask_addr >>= 1 while (mask_addr & 0x1).zero?
+
+            length = 0
+            while mask_addr & 0x1 == 0x1
+              length += 1
+              mask_addr >>= 1
+            end
+
+            length
+          end
+        end
+      end
+
+      using IPAddrExtensions
+
+      # https://en.wikipedia.org/wiki/Reserved_IP_addresses
+      IPV4_BLACKLIST = [
+        IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
+        IPAddr.new("10.0.0.0/8"), # Private network
+        IPAddr.new("100.64.0.0/10"), # Shared Address Space
+        IPAddr.new("127.0.0.0/8"), # Loopback
+        IPAddr.new("169.254.0.0/16"), # Link-local
+        IPAddr.new("172.16.0.0/12"), # Private network
+        IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
+        IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
+        IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
+        IPAddr.new("192.168.0.0/16"), # Private network
+        IPAddr.new("198.18.0.0/15"), # Network benchmark tests
+        IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
+        IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
+        IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
+        IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
+        IPAddr.new("255.255.255.255"), # Broadcast
+      ].freeze
+
+      IPV6_BLACKLIST = ([
+        IPAddr.new("::1/128"), # Loopback
+        IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
+        IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
+        IPAddr.new("2001::/32"), # Teredo tunneling
+        IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
+        IPAddr.new("2001:20::/28"), # ORCHIDv2
+        IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
+        IPAddr.new("2002::/16"), # 6to4
+        IPAddr.new("fc00::/7"), # Unique local address
+        IPAddr.new("fe80::/10"), # Link-local address
+        IPAddr.new("ff00::/8"), # Multicast
+      ] + IPV4_BLACKLIST.flat_map do |ipaddr|
+        prefixlen = ipaddr.prefixlen
+
+        ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
+        ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
+
+        [ipv4_compatible, ipv4_mapped]
+      end).freeze
+
+      class << self
+        def extra_options(options)
+          options.merge(allowed_schemes: %w[https http])
+        end
+
+        def unsafe_ip_address?(ipaddr)
+          range = ipaddr.to_range
+          return true if range.first != range.last
+
+          return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
+
+          IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
+        end
+      end
+
+      module OptionsMethods
+        def option_allowed_schemes(value)
+          Array(value)
+        end
+      end
+
+      module InstanceMethods
+        def send_requests(*requests)
+          responses = requests.map do |request|
+            next if @options.allowed_schemes.include?(request.uri.scheme)
+
+            error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
+            error.set_backtrace(caller)
+            response = ErrorResponse.new(request, error, request.options)
+            request.emit(:response, response)
+            response
+          end
+          allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
+          allowed_responses = super(*allowed_requests)
+          allowed_responses.each_with_index do |res, idx|
+            req = allowed_requests[idx]
+            responses[requests.index(req)] = res
+          end
+
+          responses
+        end
+      end
+
+      module ConnectionMethods
+        def initialize(*)
+          begin
+            super
+          rescue ServerSideRequestForgeryError => e
+            # may raise when IPs are passed as options via :addresses
+            throw(:resolve_error, e)
+          end
+        end
+
+        def addresses=(addrs)
+          addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
+
+          addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
+
+          raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
+
+          super
+        end
+      end
+    end
+
+    register_plugin :ssrf_filter, SsrfFilter
+  end
+end

data/lib/httpx/plugins/stream.rb

@@ -2,35 +2,23 @@
 
 module HTTPX
   class StreamResponse
-    def initialize(request, session, connections)
+    def initialize(request, session)
       @request = request
       @session = session
-      @connections = connections
+      @response = nil
     end
 
     def each(&block)
       return enum_for(__method__) unless block
 
-      raise Error, "response already streamed" if @response
-
       @request.stream = self
 
       begin
         @on_chunk = block
 
-        if @request.response
-          # if we've already started collecting the payload, yield it first
-          # before proceeding
-          body = @request.response.body
-
-          body.each do |chunk|
-            on_chunk(chunk)
-          end
-        end
-
         response.raise_for_status
-        response.close
       ensure
+        response.close if @response
         @on_chunk = nil
       end
     end
@@ -38,7 +26,7 @@ module HTTPX
     def each_line
       return enum_for(__method__) unless block_given?
 
-      line = +""
+      line = "".b
 
       each do |chunk|
         line << chunk
@@ -49,6 +37,8 @@ module HTTPX
           line = line.byteslice(idx + 1..-1)
         end
       end
+
+      yield line unless line.empty?
     end
 
     # This is a ghost method. It's to be used ONLY internally, when processing streams
@@ -71,9 +61,11 @@ module HTTPX
     private
 
     def response
-      @session.__send__(:receive_requests, [@request], @connections) until @request.response
+      return @response if @response
 
-      @request.response
+      @request.response || begin
+        @response = @session.request(@request)
+      end
     end
 
     def respond_to_missing?(meth, *args)
@@ -91,12 +83,14 @@ module HTTPX
     #
     # This plugin adds support for stream response (text/event-stream).
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Stream
+    # https://gitlab.com/os85/httpx/wikis/Stream
     #
     module Stream
-      module InstanceMethods
-        private
+      def self.extra_options(options)
+        options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
+      end
 
+      module InstanceMethods
         def request(*args, stream: false, **options)
           return super(*args, **options) unless stream
 
@@ -105,9 +99,7 @@ module HTTPX
 
           request = requests.first
 
-          connections = _send_requests(requests)
-
-          StreamResponse.new(request, self, connections)
+          StreamResponse.new(request, self)
         end
       end
 
@@ -117,7 +109,10 @@ module HTTPX
 
       module ResponseMethods
         def stream
-          @request.stream
+          request = @request.root_request if @request.respond_to?(:root_request)
+          request ||= @request
+
+          request.stream
         end
       end
 
@@ -130,7 +125,13 @@ module HTTPX
         def write(chunk)
           return super unless @stream
 
-          @stream.on_chunk(chunk.to_s.dup)
+          return 0 if chunk.empty?
+
+          chunk = decode_chunk(chunk)
+
+          @stream.on_chunk(chunk.dup)
+
+          chunk.size
         end
 
         private
@@ -141,12 +142,6 @@ module HTTPX
           super
         end
       end
-
-      def self.const_missing(const_name)
-        super unless const_name == :StreamResponse
-        warn "DEPRECATION WARNING: the class #{self}::StreamResponse is deprecated. Use HTTPX::StreamResponse instead."
-        HTTPX::StreamResponse
-      end
     end
     register_plugin :stream, Stream
   end

data/lib/httpx/plugins/upgrade/h2.rb

@@ -6,12 +6,12 @@ module HTTPX
     # This plugin adds support for upgrading an HTTP/1.1 connection to HTTP/2
     # via an Upgrade: h2 response declaration
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade#h2
+    # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2
     #
     module H2
       class << self
-        def configure(klass)
-          klass.default_options.upgrade_handlers.register "h2", self
+        def extra_options(options)
+          options.merge(upgrade_handlers: options.upgrade_handlers.merge("h2" => self))
         end
 
         def call(connection, _request, _response)
@@ -32,7 +32,7 @@ module HTTPX
 
           @parser = Connection::HTTP2.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
-          @upgrade_protocol = :h2
+          @upgrade_protocol = "h2"
 
           # what's happening here:
           # a deviation from the state machine is done to perform the actions when a

data/lib/httpx/plugins/upgrade.rb

@@ -6,7 +6,7 @@ module HTTPX
     # This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
     # Upgrade header.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade
+    # https://gitlab.com/os85/httpx/wikis/Upgrade
     #
     module Upgrade
       class << self
@@ -15,16 +15,13 @@ module HTTPX
         end
 
         def extra_options(options)
-          upgrade_handlers = Module.new do
-            extend Registry
-          end
-          options.merge(upgrade_handlers: upgrade_handlers)
+          options.merge(upgrade_handlers: {})
         end
       end
 
       module OptionsMethods
         def option_upgrade_handlers(value)
-          raise TypeError, ":upgrade_handlers must be a registry" unless value.respond_to?(:registry)
+          raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
 
           value
         end
@@ -35,19 +32,20 @@ module HTTPX
           response = super
 
           if response
-            return response unless response.respond_to?(:headers) && response.headers.key?("upgrade")
+            return response unless response.is_a?(Response)
+
+            return response unless response.headers.key?("upgrade")
 
             upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
 
-            return response unless upgrade_protocol && options.upgrade_handlers.registry.key?(upgrade_protocol)
+            return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
 
-            protocol_handler = options.upgrade_handlers.registry(upgrade_protocol)
+            protocol_handler = options.upgrade_handlers[upgrade_protocol]
 
             return response unless protocol_handler
 
             log { "upgrading to #{upgrade_protocol}..." }
             connection = find_connection(request, connections, options)
-            connections << connection unless connections.include?(connection)
 
             # do not upgrade already upgraded connections
             return if connection.upgrade_protocol == upgrade_protocol