httpx 0.20.0 → 1.3.1

data/lib/httpx/plugins/retries.rb

@@ -3,15 +3,20 @@
 module HTTPX
   module Plugins
     #
-    # This plugin adds support for retrying requests when certain errors happen.
+    # This plugin adds support for retrying requests when errors happen.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Retries
+    # It has a default max number of retries (see *MAX_RETRIES* and the *max_retries* option),
+    # after which it will return the last response, error or not. It will **not** raise an exception.
+    #
+    # It does not retry which are not considered idempotent (see *retry_change_requests* to override).
+    #
+    # https://gitlab.com/os85/httpx/wikis/Retries
     #
     module Retries
       MAX_RETRIES = 3
       # TODO: pass max_retries in a configure/load block
 
-      IDEMPOTENT_METHODS = %i[get options head put delete].freeze
+      IDEMPOTENT_METHODS = %w[GET OPTIONS HEAD PUT DELETE].freeze
       RETRYABLE_ERRORS = [
         IOError,
         EOFError,
@@ -23,9 +28,10 @@ module HTTPX
         Parser::Error,
         TLSError,
         TimeoutError,
+        ConnectionError,
         Connection::HTTP2::GoawayError,
       ].freeze
-      DEFAULT_JITTER = ->(interval) { interval * (0.5 * (1 + rand)) }
+      DEFAULT_JITTER = ->(interval) { interval * ((rand + 1) * 0.5) }
 
       if ENV.key?("HTTPX_NO_JITTER")
         def self.extra_options(options)
@@ -37,6 +43,14 @@ module HTTPX
         end
       end
 
+      # adds support for the following options:
+      #
+      # :max_retries :: max number of times a request will be retried (defaults to <tt>3</tt>).
+      # :retry_change_requests :: whether idempotent requests are retried (defaults to <tt>false</tt>).
+      # :retry_after:: seconds after which a request is retried; can also be a callable object (i.e. <tt>->(req, res) { ... } </tt>)
+      # :retry_jitter :: number of seconds applied to *:retry_after* (must be a callable, i.e. <tt>->(retry_after) { ... } </tt>).
+      # :retry_on :: callable which alternatively defines a different rule for when a response is to be retried
+      #              (i.e. <tt>->(res) { ... }</tt>).
       module OptionsMethods
         def option_retry_after(value)
           # return early if callable
@@ -57,7 +71,7 @@ module HTTPX
 
         def option_max_retries(value)
           num = Integer(value)
-          raise TypeError, ":max_retries must be positive" unless num.positive?
+          raise TypeError, ":max_retries must be positive" unless num >= 0
 
           num
         end
@@ -67,7 +81,7 @@ module HTTPX
         end
 
         def option_retry_on(value)
-          raise ":retry_on must be called with the response" unless value.respond_to?(:call)
+          raise TypeError, ":retry_on must be called with the response" unless value.respond_to?(:call)
 
           value
         end
@@ -75,7 +89,7 @@ module HTTPX
 
       module InstanceMethods
         def max_retries(n)
-          with(max_retries: n.to_i)
+          with(max_retries: n)
         end
 
         private
@@ -87,15 +101,14 @@ module HTTPX
              request.retries.positive? &&
              __repeatable_request?(request, options) &&
              (
-               # rubocop:disable Style/MultilineTernaryOperator
-               options.retry_on ?
-               options.retry_on.call(response) :
                (
                  response.is_a?(ErrorResponse) && __retryable_error?(response.error)
+               ) ||
+               (
+                 options.retry_on && options.retry_on.call(response)
                )
-               # rubocop:enable Style/MultilineTernaryOperator
              )
-            response.close if response.respond_to?(:close)
+            __try_partial_retry(request, response)
             log { "failed to get response, #{request.retries} tries to go..." }
             request.retries -= 1
             request.transition(:idle)
@@ -111,14 +124,20 @@ module HTTPX
 
               retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
+
+              deactivate_connection(request, connections, options)
+
               pool.after(retry_after) do
-                log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
-                connection = find_connection(request, connections, options)
-                connection.send(request)
+                if request.response
+                  # request has terminated abruptly meanwhile
+                  request.emit(:response, request.response)
+                else
+                  log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
+                  send_request(request, connections, options)
+                end
               end
             else
-              connection = find_connection(request, connections, options)
-              connection.send(request)
+              send_request(request, connections, options)
             end
 
             return
@@ -133,15 +152,66 @@ module HTTPX
         def __retryable_error?(ex)
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
+
+        def proxy_error?(request, response)
+          super && !request.retries.positive?
+        end
+
+        #
+        # Atttempt to set the request to perform a partial range request.
+        # This happens if the peer server accepts byte-range requests, and
+        # the last response contains some body payload.
+        #
+        def __try_partial_retry(request, response)
+          response = response.response if response.is_a?(ErrorResponse)
+
+          return unless response
+
+          unless response.headers.key?("accept-ranges") &&
+                 response.headers["accept-ranges"] == "bytes" && # there's nothing else supported though...
+                 (original_body = response.body)
+            response.close if response.respond_to?(:close)
+            return
+          end
+
+          request.partial_response = response
+
+          size = original_body.bytesize
+
+          request.headers["range"] = "bytes=#{size}-"
+        end
       end
 
       module RequestMethods
         attr_accessor :retries
 
+        attr_writer :partial_response
+
         def initialize(*args)
           super
           @retries = @options.max_retries
         end
+
+        def response=(response)
+          if @partial_response
+            if response.is_a?(Response) && response.status == 206
+              response.from_partial_response(@partial_response)
+            else
+              @partial_response.close
+            end
+            @partial_response = nil
+          end
+
+          super
+        end
+      end
+
+      module ResponseMethods
+        def from_partial_response(response)
+          @status = response.status
+          @headers = response.headers
+          @body = response.body
+        end
       end
     end
     register_plugin :retries, Retries

data/lib/httpx/plugins/ssrf_filter.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class ServerSideRequestForgeryError < Error; end
+
+  module Plugins
+    #
+    # This plugin adds support for preventing Server-Side Request Forgery attacks.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
+    #
+    module SsrfFilter
+      module IPAddrExtensions
+        refine IPAddr do
+          def prefixlen
+            mask_addr = @mask_addr
+            raise "Invalid mask" if mask_addr.zero?
+
+            mask_addr >>= 1 while (mask_addr & 0x1).zero?
+
+            length = 0
+            while mask_addr & 0x1 == 0x1
+              length += 1
+              mask_addr >>= 1
+            end
+
+            length
+          end
+        end
+      end
+
+      using IPAddrExtensions
+
+      # https://en.wikipedia.org/wiki/Reserved_IP_addresses
+      IPV4_BLACKLIST = [
+        IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
+        IPAddr.new("10.0.0.0/8"), # Private network
+        IPAddr.new("100.64.0.0/10"), # Shared Address Space
+        IPAddr.new("127.0.0.0/8"), # Loopback
+        IPAddr.new("169.254.0.0/16"), # Link-local
+        IPAddr.new("172.16.0.0/12"), # Private network
+        IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
+        IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
+        IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
+        IPAddr.new("192.168.0.0/16"), # Private network
+        IPAddr.new("198.18.0.0/15"), # Network benchmark tests
+        IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
+        IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
+        IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
+        IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
+        IPAddr.new("255.255.255.255"), # Broadcast
+      ].freeze
+
+      IPV6_BLACKLIST = ([
+        IPAddr.new("::1/128"), # Loopback
+        IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
+        IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
+        IPAddr.new("2001::/32"), # Teredo tunneling
+        IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
+        IPAddr.new("2001:20::/28"), # ORCHIDv2
+        IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
+        IPAddr.new("2002::/16"), # 6to4
+        IPAddr.new("fc00::/7"), # Unique local address
+        IPAddr.new("fe80::/10"), # Link-local address
+        IPAddr.new("ff00::/8"), # Multicast
+      ] + IPV4_BLACKLIST.flat_map do |ipaddr|
+        prefixlen = ipaddr.prefixlen
+
+        ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
+        ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
+
+        [ipv4_compatible, ipv4_mapped]
+      end).freeze
+
+      class << self
+        def extra_options(options)
+          options.merge(allowed_schemes: %w[https http])
+        end
+
+        def unsafe_ip_address?(ipaddr)
+          range = ipaddr.to_range
+          return true if range.first != range.last
+
+          return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
+
+          IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
+        end
+      end
+
+      # adds support for the following options:
+      #
+      # :allowed_schemes :: list of URI schemes allowed (defaults to <tt>["https", "http"]</tt>)
+      module OptionsMethods
+        def option_allowed_schemes(value)
+          Array(value)
+        end
+      end
+
+      module InstanceMethods
+        def send_requests(*requests)
+          responses = requests.map do |request|
+            next if @options.allowed_schemes.include?(request.uri.scheme)
+
+            error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
+            error.set_backtrace(caller)
+            response = ErrorResponse.new(request, error)
+            request.emit(:response, response)
+            response
+          end
+          allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
+          allowed_responses = super(*allowed_requests)
+          allowed_responses.each_with_index do |res, idx|
+            req = allowed_requests[idx]
+            responses[requests.index(req)] = res
+          end
+
+          responses
+        end
+      end
+
+      module ConnectionMethods
+        def initialize(*)
+          begin
+            super
+          rescue ServerSideRequestForgeryError => e
+            # may raise when IPs are passed as options via :addresses
+            throw(:resolve_error, e)
+          end
+        end
+
+        def addresses=(addrs)
+          addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
+
+          addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
+
+          raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
+
+          super
+        end
+      end
+    end
+
+    register_plugin :ssrf_filter, SsrfFilter
+  end
+end

data/lib/httpx/plugins/stream.rb

@@ -2,17 +2,15 @@
 
 module HTTPX
   class StreamResponse
-    def initialize(request, session, connections)
+    def initialize(request, session)
       @request = request
       @session = session
-      @connections = connections
+      @response = nil
     end
 
     def each(&block)
       return enum_for(__method__) unless block
 
-      raise Error, "response already streamed" if @response
-
       @request.stream = self
 
       begin
@@ -20,7 +18,7 @@ module HTTPX
 
         if @request.response
           # if we've already started collecting the payload, yield it first
-          # before proceeding
+          # before proceeding.
           body = @request.response.body
 
           body.each do |chunk|
@@ -29,7 +27,6 @@ module HTTPX
         end
 
         response.raise_for_status
-        response.close
       ensure
         @on_chunk = nil
       end
@@ -38,7 +35,7 @@ module HTTPX
     def each_line
       return enum_for(__method__) unless block_given?
 
-      line = +""
+      line = "".b
 
       each do |chunk|
         line << chunk
@@ -49,6 +46,8 @@ module HTTPX
           line = line.byteslice(idx + 1..-1)
         end
       end
+
+      yield line unless line.empty?
     end
 
     # This is a ghost method. It's to be used ONLY internally, when processing streams
@@ -71,9 +70,11 @@ module HTTPX
     private
 
     def response
-      @session.__send__(:receive_requests, [@request], @connections) until @request.response
+      return @response if @response
 
-      @request.response
+      @request.response || begin
+        @response = @session.request(@request)
+      end
     end
 
     def respond_to_missing?(meth, *args)
@@ -91,12 +92,14 @@ module HTTPX
     #
     # This plugin adds support for stream response (text/event-stream).
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Stream
+    # https://gitlab.com/os85/httpx/wikis/Stream
     #
     module Stream
-      module InstanceMethods
-        private
+      def self.extra_options(options)
+        options.merge(timeout: { read_timeout: Float::INFINITY, operation_timeout: 60 })
+      end
 
+      module InstanceMethods
         def request(*args, stream: false, **options)
           return super(*args, **options) unless stream
 
@@ -105,9 +108,7 @@ module HTTPX
 
           request = requests.first
 
-          connections = _send_requests(requests)
-
-          StreamResponse.new(request, self, connections)
+          StreamResponse.new(request, self)
         end
       end
 
@@ -117,7 +118,10 @@ module HTTPX
 
       module ResponseMethods
         def stream
-          @request.stream
+          request = @request.root_request if @request.respond_to?(:root_request)
+          request ||= @request
+
+          request.stream
         end
       end
 
@@ -130,7 +134,13 @@ module HTTPX
         def write(chunk)
           return super unless @stream
 
-          @stream.on_chunk(chunk.to_s.dup)
+          return 0 if chunk.empty?
+
+          chunk = decode_chunk(chunk)
+
+          @stream.on_chunk(chunk.dup)
+
+          chunk.size
         end
 
         private
@@ -141,12 +151,6 @@ module HTTPX
           super
         end
       end
-
-      def self.const_missing(const_name)
-        super unless const_name == :StreamResponse
-        warn "DEPRECATION WARNING: the class #{self}::StreamResponse is deprecated. Use HTTPX::StreamResponse instead."
-        HTTPX::StreamResponse
-      end
     end
     register_plugin :stream, Stream
   end

data/lib/httpx/plugins/upgrade/h2.rb

@@ -6,12 +6,12 @@ module HTTPX
     # This plugin adds support for upgrading an HTTP/1.1 connection to HTTP/2
     # via an Upgrade: h2 response declaration
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade#h2
+    # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2
     #
     module H2
       class << self
-        def configure(klass)
-          klass.default_options.upgrade_handlers.register "h2", self
+        def extra_options(options)
+          options.merge(upgrade_handlers: options.upgrade_handlers.merge("h2" => self))
         end
 
         def call(connection, _request, _response)
@@ -32,7 +32,7 @@ module HTTPX
 
           @parser = Connection::HTTP2.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
-          @upgrade_protocol = :h2
+          @upgrade_protocol = "h2"
 
           # what's happening here:
           # a deviation from the state machine is done to perform the actions when a

data/lib/httpx/plugins/upgrade.rb

@@ -6,7 +6,7 @@ module HTTPX
     # This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
     # Upgrade header.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade
+    # https://gitlab.com/os85/httpx/wikis/Upgrade
     #
     module Upgrade
       class << self
@@ -15,16 +15,13 @@ module HTTPX
         end
 
         def extra_options(options)
-          upgrade_handlers = Module.new do
-            extend Registry
-          end
-          options.merge(upgrade_handlers: upgrade_handlers)
+          options.merge(upgrade_handlers: {})
         end
       end
 
       module OptionsMethods
         def option_upgrade_handlers(value)
-          raise TypeError, ":upgrade_handlers must be a registry" unless value.respond_to?(:registry)
+          raise TypeError, ":upgrade_handlers must be a Hash" unless value.is_a?(Hash)
 
           value
         end
@@ -35,19 +32,20 @@ module HTTPX
           response = super
 
           if response
-            return response unless response.respond_to?(:headers) && response.headers.key?("upgrade")
+            return response unless response.is_a?(Response)
+
+            return response unless response.headers.key?("upgrade")
 
             upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
 
-            return response unless upgrade_protocol && options.upgrade_handlers.registry.key?(upgrade_protocol)
+            return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
 
-            protocol_handler = options.upgrade_handlers.registry(upgrade_protocol)
+            protocol_handler = options.upgrade_handlers[upgrade_protocol]
 
             return response unless protocol_handler
 
             log { "upgrading to #{upgrade_protocol}..." }
             connection = find_connection(request, connections, options)
-            connections << connection unless connections.include?(connection)
 
             # do not upgrade already upgraded connections
             return if connection.upgrade_protocol == upgrade_protocol

data/lib/httpx/plugins/webdav.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements convenience methods for performing WEBDAV requests.
+    #
+    # https://gitlab.com/os85/httpx/wikis/WebDav
+    #
+    module WebDav
+      module InstanceMethods
+        def copy(src, dest)
+          request("COPY", src, headers: { "destination" => @options.origin.merge(dest) })
+        end
+
+        def move(src, dest)
+          request("MOVE", src, headers: { "destination" => @options.origin.merge(dest) })
+        end
+
+        def lock(path, timeout: nil, &blk)
+          headers = {}
+          headers["timeout"] = if timeout && timeout.positive?
+            "Second-#{timeout}"
+          else
+            "Infinite, Second-4100000000"
+          end
+          xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
+                "<D:lockinfo xmlns:D=\"DAV:\">" \
+                "<D:lockscope><D:exclusive/></D:lockscope>" \
+                "<D:locktype><D:write/></D:locktype>" \
+                "<D:owner>null</D:owner>" \
+                "</D:lockinfo>"
+          response = request("LOCK", path, headers: headers, xml: xml)
+
+          return response unless response.is_a?(Response)
+
+          return response unless blk && response.status == 200
+
+          lock_token = response.headers["lock-token"]
+
+          begin
+            blk.call(response)
+          ensure
+            unlock(path, lock_token)
+          end
+        end
+
+        def unlock(path, lock_token)
+          request("UNLOCK", path, headers: { "lock-token" => lock_token })
+        end
+
+        def mkcol(dir)
+          request("MKCOL", dir)
+        end
+
+        def propfind(path, xml = nil)
+          body = case xml
+                 when :acl
+                   '<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
+                   "<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
+                 when nil
+                   '<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
+                 else
+                   xml
+          end
+
+          request("PROPFIND", path, headers: { "depth" => "1" }, xml: body)
+        end
+
+        def proppatch(path, xml)
+          body = "<?xml version=\"1.0\"?>" \
+                 "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
+          request("PROPPATCH", path, xml: body)
+        end
+        # %i[ orderpatch acl report search]
+      end
+    end
+    register_plugin(:webdav, WebDav)
+  end
+end