httpx 1.1.4 → 1.2.0

data/lib/httpx/plugins/h2c.rb

@@ -4,9 +4,9 @@ module HTTPX
   module Plugins
     #
     # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
-    # (https://tools.ietf.org/html/rfc7540#section-3.2)
+    # (https://datatracker.ietf.org/doc/html/rfc7540#section-3.2)
     #
-    # https://gitlab.com/os85/httpx/wikis/Upgrade#h2c
+    # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2c
     #
     module H2C
       VALID_H2C_VERBS = %w[GET OPTIONS HEAD].freeze
@@ -73,22 +73,34 @@ module HTTPX
             @inflight -= prev_parser.requests.size
           end
 
-          parser_options = @options.merge(max_concurrent_requests: request.options.max_concurrent_requests)
-          @parser = H2CParser.new(@write_buffer, parser_options)
+          @parser = H2CParser.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
           @inflight += 1
           @parser.upgrade(request, response)
           @upgrade_protocol = "h2c"
 
-          if request.options.max_concurrent_requests != @options.max_concurrent_requests
-            @options = @options.merge(max_concurrent_requests: nil)
-          end
-
           prev_parser.requests.each do |req|
             req.transition(:idle)
             send(req)
           end
         end
+
+        private
+
+        def send_request_to_parser(request)
+          super
+
+          return unless request.headers["upgrade"] == "h2c" && parser.is_a?(Connection::HTTP1)
+
+          max_concurrent_requests = parser.max_concurrent_requests
+
+          return if max_concurrent_requests == 1
+
+          parser.max_concurrent_requests = 1
+          request.once(:response) do
+            parser.max_concurrent_requests = max_concurrent_requests
+          end
+        end
       end
     end
     register_plugin(:h2c, H2C)

data/lib/httpx/plugins/proxy/socks4.rb

@@ -4,7 +4,7 @@ require "resolv"
 require "ipaddr"
 
 module HTTPX
-  class Socks4Error < Error; end
+  class Socks4Error < HTTPProxyError; end
 
   module Plugins
     module Proxy
@@ -85,7 +85,7 @@ module HTTPX
         end
 
         class SocksParser
-          include Callbacks
+          include HTTPX::Callbacks
 
           def initialize(buffer, options)
             @buffer = buffer

data/lib/httpx/plugins/proxy/socks5.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class Socks5Error < Error; end
+  class Socks5Error < HTTPProxyError; end
 
   module Plugins
     module Proxy
@@ -137,7 +137,7 @@ module HTTPX
         end
 
         class SocksParser
-          include Callbacks
+          include HTTPX::Callbacks
 
           def initialize(buffer, options)
             @buffer = buffer

data/lib/httpx/plugins/proxy.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
-  class HTTPProxyError < Error; end
+  class HTTPProxyError < ConnectionError; end
 
   module Plugins
     #
@@ -143,7 +143,7 @@ module HTTPX
           proxy = Parameters.new(**proxy_opts)
 
           proxy_options = options.merge(proxy: proxy)
-          connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
+          connection = pool.find_connection(uri, proxy_options) || init_connection(uri, proxy_options)
           unless connections.nil? || connections.include?(connection)
             connections << connection
             set_connection_callbacks(connection, connections, options)
@@ -151,19 +151,15 @@ module HTTPX
           connection
         end
 
-        def build_connection(uri, options)
-          proxy = options.proxy
-          return super unless proxy
-
-          init_connection("tcp", uri, options)
-        end
-
         def fetch_response(request, connections, options)
           response = super
 
-          if response.is_a?(ErrorResponse) &&
-             __proxy_error?(response) && !@_proxy_uris.empty?
+          if response.is_a?(ErrorResponse) && proxy_error?(request, response)
             @_proxy_uris.shift
+
+            # return last error response if no more proxies to try
+            return response if @_proxy_uris.empty?
+
             log { "failed connecting to proxy, trying next..." }
             request.transition(:idle)
             send_request(request, connections, options)
@@ -172,13 +168,7 @@ module HTTPX
           response
         end
 
-        def build_altsvc_connection(_, _, _, _, _, options)
-          return if options.proxy
-
-          super
-        end
-
-        def __proxy_error?(response)
+        def proxy_error?(_request, response)
           error = response.error
           case error
           when NativeResolveError
@@ -235,14 +225,6 @@ module HTTPX
           end
         end
 
-        def send(request)
-          return super unless (
-            @options.proxy && @state != :idle && connecting?
-          )
-
-          (@proxy_pending ||= []) << request
-        end
-
         def connecting?
           return super unless @options.proxy
 
@@ -271,6 +253,12 @@ module HTTPX
 
         private
 
+        def initialize_type(uri, options)
+          return super unless options.proxy
+
+          "tcp"
+        end
+
         def connect
           return super unless @options.proxy
 
@@ -278,12 +266,6 @@ module HTTPX
           when :idle
             transition(:connecting)
           when :connected
-            if @proxy_pending
-              while (req = @proxy_pendind.shift)
-                send(req)
-              end
-            end
-
             transition(:open)
           end
         end

data/lib/httpx/plugins/rate_limiter.rb

@@ -9,7 +9,7 @@ module HTTPX
     # * when the server is unavailable (503);
     # * when a 3xx request comes with a "retry-after" value
     #
-    # https://gitlab.com/os85/httpx/wikis/RateLimiter
+    # https://gitlab.com/os85/httpx/wikis/Rate-Limiter
     #
     module RateLimiter
       class << self

data/lib/httpx/plugins/retries.rb

@@ -132,6 +132,10 @@ module HTTPX
           RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
         end
 
+        def proxy_error?(request, response)
+          super && !request.retries.positive?
+        end
+
         #
         # Atttempt to set the request to perform a partial range request.
         # This happens if the peer server accepts byte-range requests, and

data/lib/httpx/plugins/ssrf_filter.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module HTTPX
+  class ServerSideRequestForgeryError < Error; end
+
+  module Plugins
+    #
+    # This plugin adds support for preventing Server-Side Request Forgery attacks.
+    #
+    # https://gitlab.com/os85/httpx/wikis/Server-Side-Request-Forgery-Filter
+    #
+    module SsrfFilter
+      module IPAddrExtensions
+        refine IPAddr do
+          def prefixlen
+            mask_addr = @mask_addr
+            raise "Invalid mask" if mask_addr.zero?
+
+            mask_addr >>= 1 while (mask_addr & 0x1).zero?
+
+            length = 0
+            while mask_addr & 0x1 == 0x1
+              length += 1
+              mask_addr >>= 1
+            end
+
+            length
+          end
+        end
+      end
+
+      using IPAddrExtensions
+
+      # https://en.wikipedia.org/wiki/Reserved_IP_addresses
+      IPV4_BLACKLIST = [
+        IPAddr.new("0.0.0.0/8"), # Current network (only valid as source address)
+        IPAddr.new("10.0.0.0/8"), # Private network
+        IPAddr.new("100.64.0.0/10"), # Shared Address Space
+        IPAddr.new("127.0.0.0/8"), # Loopback
+        IPAddr.new("169.254.0.0/16"), # Link-local
+        IPAddr.new("172.16.0.0/12"), # Private network
+        IPAddr.new("192.0.0.0/24"), # IETF Protocol Assignments
+        IPAddr.new("192.0.2.0/24"), # TEST-NET-1, documentation and examples
+        IPAddr.new("192.88.99.0/24"), # IPv6 to IPv4 relay (includes 2002::/16)
+        IPAddr.new("192.168.0.0/16"), # Private network
+        IPAddr.new("198.18.0.0/15"), # Network benchmark tests
+        IPAddr.new("198.51.100.0/24"), # TEST-NET-2, documentation and examples
+        IPAddr.new("203.0.113.0/24"), # TEST-NET-3, documentation and examples
+        IPAddr.new("224.0.0.0/4"), # IP multicast (former Class D network)
+        IPAddr.new("240.0.0.0/4"), # Reserved (former Class E network)
+        IPAddr.new("255.255.255.255"), # Broadcast
+      ].freeze
+
+      IPV6_BLACKLIST = ([
+        IPAddr.new("::1/128"), # Loopback
+        IPAddr.new("64:ff9b::/96"), # IPv4/IPv6 translation (RFC 6052)
+        IPAddr.new("100::/64"), # Discard prefix (RFC 6666)
+        IPAddr.new("2001::/32"), # Teredo tunneling
+        IPAddr.new("2001:10::/28"), # Deprecated (previously ORCHID)
+        IPAddr.new("2001:20::/28"), # ORCHIDv2
+        IPAddr.new("2001:db8::/32"), # Addresses used in documentation and example source code
+        IPAddr.new("2002::/16"), # 6to4
+        IPAddr.new("fc00::/7"), # Unique local address
+        IPAddr.new("fe80::/10"), # Link-local address
+        IPAddr.new("ff00::/8"), # Multicast
+      ] + IPV4_BLACKLIST.flat_map do |ipaddr|
+        prefixlen = ipaddr.prefixlen
+
+        ipv4_compatible = ipaddr.ipv4_compat.mask(96 + prefixlen)
+        ipv4_mapped = ipaddr.ipv4_mapped.mask(80 + prefixlen)
+
+        [ipv4_compatible, ipv4_mapped]
+      end).freeze
+
+      class << self
+        def extra_options(options)
+          options.merge(allowed_schemes: %w[https http])
+        end
+
+        def unsafe_ip_address?(ipaddr)
+          range = ipaddr.to_range
+          return true if range.first != range.last
+
+          return IPV6_BLACKLIST.any? { |r| r.include?(ipaddr) } if ipaddr.ipv6?
+
+          IPV4_BLACKLIST.any? { |r| r.include?(ipaddr) } # then it's IPv4
+        end
+      end
+
+      module OptionsMethods
+        def option_allowed_schemes(value)
+          Array(value)
+        end
+      end
+
+      module InstanceMethods
+        def send_requests(*requests)
+          responses = requests.map do |request|
+            next if @options.allowed_schemes.include?(request.uri.scheme)
+
+            error = ServerSideRequestForgeryError.new("#{request.uri} URI scheme not allowed")
+            error.set_backtrace(caller)
+            response = ErrorResponse.new(request, error, request.options)
+            request.emit(:response, response)
+            response
+          end
+          allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }
+          allowed_responses = super(*allowed_requests)
+          allowed_responses.each_with_index do |res, idx|
+            req = allowed_requests[idx]
+            responses[requests.index(req)] = res
+          end
+
+          responses
+        end
+      end
+
+      module ConnectionMethods
+        def initialize(*)
+          begin
+            super
+          rescue ServerSideRequestForgeryError => e
+            # may raise when IPs are passed as options via :addresses
+            throw(:resolve_error, e)
+          end
+        end
+
+        def addresses=(addrs)
+          addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
+
+          addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
+
+          raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?
+
+          super
+        end
+      end
+    end
+
+    register_plugin :ssrf_filter, SsrfFilter
+  end
+end

data/lib/httpx/plugins/stream.rb

@@ -5,6 +5,7 @@ module HTTPX
     def initialize(request, session)
       @request = request
       @session = session
+      @response = nil
     end
 
     def each(&block)
@@ -25,7 +26,7 @@ module HTTPX
     def each_line
       return enum_for(__method__) unless block_given?
 
-      line = +""
+      line = "".b
 
       each do |chunk|
         line << chunk
@@ -36,6 +37,8 @@ module HTTPX
           line = line.byteslice(idx + 1..-1)
         end
       end
+
+      yield line unless line.empty?
     end
 
     # This is a ghost method. It's to be used ONLY internally, when processing streams
@@ -58,8 +61,10 @@ module HTTPX
     private
 
     def response
-      @response ||= begin
-        @request.response || @session.request(@request)
+      return @response if @response
+
+      @request.response || begin
+        @response = @session.request(@request)
       end
     end
 
@@ -78,7 +83,7 @@ module HTTPX
     #
     # This plugin adds support for stream response (text/event-stream).
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Stream
+    # https://gitlab.com/os85/httpx/wikis/Stream
     #
     module Stream
       def self.extra_options(options)

data/lib/httpx/plugins/upgrade/h2.rb

@@ -6,7 +6,7 @@ module HTTPX
     # This plugin adds support for upgrading an HTTP/1.1 connection to HTTP/2
     # via an Upgrade: h2 response declaration
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade#h2
+    # https://gitlab.com/os85/httpx/wikis/Connection-Upgrade#h2
     #
     module H2
       class << self

data/lib/httpx/plugins/upgrade.rb

@@ -6,7 +6,7 @@ module HTTPX
     # This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
     # Upgrade header.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade
+    # https://gitlab.com/os85/httpx/wikis/Upgrade
     #
     module Upgrade
       class << self

data/lib/httpx/plugins/webdav.rb

@@ -5,7 +5,7 @@ module HTTPX
     #
     # This plugin implements convenience methods for performing WEBDAV requests.
     #
-    # https://gitlab.com/honeyryderchuck/httpx/wikis/WEBDAV
+    # https://gitlab.com/os85/httpx/wikis/WebDav
     #
     module WebDav
       module InstanceMethods

data/lib/httpx/pool.rb

@@ -17,8 +17,6 @@ module HTTPX
       @timers = Timers.new
       @selector = Selector.new
       @connections = []
-      @eden_connections = []
-      @connected_connections = 0
     end
 
     def empty?
@@ -45,16 +43,15 @@ module HTTPX
         connection.emit(:error, e)
       end
     rescue Exception # rubocop:disable Lint/RescueException
-      @connections.each(&:reset)
+      @connections.each(&:force_reset)
       raise
     end
 
     def close(connections = @connections)
       return if connections.empty?
 
-      @eden_connections.clear
       connections = connections.reject(&:inflight?)
-      connections.each(&:close)
+      connections.each(&:terminate)
       next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
 
       # close resolvers
@@ -68,22 +65,36 @@ module HTTPX
         resolver.close unless resolver.closed?
       end
       # for https resolver
-      resolver_connections.each(&:close)
+      resolver_connections.each(&:terminate)
       next_tick until resolver_connections.none? { |c| c.state != :idle && @connections.include?(c) }
     end
 
     def init_connection(connection, _options)
-      resolve_connection(connection) unless connection.family
       connection.timers = @timers
-      connection.on(:open) do
-        @connected_connections += 1
-      end
       connection.on(:activate) do
         select_connection(connection)
       end
+      connection.on(:exhausted) do
+        case connection.state
+        when :closed
+          connection.idling
+          @connections << connection
+          select_connection(connection)
+        when :closing
+          connection.once(:close) do
+            connection.idling
+            @connections << connection
+            select_connection(connection)
+          end
+        end
+      end
       connection.on(:close) do
         unregister_connection(connection)
       end
+      connection.on(:terminate) do
+        unregister_connection(connection, true)
+      end
+      resolve_connection(connection) unless connection.family
     end
 
     def deactivate(connections)
@@ -102,16 +113,15 @@ module HTTPX
         connection.match?(uri, options)
       end
 
-      unless conn
-        @eden_connections.delete_if do |connection|
-          is_expired = connection.expired?
-          conn = connection if conn.nil? && !is_expired && connection.match?(uri, options)
-          is_expired
-        end
+      return unless conn
 
-        if conn
+      case conn.state
+      when :closed
+        conn.idling
+        select_connection(conn)
+      when :closing
+        conn.once(:close) do
           conn.idling
-          @connections << conn
           select_connection(conn)
         end
       end
@@ -151,7 +161,7 @@ module HTTPX
 
       return connection if connection.family == family
 
-      new_connection = connection.class.new(connection.type, connection.origin, connection.options)
+      new_connection = connection.class.new(connection.origin, connection.options)
       new_connection.family = family
 
       connection.once(:tcp_open) { new_connection.force_reset }
@@ -218,18 +228,12 @@ module HTTPX
     end
 
     def register_connection(connection)
-      if connection.state == :open
-        # if open, an IO was passed upstream, therefore
-        # consider it connected already.
-        @connected_connections += 1
-      end
       select_connection(connection)
     end
 
-    def unregister_connection(connection)
-      @connections.delete(connection)
-      @eden_connections << connection if connection.used? && !@eden_connections.include?(connection)
-      @connected_connections -= 1 if deselect_connection(connection)
+    def unregister_connection(connection, cleanup = !connection.used?)
+      @connections.delete(connection) if cleanup
+      deselect_connection(connection)
     end
 
     def select_connection(connection)

data/lib/httpx/request/body.rb

@@ -70,9 +70,9 @@ module HTTPX
 
     # sets the body to yield using chunked trannsfer encoding format.
     def stream(body)
-      encoded = body
-      encoded = Transcoder::Chunker.encode(body.enum_for(:each)) if chunked?
-      encoded
+      return body unless chunked?
+
+      Transcoder::Chunker.encode(body.enum_for(:each))
     end
 
     # returns whether the body yields infinitely.

data/lib/httpx/request.rb

@@ -119,10 +119,19 @@ module HTTPX
     def response=(response)
       return unless response
 
-      if response.is_a?(Response) && response.status == 100 && @headers.key?("expect")
-        @informational_status = response.status
-        return
+      if response.is_a?(Response) && response.status < 200
+        # deal with informational responses
+
+        if response.status == 100 && @headers.key?("expect")
+          @informational_status = response.status
+          return
+        end
+
+        # 103 Early Hints advertises resources in document to browsers.
+        # not very relevant for an HTTP client, discard.
+        return if response.status >= 103
       end
+
       @response = response
 
       emit(:response_started, response)

data/lib/httpx/resolver/https.rb

@@ -27,7 +27,7 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate
 
     def initialize(_, options)
       super
@@ -50,6 +50,7 @@ module HTTPX
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
+        connection.force_reset
         throw(:resolve_error, ex)
       end
 
@@ -67,7 +68,7 @@ module HTTPX
     def resolver_connection
       @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
         @building_connection = true
-        connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
+        connection = @options.connection_class.new(@uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
         @pool.init_connection(connection, @options)
         # only explicity emit addresses if connection didn't pre-resolve, i.e. it's not an IP.
         emit_addresses(connection, @family, @uri_addresses) unless connection.addresses

data/lib/httpx/resolver/native.rb

@@ -88,6 +88,7 @@ module HTTPX
       if @nameserver.nil?
         ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
+        connection.force_reset
         throw(:resolve_error, ex)
       else
         @connections << connection

data/lib/httpx/resolver/resolver.rb

@@ -38,6 +38,8 @@ module HTTPX
 
     def close; end
 
+    alias_method :terminate, :close
+
     def closed?
       true
     end
@@ -65,20 +67,29 @@ module HTTPX
           unless connection.state == :closed ||
                  # double emission check
                  (connection.addresses && addresses.intersect?(connection.addresses))
-            emit_resolved_connection(connection, addresses)
+            emit_resolved_connection(connection, addresses, early_resolve)
           end
         end
       else
-        emit_resolved_connection(connection, addresses)
+        emit_resolved_connection(connection, addresses, early_resolve)
       end
     end
 
     private
 
-    def emit_resolved_connection(connection, addresses)
-      connection.addresses = addresses
-
-      emit(:resolve, connection)
+    def emit_resolved_connection(connection, addresses, early_resolve)
+      begin
+        connection.addresses = addresses
+
+        emit(:resolve, connection)
+      rescue StandardError => e
+        if early_resolve
+          connection.force_reset
+          throw(:resolve_error, e)
+        else
+          emit(:error, connection, e)
+        end
+      end
     end
 
     def early_resolve(connection, hostname: connection.origin.host)

data/lib/httpx/response.rb

@@ -264,4 +264,4 @@ end
 
 require_relative "response/body"
 require_relative "response/buffer"
-require_relative "pmatch_extensions" if RUBY_VERSION >= "3.0.0"
+require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"