url_canonicalize 1.0.0 → 2.0.0

data/lib/url_canonicalize/destination.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require 'ipaddr'
+require 'socket'
+
+module URLCanonicalize
+  # Resolves a URI to an address that is permitted by the fetch security policy
+  class Destination
+    FORBIDDEN_IPV4_RANGES = %w[
+      0.0.0.0/8
+      10.0.0.0/8
+      100.64.0.0/10
+      127.0.0.0/8
+      169.254.0.0/16
+      172.16.0.0/12
+      192.0.0.0/24
+      192.0.2.0/24
+      192.88.99.0/24
+      192.168.0.0/16
+      198.18.0.0/15
+      198.51.100.0/24
+      203.0.113.0/24
+      224.0.0.0/4
+      240.0.0.0/4
+    ].map { |range| IPAddr.new(range) }.freeze
+
+    # Fail closed: only prefixes in IANA's IPv6 Global Unicast Address Space registry are accepted.
+    ALLOCATED_IPV6_RANGES = %w[
+      2001:200::/23
+      2001:400::/23
+      2001:600::/23
+      2001:800::/22
+      2001:c00::/23
+      2001:e00::/23
+      2001:1200::/23
+      2001:1400::/22
+      2001:1800::/23
+      2001:1a00::/23
+      2001:1c00::/22
+      2001:2000::/19
+      2001:4000::/23
+      2001:4200::/23
+      2001:4400::/23
+      2001:4600::/23
+      2001:4800::/23
+      2001:4a00::/23
+      2001:4c00::/23
+      2001:5000::/20
+      2001:8000::/19
+      2001:a000::/20
+      2001:b000::/20
+      2003::/18
+      2400::/12
+      2410::/12
+      2600::/12
+      2610::/23
+      2620::/23
+      2630::/12
+      2800::/12
+      2a00::/12
+      2a10::/12
+      2c00::/12
+    ].map { |range| IPAddr.new(range) }.freeze
+
+    FORBIDDEN_IPV6_RANGES = %w[
+      2001::/23
+      2001:db8::/32
+      2002::/16
+    ].map { |range| IPAddr.new(range) }.freeze
+
+    class << self
+      def resolve(uri, options)
+        validate!(uri, options)
+        addresses = resolve_addresses(uri)
+        validate_addresses!(uri, addresses, options)
+        addresses.first
+      end
+
+      def validate!(uri, options)
+        raise URLCanonicalize::Exception::Security, 'URLs containing userinfo are not allowed' if uri.userinfo
+        return if options[:allowed_ports].include?(uri.port)
+
+        raise URLCanonicalize::Exception::Security, "Port #{uri.port} is not allowed"
+      end
+
+      private
+
+      def resolve_addresses(uri)
+        addresses = Addrinfo.getaddrinfo(uri.host, uri.port, nil, Socket::SOCK_STREAM)
+        addresses = addresses.map(&:ip_address).uniq
+        raise SocketError, "No addresses found for #{uri.host}" if addresses.empty?
+
+        addresses
+      end
+
+      def validate_addresses!(uri, addresses, options)
+        return if options[:allow_private_networks]
+
+        addresses.each do |address|
+          next if public_address?(address)
+
+          raise URLCanonicalize::Exception::Security,
+                "Address #{address} for #{uri.host} is not publicly routable"
+        end
+      end
+
+      def public_address?(value)
+        address = IPAddr.new(value)
+        address = address.native if address.ipv4_mapped?
+
+        if address.ipv4?
+          FORBIDDEN_IPV4_RANGES.none? { |range| range.include?(address) }
+        else
+          ALLOCATED_IPV6_RANGES.any? { |range| range.include?(address) } &&
+            FORBIDDEN_IPV6_RANGES.none? { |range| range.include?(address) }
+        end
+      end
+    end
+  end
+end

data/lib/url_canonicalize/exception.rb

@@ -7,6 +7,9 @@ module URLCanonicalize
     Failure = Class.new(self)
     Redirect = Class.new(self)
     Request = Class.new(self)
+    ResponseTooLarge = Class.new(self)
+    Security = Class.new(self)
+    Timeout = Class.new(self)
     URI = Class.new(self)
   end
 end

data/lib/url_canonicalize/http.rb

@@ -4,7 +4,12 @@ module URLCanonicalize
   # Persistent connection for possible repeated requests to the same host
   class HTTP
     def fetch
-      loop { break last_known_good if handle_response }
+      final = ::Timeout.timeout(
+        options[:total_timeout],
+        URLCanonicalize::Exception::Timeout,
+        "Canonicalization exceeded #{options[:total_timeout]} seconds"
+      ) { fetch_within_deadline }
+      build_result(final)
     end
 
     def uri
@@ -17,15 +22,31 @@ module URLCanonicalize
     end
 
     def do_request(http_request)
-      http.request http_request
+      http.request(http_request) { |response| read_response_body(http_request, response) }
     end
 
+    attr_reader :options
+
     private
 
     attr_accessor :last_known_good
 
-    def initialize(raw_url)
+    def initialize(raw_url, options: nil, **values)
+      raise ArgumentError, 'Pass either options: or keyword options, not both' if options && !values.empty?
+
       @raw_url = raw_url
+      @options = options || URLCanonicalize::Options.new(**values)
+    end
+
+    def build_result(final)
+      Result.new(url: final.url, response: final.response, html: final.html, chain: chain, source: @final_source)
+    end
+
+    def fetch_within_deadline
+      loop do
+        result = handle_response
+        break result if result
+      end
     end
 
     # Fetch the response
@@ -45,7 +66,8 @@ module URLCanonicalize
       request.with_uri(uri).fetch
     end
 
-    # Parse the response, and clear the response ready to follow the next redirect
+    # Parse the response, and clear the response ready to follow the next redirect.
+    # Returns the final response when canonicalization is complete, or nil to continue
     def handle_response
       result = parse_response
       @response = nil
@@ -59,7 +81,7 @@ module URLCanonicalize
       when URLCanonicalize::Response::Success
         handle_success
       when URLCanonicalize::Response::Redirect
-        redirect_loop_detected? || max_redirects_reached?
+        handle_redirect
       when URLCanonicalize::Response::CanonicalFound
         handle_canonical_found
       when URLCanonicalize::Response::Failure
@@ -69,54 +91,62 @@ module URLCanonicalize
       end
     end
 
-    def redirect_loop_detected?
-      if redirect_list.include?(response_url)
-        return true if last_known_good
+    def handle_redirect
+      reason = hop_refusal_reason
+      return follow_hop(:redirect) unless reason
 
-        raise URLCanonicalize::Exception::Redirect, 'Redirect loop detected'
-      end
-
-      redirect_list << response_url
-      increment_redirects
-      set_url_from_response
-      false
+      finish_with_last_known_good || raise(URLCanonicalize::Exception::Redirect, reason)
     end
 
-    def max_redirects_reached?
-      return false unless @redirects > options[:max_redirects]
-      return true if last_known_good
+    def handle_canonical_found
+      self.last_known_good = response.response
+      return finish_with_last_known_good if hop_refusal_reason
 
-      raise URLCanonicalize::Exception::Redirect, "#{@redirects} redirects is too many"
+      follow_hop(:canonical_link)
     end
 
-    def redirect_list
-      @redirect_list ||= []
+    # A terminal response whose URL came from a server-declared canonical link
+    def finish_with_last_known_good
+      return unless last_known_good
+
+      @final_source = :canonical_link
+      last_known_good
     end
 
-    def increment_redirects
-      @redirects = redirects + 1
+    # Redirects and followed canonical links share one visited-URL set and one
+    # hop budget, so any cycle or over-long chain terminates deterministically
+    def hop_refusal_reason
+      return 'Redirect loop detected' if visited?(response_url)
+      return "#{hops + 1} redirects is too many" if hops >= options[:max_redirects]
+
+      nil
     end
 
-    def redirects
-      @redirects ||= 0
+    def follow_hop(via)
+      chain << Result::Hop.new(url: response_url, via: via)
+      self.url = response_url
+      nil
     end
 
-    def handle_canonical_found
-      self.last_known_good = response.response
-      return true if response_url == url || redirect_list.include?(response_url)
+    # The URLs requested so far and how each one was discovered
+    def chain
+      @chain ||= [Result::Hop.new(url: url, via: :initial)]
+    end
 
-      set_url_from_response
-      false
+    def visited?(candidate)
+      chain.any? { |hop| hop.url == candidate }
     end
 
-    def set_url_from_response
-      self.url = response_url
+    def hops
+      chain.length - 1
     end
 
     def handle_failure
-      return true if last_known_good
-
-      raise URLCanonicalize::Exception::Failure, "#{response.failure_class}: #{response.message}"
+      finish_with_last_known_good || raise(
+        URLCanonicalize::Exception::Failure,
+        "#{response.failure_class}: #{response.message}",
+        cause: response.error
+      )
     end
 
     def handle_unhandled_response
@@ -124,8 +154,8 @@ module URLCanonicalize
     end
 
     def handle_success
+      @final_source = chain.last.via
       self.last_known_good = response
-      true
     end
 
     def url
@@ -133,46 +163,39 @@ module URLCanonicalize
     end
 
     def http
-      return @http if same_host_and_port # reuse connection
+      URLCanonicalize::Destination.validate!(uri, options)
+      return @http if same_origin? # reuse connection
 
       @previous = uri
-      @http = new_http
+      @http = options[:transport].call(uri, options)
     end
 
-    def same_host_and_port
-      uri.host == previous.host && uri.port == previous.port
+    def same_origin?
+      uri.scheme == previous.scheme && uri.host == previous.host && uri.port == previous.port
     end
 
     def previous
-      @previous ||= Struct.new(:host, :port).new
+      @previous ||= Struct.new(:scheme, :host, :port).new
     end
 
-    def new_http
-      h = Net::HTTP.new uri.host, uri.port
-
-      h.open_timeout = options[:open_timeout]
-      h.read_timeout = options[:read_timeout]
-
-      ssl!(h)
-
-      h
-    end
-
-    def ssl!(http)
-      if uri.scheme == 'https'
-        http.use_ssl = true # Can generate exception
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    def read_response_body(http_request, response)
+      if buffer_response_body?(http_request, response)
+        enforce_content_length!(response)
+        response.read_body(URLCanonicalize::BoundedBody.new(options[:max_body_bytes]))
       else
-        http.use_ssl = false
+        response.read_body { |_chunk| nil }
       end
     end
 
-    def options
-      @options ||= {
-        open_timeout: 8, # Twitter responds in >5s
-        read_timeout: 15,
-        max_redirects: 10
-      }
+    def buffer_response_body?(http_request, response)
+      http_request.method == 'GET' && response.is_a?(Net::HTTPSuccess) && URLCanonicalize::MediaType.buffered?(response)
+    end
+
+    def enforce_content_length!(response)
+      return unless response.content_length && response.content_length > options[:max_body_bytes]
+
+      raise URLCanonicalize::Exception::ResponseTooLarge,
+            "Response body exceeds #{options[:max_body_bytes]} bytes"
     end
   end
 end

data/lib/url_canonicalize/link_header.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # Parses HTTP Link header fields (RFC 8288) far enough to find the target of
+  # the first link whose rel tokens include "canonical"
+  module LinkHeader
+    LINK_VALUE = /<([^>]*)>((?:\s*;\s*[\w!#$%&'*+.^`|~-]+\s*=\s*(?:"[^"]*"|[^",;]+))*)/
+    REL_PARAM = /;\s*rel\s*=\s*(?:"([^"]*)"|([^",;\s]+))/i
+    CANONICAL = 'canonical'
+
+    extend self
+
+    def canonical(response)
+      fields = response.get_fields('link')
+      return unless fields
+
+      fields.each do |field|
+        field.scan(LINK_VALUE) do |target, params|
+          return target if canonical?(params)
+        end
+      end
+
+      nil
+    end
+
+    def canonical?(params)
+      match = REL_PARAM.match(params)
+      return false unless match
+
+      rel = match[1] || match[2]
+      rel.split.any? { |token| token.casecmp?(CANONICAL) }
+    end
+  end
+end

data/lib/url_canonicalize/media_type.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # Normalizes response media types for safe buffering and parsing decisions
+  module MediaType
+    HTML = %w[text/html application/xhtml+xml].freeze
+    BUFFERED = (HTML + %w[application/xml text/xml]).freeze
+
+    extend self
+
+    def buffered?(response)
+      BUFFERED.include?(normalized(response))
+    end
+
+    def html?(response)
+      HTML.include?(normalized(response))
+    end
+
+    def normalized(response)
+      response['content-type'].to_s.split(';', 2).first.to_s.strip.downcase
+    end
+  end
+end

data/lib/url_canonicalize/options.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # Validated configuration for one canonicalization operation
+  class Options
+    DEFAULTS = {
+      allow_private_networks: false,
+      allowed_ports: [80, 443].freeze,
+      headers: {
+        'Accept-Language' => 'en-US,en;q=0.8',
+        'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) ' \
+                        'AppleWebKit/537.36 (KHTML, like Gecko) ' \
+                        'Chrome/51.0.2704.103 Safari/537.36'
+      }.freeze,
+      logger: nil,
+      max_body_bytes: 1_048_576,
+      total_timeout: 30,
+      open_timeout: 8,
+      read_timeout: 15,
+      write_timeout: 8,
+      max_redirects: 10,
+      transport: Transport.new
+    }.freeze
+
+    POSITIVE_NUMERIC_OPTIONS = %i[total_timeout open_timeout read_timeout write_timeout].freeze
+    POSITIVE_INTEGER_OPTIONS = %i[max_body_bytes max_redirects].freeze
+
+    def initialize(**values)
+      validate_known_options!(values)
+
+      @values = DEFAULTS.merge(values)
+      validate!
+      @values[:allowed_ports] = @values[:allowed_ports].dup.freeze
+      @values[:headers] = @values[:headers].dup.freeze
+      @values.freeze
+      freeze
+    end
+
+    def [](key)
+      @values.fetch(key)
+    end
+
+    def to_h
+      @values.merge(allowed_ports: @values[:allowed_ports].dup, headers: @values[:headers].dup)
+    end
+
+    private
+
+    def validate!
+      validate_private_networks!
+      validate_ports!
+      validate_positive_values!
+      validate_headers!
+      validate_logger!
+      validate_transport!
+    end
+
+    def validate_known_options!(values)
+      unknown = values.keys - DEFAULTS.keys
+      raise ArgumentError, "Unknown options: #{unknown.join(', ')}" unless unknown.empty?
+    end
+
+    def validate_private_networks!
+      return if [true, false].include?(@values[:allow_private_networks])
+
+      raise ArgumentError, 'allow_private_networks must be true or false'
+    end
+
+    def validate_ports!
+      ports = @values[:allowed_ports]
+      valid = ports.is_a?(Array) && !ports.empty? && ports.all? do |port|
+        port.is_a?(Integer) && port.between?(1, 65_535)
+      end
+      return if valid
+
+      raise ArgumentError, 'allowed_ports must contain only valid TCP ports'
+    end
+
+    def validate_positive_values!
+      POSITIVE_NUMERIC_OPTIONS.each do |name|
+        value = @values[name]
+        raise ArgumentError, "#{name} must be positive" unless value.is_a?(Numeric) && value.positive?
+      end
+
+      POSITIVE_INTEGER_OPTIONS.each do |name|
+        value = @values[name]
+        raise ArgumentError, "#{name} must be positive" unless value.is_a?(Integer) && value.positive?
+      end
+    end
+
+    def validate_headers!
+      headers = @values[:headers]
+      valid = headers.is_a?(Hash) && headers.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
+      return if valid
+
+      raise ArgumentError, 'headers must be a Hash of String keys and String values'
+    end
+
+    def validate_logger!
+      logger = @values[:logger]
+      return if logger.nil? || logger.respond_to?(:debug)
+
+      raise ArgumentError, 'logger must respond to debug'
+    end
+
+    def validate_transport!
+      return if @values[:transport].respond_to?(:call)
+
+      raise ArgumentError, 'transport must respond to call'
+    end
+  end
+end