url_canonicalize 0.2.1 → 1.0.2

data/lib/url_canonicalize/http.rb

@@ -4,7 +4,11 @@ module URLCanonicalize
   # Persistent connection for possible repeated requests to the same host
   class HTTP
     def fetch
-      loop { break last_known_good if handle_response }
+      ::Timeout.timeout(
+        options[:total_timeout],
+        URLCanonicalize::Exception::Timeout,
+        "Canonicalization exceeded #{options[:total_timeout]} seconds"
+      ) { fetch_within_deadline }
     end
 
     def uri
@@ -17,15 +21,24 @@ module URLCanonicalize
     end
 
     def do_request(http_request)
-      http.request http_request
+      http.request(http_request) { |response| read_response_body(http_request, response) }
     end
 
     private
 
     attr_accessor :last_known_good
+    attr_reader :options
 
-    def initialize(raw_url)
+    def fetch_within_deadline
+      loop do
+        result = handle_response
+        break result if result
+      end
+    end
+
+    def initialize(raw_url, **options)
       @raw_url = raw_url
+      @options = URLCanonicalize::Options.new(**options)
     end
 
     # Fetch the response
@@ -45,7 +58,8 @@ module URLCanonicalize
       request.with_uri(uri).fetch
     end
 
-    # Parse the response, and clear the response ready to follow the next redirect
+    # Parse the response, and clear the response ready to follow the next redirect.
+    # Returns the final response when canonicalization is complete, or nil to continue
     def handle_response
       result = parse_response
       @response = nil
@@ -59,7 +73,7 @@ module URLCanonicalize
       when URLCanonicalize::Response::Success
         handle_success
       when URLCanonicalize::Response::Redirect
-        redirect_loop_detected? || max_redirects_reached?
+        handle_redirect
       when URLCanonicalize::Response::CanonicalFound
         handle_canonical_found
       when URLCanonicalize::Response::Failure
@@ -69,54 +83,49 @@ module URLCanonicalize
       end
     end
 
-    def redirect_loop_detected?
-      if redirect_list.include?(response_url)
-        return true if last_known_good
+    def handle_redirect
+      reason = hop_refusal_reason
+      return follow_hop unless reason
 
-        raise URLCanonicalize::Exception::Redirect, 'Redirect loop detected'
-      end
-
-      redirect_list << response_url
-      increment_redirects
-      set_url_from_response
-      false
+      last_known_good || raise(URLCanonicalize::Exception::Redirect, reason)
     end
 
-    def max_redirects_reached?
-      return false unless @redirects > options[:max_redirects]
-      return true if last_known_good
+    def handle_canonical_found
+      self.last_known_good = response.response
+      return last_known_good if hop_refusal_reason
 
-      raise URLCanonicalize::Exception::Redirect, "#{@redirects} redirects is too many"
+      follow_hop
     end
 
-    def redirect_list
-      @redirect_list ||= []
-    end
+    # Redirects and followed canonical links share one visited-URL set and one
+    # hop budget, so any cycle or over-long chain terminates deterministically
+    def hop_refusal_reason
+      return 'Redirect loop detected' if visited.include?(response_url)
+      return "#{hops + 1} redirects is too many" if hops >= options[:max_redirects]
 
-    def increment_redirects
-      @redirects = redirects + 1
+      nil
     end
 
-    def redirects
-      @redirects ||= 0
+    def follow_hop
+      visited << response_url
+      @hops = hops + 1
+      self.url = response_url
+      nil
     end
 
-    def handle_canonical_found
-      self.last_known_good = response.response
-      return true if response_url == url || redirect_list.include?(response_url)
-
-      set_url_from_response
-      false
+    def visited
+      @visited ||= [url]
     end
 
-    def set_url_from_response
-      self.url = response_url
+    def hops
+      @hops ||= 0
     end
 
     def handle_failure
-      return true if last_known_good
+      return last_known_good if last_known_good
 
-      raise URLCanonicalize::Exception::Failure, "#{response.failure_class}: #{response.message}"
+      raise URLCanonicalize::Exception::Failure, "#{response.failure_class}: #{response.message}",
+            cause: response.error
     end
 
     def handle_unhandled_response
@@ -125,7 +134,6 @@ module URLCanonicalize
 
     def handle_success
       self.last_known_good = response
-      true
     end
 
     def url
@@ -133,46 +141,65 @@ module URLCanonicalize
     end
 
     def http
-      return @http if same_host_and_port # reuse connection
+      URLCanonicalize::Destination.validate!(uri, options)
+      return @http if same_origin? # reuse connection
 
       @previous = uri
       @http = new_http
     end
 
-    def same_host_and_port
-      uri.host == previous.host && uri.port == previous.port
+    def same_origin?
+      uri.scheme == previous.scheme && uri.host == previous.host && uri.port == previous.port
     end
 
     def previous
-      @previous ||= Struct.new(:host, :port).new
+      @previous ||= Struct.new(:scheme, :host, :port).new
     end
 
     def new_http
-      h = Net::HTTP.new uri.host, uri.port
-
-      h.open_timeout = options[:open_timeout]
-      h.read_timeout = options[:read_timeout]
+      h = Net::HTTP.new uri.host, uri.port, nil
 
+      h.ipaddr = URLCanonicalize::Destination.resolve(uri, options)
+      configure_timeouts(h)
       ssl!(h)
 
       h
     end
 
+    def read_response_body(http_request, response)
+      if buffer_response_body?(http_request, response)
+        enforce_content_length!(response)
+        response.read_body(URLCanonicalize::BoundedBody.new(options[:max_body_bytes]))
+      else
+        response.read_body { |_chunk| nil }
+      end
+    end
+
+    def buffer_response_body?(http_request, response)
+      http_request.method == 'GET' && response.is_a?(Net::HTTPSuccess) && URLCanonicalize::MediaType.buffered?(response)
+    end
+
+    def enforce_content_length!(response)
+      return unless response.content_length && response.content_length > options[:max_body_bytes]
+
+      raise URLCanonicalize::Exception::ResponseTooLarge,
+            "Response body exceeds #{options[:max_body_bytes]} bytes"
+    end
+
+    def configure_timeouts(http)
+      http.open_timeout = options[:open_timeout]
+      http.read_timeout = options[:read_timeout]
+      http.write_timeout = options[:write_timeout]
+    end
+
     def ssl!(http)
       if uri.scheme == 'https'
         http.use_ssl = true # Can generate exception
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        http.verify_hostname = true
       else
         http.use_ssl = false
       end
     end
-
-    def options
-      @options ||= {
-        open_timeout: 8, # Twitter responds in >5s
-        read_timeout: 15,
-        max_redirects: 10
-      }
-    end
   end
 end

data/lib/url_canonicalize/link_header.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # Parses HTTP Link header fields (RFC 8288) far enough to find the target of
+  # the first link whose rel tokens include "canonical"
+  module LinkHeader
+    LINK_VALUE = /<([^>]*)>((?:\s*;\s*[\w!#$%&'*+.^`|~-]+\s*=\s*(?:"[^"]*"|[^",;]+))*)/
+    REL_PARAM = /;\s*rel\s*=\s*(?:"([^"]*)"|([^",;\s]+))/i
+    CANONICAL = 'canonical'
+
+    extend self
+
+    def canonical(response)
+      fields = response.get_fields('link')
+      return unless fields
+
+      fields.each do |field|
+        field.scan(LINK_VALUE) do |target, params|
+          return target if canonical?(params)
+        end
+      end
+
+      nil
+    end
+
+    def canonical?(params)
+      match = REL_PARAM.match(params)
+      return false unless match
+
+      rel = match[1] || match[2]
+      rel.split.any? { |token| token.casecmp?(CANONICAL) }
+    end
+  end
+end

data/lib/url_canonicalize/media_type.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # Normalizes response media types for safe buffering and parsing decisions
+  module MediaType
+    HTML = %w[text/html application/xhtml+xml].freeze
+    BUFFERED = (HTML + %w[application/xml text/xml]).freeze
+
+    extend self
+
+    def buffered?(response)
+      BUFFERED.include?(normalized(response))
+    end
+
+    def html?(response)
+      HTML.include?(normalized(response))
+    end
+
+    def normalized(response)
+      response['content-type'].to_s.split(';', 2).first.to_s.strip.downcase
+    end
+  end
+end

data/lib/url_canonicalize/options.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # Validated configuration for one canonicalization operation
+  class Options
+    DEFAULTS = {
+      allow_private_networks: false,
+      allowed_ports: [80, 443].freeze,
+      max_body_bytes: 1_048_576,
+      total_timeout: 30,
+      open_timeout: 8,
+      read_timeout: 15,
+      write_timeout: 8,
+      max_redirects: 10
+    }.freeze
+
+    POSITIVE_NUMERIC_OPTIONS = %i[total_timeout open_timeout read_timeout write_timeout].freeze
+    POSITIVE_INTEGER_OPTIONS = %i[max_body_bytes max_redirects].freeze
+
+    def initialize(**values)
+      validate_known_options!(values)
+
+      @values = DEFAULTS.merge(values)
+      validate!
+      @values[:allowed_ports] = @values[:allowed_ports].dup.freeze
+      @values.freeze
+      freeze
+    end
+
+    def [](key)
+      @values.fetch(key)
+    end
+
+    def to_h
+      @values.merge(allowed_ports: @values[:allowed_ports].dup)
+    end
+
+    private
+
+    def validate!
+      validate_private_networks!
+      validate_ports!
+      validate_positive_values!
+    end
+
+    def validate_known_options!(values)
+      unknown = values.keys - DEFAULTS.keys
+      raise ArgumentError, "Unknown options: #{unknown.join(', ')}" unless unknown.empty?
+    end
+
+    def validate_private_networks!
+      return if [true, false].include?(@values[:allow_private_networks])
+
+      raise ArgumentError, 'allow_private_networks must be true or false'
+    end
+
+    def validate_ports!
+      ports = @values[:allowed_ports]
+      valid = ports.is_a?(Array) && !ports.empty? && ports.all? do |port|
+        port.is_a?(Integer) && port.between?(1, 65_535)
+      end
+      return if valid
+
+      raise ArgumentError, 'allowed_ports must contain only valid TCP ports'
+    end
+
+    def validate_positive_values!
+      POSITIVE_NUMERIC_OPTIONS.each do |name|
+        value = @values[name]
+        raise ArgumentError, "#{name} must be positive" unless value.is_a?(Numeric) && value.positive?
+      end
+
+      POSITIVE_INTEGER_OPTIONS.each do |name|
+        value = @values[name]
+        raise ArgumentError, "#{name} must be positive" unless value.is_a?(Integer) && value.positive?
+      end
+    end
+  end
+end

data/lib/url_canonicalize/request.rb

@@ -3,6 +3,30 @@
 module URLCanonicalize
   # Make an HTTP request
   class Request
+    NETWORK_EXCEPTIONS = [
+      EOFError,
+      Errno::ECONNREFUSED,
+      Errno::ECONNRESET,
+      Errno::EHOSTUNREACH,
+      Errno::EINVAL,
+      Errno::ENETUNREACH,
+      Errno::ETIMEDOUT,
+      Net::OpenTimeout,
+      Net::ReadTimeout,
+      OpenSSL::SSL::SSLError,
+      SocketError,
+      Timeout::Error,
+      Zlib::BufError,
+      Zlib::DataError
+    ].freeze
+
+    # Unsuccessful responses to a HEAD request that trigger a retry with GET
+    HEAD_FALLBACK_RESPONSES = [
+      Net::HTTPForbidden,
+      Net::HTTPMethodNotAllowed,
+      Net::HTTPNotImplemented
+    ].freeze
+
     def fetch
       handle_response
     end
@@ -11,15 +35,13 @@ module URLCanonicalize
       @location ||= relative_to_absolute(response['location'])
     end
 
+    # Point this request at a new URI, discarding all state from the previous
+    # response so nothing leaks between hops
     def with_uri(uri)
       @uri = uri
 
       @url = nil
-      @host = nil
-      @request = nil
-      @response = nil
-      @location = nil
-      @html = nil
+      self.http_method = @default_http_method
 
       self
     end
@@ -31,6 +53,7 @@ module URLCanonicalize
     def initialize(http, http_method = :head)
       @http = http
       @http_method = http_method
+      @default_http_method = http_method
     end
 
     def response
@@ -55,41 +78,51 @@ module URLCanonicalize
       when Net::HTTPRedirection
         handle_redirection
       else
-        handle_failure
+        handle_unsuccessful
       end
     rescue *NETWORK_EXCEPTIONS => e
-      handle_failure(e.class, e.message)
+      handle_failure(e.class, e.message, e)
     end
 
     def handle_success
-      @canonical_url = $LAST_MATCH_INFO['url'] if (response['link'] || '') =~ /<(?<url>.+)>\s*;\s*rel="canonical"/i
+      @canonical_url = relative_to_absolute(URLCanonicalize::LinkHeader.canonical(response))
 
       return enhanced_response if canonical_url || http_method == :get
 
+      fallback_to_get
+    end
+
+    # Some servers reject HEAD requests outright, so any HEAD request rejected
+    # with one of these statuses is retried as a GET before giving up
+    def handle_unsuccessful
+      return fallback_to_get if http_method == :head && HEAD_FALLBACK_RESPONSES.any? { |klass| response.is_a?(klass) }
+
+      handle_failure
+    end
+
+    def fallback_to_get
       self.http_method = :get
       fetch
     end
 
+    # Follow any redirection that carries a usable Location header, whether the
+    # redirect is temporary or permanent. A redirection without one cannot be
+    # followed, so it is reported as a failure.
     def handle_redirection
-      case response
-      when Net::HTTPFound, Net::HTTPMovedTemporarily, Net::HTTPTemporaryRedirect # Temporary redirection
-        handle_success
-      else # Permanent redirection
-        if location
-          URLCanonicalize::Response::Redirect.new(location)
-        else
-          URLCanonicalize::Response::Failure.new(::URI::InvalidURIError, response['location'])
-        end
+      if location
+        URLCanonicalize::Response::Redirect.new(location)
+      else
+        URLCanonicalize::Response::Failure.new(::URI::InvalidURIError, response['location'])
       end
     end
 
-    def handle_failure(klass = response.class, message = response.message)
-      URLCanonicalize::Response::Failure.new(klass, message)
+    def handle_failure(klass = response.class, message = response.message, error = nil)
+      URLCanonicalize::Response::Failure.new(klass, message, error)
     end
 
     def enhanced_response
       if canonical_url
-        puts "  * canonical_url:\t#{canonical_url}" if ENV['DEBUG']
+        puts "  * canonical_url:\t#{canonical_url}" if ENV.fetch('DEBUG', nil)
         response_plus = URLCanonicalize::Response::Success.new(canonical_url, response, html)
         URLCanonicalize::Response::CanonicalFound.new(canonical_url, response_plus)
       else
@@ -98,19 +131,26 @@ module URLCanonicalize
     end
 
     def html
-      @html ||= Nokogiri::HTML response.body
+      @html ||= Nokogiri::HTML(response.body) if URLCanonicalize::MediaType.html?(response)
     end
 
     def canonical_url
-      @canonical_url ||= relative_to_absolute(canonical_url_raw)
+      @canonical_url ||= relative_to_absolute(canonical_url_element&.[]('href'), document_base_url)
     end
 
-    def canonical_url_raw
-      @canonical_url ||= canonical_url_element['href'] if canonical_url_element.is_a?(Nokogiri::XML::Element)
+    # The first HTML link element whose rel tokens include "canonical", however
+    # the tokens are cased, provided it has a usable href
+    def canonical_url_element
+      @canonical_url_element ||= html&.css('head link[rel]')&.find do |element|
+        element['rel'].split.any? { |token| token.casecmp?('canonical') } && !element['href'].to_s.strip.empty?
+      end
     end
 
-    def canonical_url_element
-      @canonical_url_element ||= html.xpath('//head/link[@rel="canonical"]').first
+    # HTML canonical links resolve against the document base URL, not the
+    # request URL, when the document declares one
+    def document_base_url
+      base_href = html&.at_css('head base[href]')&.[]('href')
+      relative_to_absolute(base_href) || url
     end
 
     def uri
@@ -121,10 +161,6 @@ module URLCanonicalize
       @url ||= uri.to_s
     end
 
-    def host
-      @host ||= uri.host
-    end
-
     def request_for_method
       r = base_request
       headers.each { |header_key, header_value| r[header_key] = header_value }
@@ -132,8 +168,6 @@ module URLCanonicalize
     end
 
     def base_request
-      check_http_method
-
       case http_method
       when :head
         Net::HTTP::Head.new uri
@@ -147,8 +181,8 @@ module URLCanonicalize
     def headers
       @headers ||= {
         'Accept-Language' => 'en-US,en;q=0.8',
-        'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) '\
-                        'AppleWebKit/537.36 (KHTML, like Gecko) '\
+        'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) ' \
+                        'AppleWebKit/537.36 (KHTML, like Gecko) ' \
                         'Chrome/51.0.2704.103 Safari/537.36'
       }
     end
@@ -159,54 +193,31 @@ module URLCanonicalize
       @response = nil
       @location = nil
       @html = nil
+      @canonical_url = nil
+      @canonical_url_element = nil
     end
 
-    # Some sites treat HEAD requests as suspicious activity and block the
-    # requester after a few attempts. For these sites we'll use GET requests
-    # only
-    def check_http_method
-      @http_method = :get if /(linkedin|crunchbase).com/ =~ host
-    end
+    # Resolve absolute, protocol-relative, root-relative, path-relative and
+    # query-only references against the base URL per RFC 3986. Only http(s)
+    # results are usable
+    def relative_to_absolute(reference, base = url)
+      return unless reference
 
-    def relative_to_absolute(partial_url)
-      return unless partial_url
-
-      partial_uri = ::URI.parse(partial_url)
-
-      if partial_uri.host
-        partial_url # It's already absolute
-      else
-        ::URI.join((uri || url), partial_url).to_s
-      end
-    rescue ::URI::InvalidURIError
+      absolute = ::URI.join(base, reference.strip)
+      absolute.to_s if absolute.is_a?(::URI::HTTP)
+    rescue ::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError
       nil
     end
 
     def log_response
-      return unless ENV['DEBUG']
+      debug = ENV.fetch('DEBUG', nil)
+      return unless debug
 
       puts "#{http_method.upcase} #{url} #{response.code} #{response.message}"
 
-      return unless ENV['DEBUG'].casecmp('headers')
+      return unless debug.casecmp?('headers')
 
       response.each { |k, v| puts "  #{k}:\t#{v}" }
     end
-
-    NETWORK_EXCEPTIONS = [
-      EOFError,
-      Errno::ECONNREFUSED,
-      Errno::ECONNRESET,
-      Errno::EHOSTUNREACH,
-      Errno::EINVAL,
-      Errno::ENETUNREACH,
-      Errno::ETIMEDOUT,
-      Net::OpenTimeout,
-      Net::ReadTimeout,
-      OpenSSL::SSL::SSLError,
-      SocketError,
-      Timeout::Error,
-      Zlib::BufError,
-      Zlib::DataError
-    ].freeze
   end
 end

data/lib/url_canonicalize/response.rb

@@ -13,7 +13,8 @@ module URLCanonicalize
       end
     end
 
-    Redirect = Class.new(Generic)
+    # A redirect to another URL
+    class Redirect < Generic; end
 
     # Add HTML to a successful response
     class Success < Generic
@@ -28,7 +29,7 @@ module URLCanonicalize
       def initialize(url, response, html)
         @response = response
         @html = html
-        super url
+        super(url)
       end
     end
 
@@ -40,19 +41,21 @@ module URLCanonicalize
 
       def initialize(url, response)
         @response = response
-        super url
+        super(url)
       end
     end
 
-    # It barfed
+    # It barfed. When the failure came from a rescued exception, `error` holds
+    # the original exception so callers can see the real cause
     class Failure
-      attr_reader :failure_class, :message
+      attr_reader :failure_class, :message, :error
 
       private
 
-      def initialize(failure_class, message)
+      def initialize(failure_class, message, error = nil)
         @failure_class = failure_class
         @message = message
+        @error = error
       end
     end
   end