url_canonicalize 1.0.0 → 2.0.0

data/lib/url_canonicalize/request.rb

@@ -3,6 +3,30 @@
 module URLCanonicalize
   # Make an HTTP request
   class Request
+    NETWORK_EXCEPTIONS = [
+      EOFError,
+      Errno::ECONNREFUSED,
+      Errno::ECONNRESET,
+      Errno::EHOSTUNREACH,
+      Errno::EINVAL,
+      Errno::ENETUNREACH,
+      Errno::ETIMEDOUT,
+      Net::OpenTimeout,
+      Net::ReadTimeout,
+      OpenSSL::SSL::SSLError,
+      SocketError,
+      Timeout::Error,
+      Zlib::BufError,
+      Zlib::DataError
+    ].freeze
+
+    # Unsuccessful responses to a HEAD request that trigger a retry with GET
+    HEAD_FALLBACK_RESPONSES = [
+      Net::HTTPForbidden,
+      Net::HTTPMethodNotAllowed,
+      Net::HTTPNotImplemented
+    ].freeze
+
     def fetch
       handle_response
     end
@@ -11,15 +35,13 @@ module URLCanonicalize
       @location ||= relative_to_absolute(response['location'])
     end
 
+    # Point this request at a new URI, discarding all state from the previous
+    # response so nothing leaks between hops
     def with_uri(uri)
       @uri = uri
 
       @url = nil
-      @host = nil
-      @request = nil
-      @response = nil
-      @location = nil
-      @html = nil
+      self.http_method = @default_http_method
 
       self
     end
@@ -31,6 +53,7 @@ module URLCanonicalize
     def initialize(http, http_method = :head)
       @http = http
       @http_method = http_method
+      @default_http_method = http_method
     end
 
     def response
@@ -55,41 +78,51 @@ module URLCanonicalize
       when Net::HTTPRedirection
         handle_redirection
       else
-        handle_failure
+        handle_unsuccessful
       end
     rescue *NETWORK_EXCEPTIONS => e
-      handle_failure(e.class, e.message)
+      handle_failure(e.class, e.message, e)
     end
 
     def handle_success
-      @canonical_url = $LAST_MATCH_INFO['url'] if (response['link'] || '') =~ /<(?<url>.+)>\s*;\s*rel="canonical"/i
+      @canonical_url = normalized(relative_to_absolute(URLCanonicalize::LinkHeader.canonical(response)))
 
       return enhanced_response if canonical_url || http_method == :get
 
+      fallback_to_get
+    end
+
+    # Some servers reject HEAD requests outright, so any HEAD request rejected
+    # with one of these statuses is retried as a GET before giving up
+    def handle_unsuccessful
+      return fallback_to_get if http_method == :head && HEAD_FALLBACK_RESPONSES.any? { |klass| response.is_a?(klass) }
+
+      handle_failure
+    end
+
+    def fallback_to_get
       self.http_method = :get
       fetch
     end
 
+    # Follow any redirection that carries a usable Location header, whether the
+    # redirect is temporary or permanent. A redirection without one cannot be
+    # followed, so it is reported as a failure.
     def handle_redirection
-      case response
-      when Net::HTTPFound, Net::HTTPMovedTemporarily, Net::HTTPTemporaryRedirect # Temporary redirection
-        handle_success
-      else # Permanent redirection
-        if location
-          URLCanonicalize::Response::Redirect.new(location)
-        else
-          URLCanonicalize::Response::Failure.new(::URI::InvalidURIError, response['location'])
-        end
+      if location
+        URLCanonicalize::Response::Redirect.new(location)
+      else
+        URLCanonicalize::Response::Failure.new(::URI::InvalidURIError, response['location'])
       end
     end
 
-    def handle_failure(klass = response.class, message = response.message)
-      URLCanonicalize::Response::Failure.new(klass, message)
+    def handle_failure(klass = response.class, message = response.message, error = nil)
+      URLCanonicalize::Response::Failure.new(klass, message, error)
     end
 
     def enhanced_response
       if canonical_url
-        puts "  * canonical_url:\t#{canonical_url}" if ENV['DEBUG']
+        logger&.debug { "canonical_url: #{canonical_url}" }
         response_plus = URLCanonicalize::Response::Success.new(canonical_url, response, html)
         URLCanonicalize::Response::CanonicalFound.new(canonical_url, response_plus)
       else
@@ -98,19 +131,32 @@ module URLCanonicalize
     end
 
     def html
-      @html ||= Nokogiri::HTML response.body
+      @html ||= Nokogiri::HTML(response.body) if URLCanonicalize::MediaType.html?(response)
     end
 
     def canonical_url
-      @canonical_url ||= relative_to_absolute(canonical_url_raw)
+      @canonical_url ||= normalized(relative_to_absolute(canonical_url_element&.[]('href'), document_base_url))
     end
 
-    def canonical_url_raw
-      @canonical_url ||= canonical_url_element['href'] if canonical_url_element.is_a?(Nokogiri::XML::Element)
+    # Declared canonical URLs are returned to callers, so they get the same
+    # syntactic normalization as every requested URL
+    def normalized(url)
+      URLCanonicalize::URI.normalize(url) if url
     end
 
+    # The first HTML link element whose rel tokens include "canonical", however
+    # the tokens are cased, provided it has a usable href
     def canonical_url_element
-      @canonical_url_element ||= html.xpath('//head/link[@rel="canonical"]').first
+      @canonical_url_element ||= html&.css('head link[rel]')&.find do |element|
+        element['rel'].split.any? { |token| token.casecmp?('canonical') } && !element['href'].to_s.strip.empty?
+      end
+    end
+
+    # HTML canonical links resolve against the document base URL, not the
+    # request URL, when the document declares one
+    def document_base_url
+      base_href = html&.at_css('head base[href]')&.[]('href')
+      relative_to_absolute(base_href) || url
     end
 
     def uri
@@ -121,10 +167,6 @@ module URLCanonicalize
       @url ||= uri.to_s
     end
 
-    def host
-      @host ||= uri.host
-    end
-
     def request_for_method
       r = base_request
       headers.each { |header_key, header_value| r[header_key] = header_value }
@@ -132,8 +174,6 @@ module URLCanonicalize
     end
 
     def base_request
-      check_http_method
-
       case http_method
       when :head
         Net::HTTP::Head.new uri
@@ -145,12 +185,7 @@ module URLCanonicalize
     end
 
     def headers
-      @headers ||= {
-        'Accept-Language' => 'en-US,en;q=0.8',
-        'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) ' \
-                        'AppleWebKit/537.36 (KHTML, like Gecko) ' \
-                        'Chrome/51.0.2704.103 Safari/537.36'
-      }
+      http.options[:headers]
     end
 
     def http_method=(value)
@@ -159,54 +194,28 @@ module URLCanonicalize
       @response = nil
       @location = nil
       @html = nil
+      @canonical_url = nil
+      @canonical_url_element = nil
     end
 
-    # Some sites treat HEAD requests as suspicious activity and block the
-    # requester after a few attempts. For these sites we'll use GET requests
-    # only
-    def check_http_method
-      @http_method = :get if /(linkedin|crunchbase).com/ =~ host
-    end
-
-    def relative_to_absolute(partial_url)
-      return unless partial_url
+    # Resolve absolute, protocol-relative, root-relative, path-relative and
+    # query-only references against the base URL per RFC 3986. Only http(s)
+    # results are usable
+    def relative_to_absolute(reference, base = url)
+      return unless reference
 
-      partial_uri = ::URI.parse(partial_url)
-
-      if partial_uri.host
-        partial_url # It's already absolute
-      else
-        ::URI.join((uri || url), partial_url).to_s
-      end
-    rescue ::URI::InvalidURIError
+      absolute = ::URI.join(base, reference.strip)
+      absolute.to_s if absolute.is_a?(::URI::HTTP)
+    rescue ::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError
       nil
     end
 
     def log_response
-      return unless ENV['DEBUG']
-
-      puts "#{http_method.upcase} #{url} #{response.code} #{response.message}"
-
-      return unless ENV['DEBUG'].casecmp('headers')
-
-      response.each { |k, v| puts "  #{k}:\t#{v}" }
+      logger&.debug { "#{http_method.upcase} #{url} #{response.code} #{response.message}" }
     end
 
-    NETWORK_EXCEPTIONS = [
-      EOFError,
-      Errno::ECONNREFUSED,
-      Errno::ECONNRESET,
-      Errno::EHOSTUNREACH,
-      Errno::EINVAL,
-      Errno::ENETUNREACH,
-      Errno::ETIMEDOUT,
-      Net::OpenTimeout,
-      Net::ReadTimeout,
-      OpenSSL::SSL::SSLError,
-      SocketError,
-      Timeout::Error,
-      Zlib::BufError,
-      Zlib::DataError
-    ].freeze
+    def logger
+      http.options[:logger]
+    end
   end
 end

data/lib/url_canonicalize/response.rb

@@ -13,7 +13,8 @@ module URLCanonicalize
       end
     end
 
-    Redirect = Class.new(Generic)
+    # A redirect to another URL
+    class Redirect < Generic; end
 
     # Add HTML to a successful response
     class Success < Generic
@@ -44,15 +45,17 @@ module URLCanonicalize
       end
     end
 
-    # It barfed
+    # It barfed. When the failure came from a rescued exception, `error` holds
+    # the original exception so callers can see the real cause
     class Failure
-      attr_reader :failure_class, :message
+      attr_reader :failure_class, :message, :error
 
       private
 
-      def initialize(failure_class, message)
+      def initialize(failure_class, message, error = nil)
         @failure_class = failure_class
         @message = message
+        @error = error
       end
     end
   end

data/lib/url_canonicalize/result.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # The outcome of one canonicalization: the canonical URL, the response
+  # that confirmed it, and the chain of hops that led there. Immutable
+  class Result
+    # One step in the request chain: the URL that was requested and how it
+    # was discovered (:initial, :redirect or :canonical_link)
+    Hop = Struct.new(:url, :via, keyword_init: true)
+
+    attr_reader :url, :response, :html, :chain, :source
+
+    def xml
+      Nokogiri::XML(response.body)
+    end
+
+    private
+
+    def initialize(url:, response:, html:, chain:, source:)
+      @url = url
+      @response = response
+      @html = html
+      @chain = chain.map(&:freeze).freeze
+      @source = source
+      freeze
+    end
+  end
+end

data/lib/url_canonicalize/transport.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # Builds the connection for a destination, resolving the host through the
+  # fetch security policy. Injectable via the :transport option: anything
+  # responding to call(uri, options) and returning a Net::HTTP-compatible
+  # connection can replace it
+  class Transport
+    # Resolved lazily so the destination policy can be stubbed in tests
+    DEFAULT_RESOLVER = lambda do |uri, options|
+      URLCanonicalize::Destination.resolve(uri, options)
+    end
+
+    def initialize(resolver: DEFAULT_RESOLVER)
+      @resolver = resolver
+      freeze
+    end
+
+    def call(uri, options)
+      http = Net::HTTP.new(uri.host, uri.port, nil)
+
+      http.ipaddr = @resolver.call(uri, options)
+      configure_timeouts(http, options)
+      configure_tls(http, uri)
+
+      http
+    end
+
+    private
+
+    def configure_timeouts(http, options)
+      http.open_timeout = options[:open_timeout]
+      http.read_timeout = options[:read_timeout]
+      http.write_timeout = options[:write_timeout]
+    end
+
+    def configure_tls(http, uri)
+      if uri.scheme == 'https'
+        http.use_ssl = true # Can generate exception
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        http.verify_hostname = true
+      else
+        http.use_ssl = false
+      end
+    end
+  end
+end

data/lib/url_canonicalize/uri.rb

@@ -1,36 +1,49 @@
 # frozen_string_literal: true
 
 module URLCanonicalize
-  # Manage the URL into a URI with local exception handling
+  # Parses and normalizes URLs with local exception handling
   class URI
     class << self
+      VALID_CLASSES = [::URI::HTTP, ::URI::HTTPS].freeze
+
+      # A scheme is recognized only as a leading scheme: token; a colon
+      # anywhere else no longer suppresses the default scheme
+      SCHEME = /\A[A-Za-z][A-Za-z0-9+.-]*:/
+
       def parse(url)
-        uri = ::URI.parse decorate(url)
-        uri if valid? uri
+        uri = ::URI.parse normalize(url)
+        validate! uri
+        uri
       rescue ::URI::InvalidURIError => e
-        new_exception = URLCanonicalize::Exception::URI.new("#{e.class}: #{e.message}")
-        new_exception.set_backtrace e.backtrace
-        raise new_exception
+        raise URLCanonicalize::Exception::URI, "#{e.class}: #{e.message}" # the original error becomes the cause
+      end
+
+      # Syntactic normalization per RFC 3986, via Addressable: lowercase
+      # scheme and host, internationalized hosts to punycode, uppercase
+      # percent-encodings with unreserved characters decoded, dot segments
+      # resolved, default ports removed and the empty path replaced by "/".
+      # Fragments are client-side state, so no canonical URL carries one
+      def normalize(url)
+        addressable = Addressable::URI.parse(decorate(url.to_s.strip)).normalize
+        addressable.fragment = nil
+        addressable.to_s
+      rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError => e
+        raise URLCanonicalize::Exception::URI, "#{e.class}: #{e.message}" # the original error becomes the cause
       end
 
       private
 
-      def valid?(uri)
+      def validate!(uri)
         raise URLCanonicalize::Exception::URI, "#{uri} must be http or https" unless VALID_CLASSES.include?(uri.class)
         raise URLCanonicalize::Exception::URI, "Missing host name in #{uri}" unless uri.host
         raise URLCanonicalize::Exception::URI, "Empty host name in #{uri}" if uri.host.empty?
-
-        true
       end
 
       def decorate(url)
-        return url if url.include? COLON
+        return url if url.match?(SCHEME)
 
         "http://#{url}" # Add protocol if we just receive a host name
       end
-
-      VALID_CLASSES = [::URI::HTTP, ::URI::HTTPS].freeze
-      COLON = ':'
     end
   end
 end

data/lib/url_canonicalize/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module URLCanonicalize
-  VERSION = '1.0.0'
+  VERSION = '2.0.0'
 end

data/lib/url_canonicalize.rb

@@ -4,30 +4,34 @@ require 'uri'
 require 'addressable/uri'
 require 'net/http'
 require 'nokogiri'
+require 'timeout'
 
 autoload :OpenSSL, 'openssl'
 
 # Core methods
 module URLCanonicalize
+  autoload :BoundedBody, 'url_canonicalize/bounded_body'
+  autoload :Client, 'url_canonicalize/client'
+  autoload :Destination, 'url_canonicalize/destination'
   autoload :Exception, 'url_canonicalize/exception'
   autoload :HTTP, 'url_canonicalize/http'
+  autoload :LinkHeader, 'url_canonicalize/link_header'
+  autoload :MediaType, 'url_canonicalize/media_type'
+  autoload :Options, 'url_canonicalize/options'
   autoload :Request, 'url_canonicalize/request'
   autoload :Response, 'url_canonicalize/response'
+  autoload :Result, 'url_canonicalize/result'
+  autoload :Transport, 'url_canonicalize/transport'
   autoload :URI, 'url_canonicalize/uri'
   autoload :VERSION, 'url_canonicalize/version'
 
   class << self
-    def canonicalize(url)
-      fetch(url).url
+    def canonicalize(url, **)
+      Client.new(**).canonicalize(url)
     end
 
-    def fetch(url)
-      URLCanonicalize::HTTP.new(url).fetch
+    def fetch(url, **)
+      Client.new(**).fetch(url)
     end
   end
 end
-
-require 'monkey_patches/uri'
-require 'monkey_patches/string'
-require 'monkey_patches/addressable/uri'
-require 'English' # Needed for $LAST_MATCH_INFO

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: url_canonicalize
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Dominic Sayers
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-01-27 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -46,40 +45,39 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".codeclimate.yml"
-- ".github/workflows/codeql-analysis.yml"
-- ".gitignore"
-- ".hound.yml"
-- ".rspec"
-- ".rubocop.yml"
-- ".ruby-gemset"
-- ".travis.yml"
 - CHANGELOG.md
-- Gemfile
-- Gemfile.lock
-- Guardfile
 - LICENSE
 - README.md
-- Rakefile
-- circle.yml
-- lib/monkey_patches/addressable/uri.rb
-- lib/monkey_patches/string.rb
-- lib/monkey_patches/uri.rb
+- SECURITY.md
 - lib/url_canonicalize.rb
+- lib/url_canonicalize/bounded_body.rb
+- lib/url_canonicalize/client.rb
+- lib/url_canonicalize/core_ext.rb
+- lib/url_canonicalize/core_ext/addressable.rb
+- lib/url_canonicalize/core_ext/string.rb
+- lib/url_canonicalize/core_ext/uri.rb
+- lib/url_canonicalize/destination.rb
 - lib/url_canonicalize/exception.rb
 - lib/url_canonicalize/http.rb
+- lib/url_canonicalize/link_header.rb
+- lib/url_canonicalize/media_type.rb
+- lib/url_canonicalize/options.rb
 - lib/url_canonicalize/request.rb
 - lib/url_canonicalize/response.rb
+- lib/url_canonicalize/result.rb
+- lib/url_canonicalize/transport.rb
 - lib/url_canonicalize/uri.rb
 - lib/url_canonicalize/version.rb
-- renovate.json
-- url_canonicalize.gemspec
 homepage: https://github.com/dominicsayers/url_canonicalize
 licenses:
 - MIT
 metadata:
+  bug_tracker_uri: https://github.com/dominicsayers/url_canonicalize/issues
+  changelog_uri: https://github.com/dominicsayers/url_canonicalize/blob/main/CHANGELOG.md
+  documentation_uri: https://www.rubydoc.info/gems/url_canonicalize
+  homepage_uri: https://github.com/dominicsayers/url_canonicalize
+  source_code_uri: https://github.com/dominicsayers/url_canonicalize
   rubygems_mfa_required: 'true'
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -87,15 +85,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
-signing_key: 
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Finds the canonical version of a URL
 test_files: []

data/.codeclimate.yml

@@ -1,25 +0,0 @@
----
-engines:
-  duplication:
-    enabled: true
-    config:
-      languages:
-      - ruby
-      - javascript
-      - python
-      - php
-  fixme:
-    enabled: true
-  rubocop:
-    enabled: true
-ratings:
-  paths:
-  - "**.inc"
-  - "**.js"
-  - "**.jsx"
-  - "**.module"
-  - "**.php"
-  - "**.py"
-  - "**.rb"
-exclude_paths:
-  - spec/

data/.github/workflows/codeql-analysis.yml

@@ -1,70 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ main ]
-  schedule:
-    - cron: '28 12 * * 5'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'ruby' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://git.io/codeql-language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3