url_canonicalize 1.0.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b8529a2f607ea6c3be592b855fa12bb79256bc62a11dfb94aa7b2d6aec1b45f
-  data.tar.gz: 54a13bd40248b52cab8b3058fe04216bef20598896431cd90dd7e4cb4089d67f
+  metadata.gz: 503c645a9625985b4391f2743587f148d750d09d43d473c7a9923be2a2ec8c7f
+  data.tar.gz: 3983c156ec6754bac592880a968a07af984a5c11109ab31d096376c0774584a2
 SHA512:
-  metadata.gz: 76310623c993c4550972e0b08bb28516c1673d246f3b0173738fbdb8e65f416d6375b7174d83e513d640a7c71c56de06fc115e2e91bb37a3d1e583c12a0231a6
-  data.tar.gz: a68a7ee377c1cf90d162ee44a76c54a01936e0e1ace7055c0cd084a89bf3ade561060651aed21231f23a8e4d0cc129384534d9b9ced4c7dffb7e2a0bbaef7d9d
+  metadata.gz: b85dc8c4af5866b0bd9d79c92181672d3401419587f0d6868fdce28cfc2c1e973c86e208c633d3062b09dcad9e9e4794badff12b157c5e6cf0a3a95a0426f041
+  data.tar.gz: 498beaf7ca0e0a56d0f2f9b3280c42b7417ff8a14f38453ae757a480cd78388e5391ed7c3896a596648796226be8e3ee745446a412217a51f36b3025bf2a6295

data/CHANGELOG.md

@@ -8,6 +8,60 @@ and this project adheres to
 
 ## [Unreleased]
 
+## [2.0.0] - 2026-07-25
+
+### Added
+
+- `URLCanonicalize::Client`, a reusable, thread-safe client whose options are
+  validated and frozen at construction; request headers, a logger and the
+  transport are configurable, so tests can inject a fake transport or
+  resolver
+  ([#186](https://github.com/dominicsayers/url_canonicalize/issues/186))
+- Documented, table-tested URL normalization per RFC 3986: lowercase scheme
+  and host, punycode hosts, normalized percent-encoding, resolved dot
+  segments, removed default ports and stripped fragments, applied to the
+  input, every hop and the returned URL
+  ([#187](https://github.com/dominicsayers/url_canonicalize/issues/187))
+- SonarQube quality gate badge in the README
+  ([#196](https://github.com/dominicsayers/url_canonicalize/issues/196))
+
+### Changed
+
+- **Breaking:** the core-class extensions (`String#canonicalize` and friends)
+  are opt-in via `require 'url_canonicalize/core_ext'`; requiring the gem
+  alone no longer patches anything
+  ([#186](https://github.com/dominicsayers/url_canonicalize/issues/186))
+- **Breaking:** `URLCanonicalize.fetch` returns an immutable
+  `URLCanonicalize::Result` exposing the canonical URL, the confirming
+  response, the parsed HTML, the request chain and the discovery source
+  ([#186](https://github.com/dominicsayers/url_canonicalize/issues/186))
+- **Breaking:** returned URLs are normalized (an empty path becomes `/`) and
+  a scheme is recognized only as a leading `scheme:` token, so a colon later
+  in a bare host's path no longer causes a parse error
+  ([#187](https://github.com/dominicsayers/url_canonicalize/issues/187))
+- **Breaking:** Ruby 3.3 or later is required; CI tests Ruby 3.3, 3.4 and
+  4.0 on GitHub Actions and the Travis and CircleCI configurations are gone
+  ([#188](https://github.com/dominicsayers/url_canonicalize/issues/188))
+- Debug output moved from the `DEBUG` environment variable to the `logger`
+  option
+  ([#186](https://github.com/dominicsayers/url_canonicalize/issues/186))
+- Update nokogiri to 1.19.4, fixing all eleven open Dependabot alerts
+  ([#210](https://github.com/dominicsayers/url_canonicalize/issues/210))
+- Replace the `bin/release` scripts with the
+  [`release_ceremony`](https://github.com/dominicsayers/release_ceremony) gem
+  they were extracted into; the release commands are now
+  `bundle exec release_ceremony prepare|publish VERSION`
+
+### Removed
+
+- Hound and CodeClimate configuration; RuboCop in CI and SonarQube Cloud
+  cover static analysis
+  ([#211](https://github.com/dominicsayers/url_canonicalize/issues/211),
+  [#195](https://github.com/dominicsayers/url_canonicalize/issues/195))
+- The RuboCop to-do file; its offences are fixed and the deliberate spec
+  limits are explicit policy in `.rubocop.yml`
+  ([#198](https://github.com/dominicsayers/url_canonicalize/issues/198))
+
 ## [1.0.2] - 2026-07-24
 
 ### Fixed
@@ -89,7 +143,8 @@ changelog format. Their history is available from the
 and the
 [commit log](https://github.com/dominicsayers/url_canonicalize/commits/v0.1.15).
 
-[Unreleased]: https://github.com/dominicsayers/url_canonicalize/compare/v1.0.2...HEAD
+[Unreleased]: https://github.com/dominicsayers/url_canonicalize/compare/v2.0.0...HEAD
+[2.0.0]: https://github.com/dominicsayers/url_canonicalize/compare/v1.0.2...v2.0.0
 [1.0.2]: https://github.com/dominicsayers/url_canonicalize/compare/v1.0.1...v1.0.2
 [1.0.1]: https://github.com/dominicsayers/url_canonicalize/compare/v1.0.0...v1.0.1
 [1.0.0]: https://github.com/dominicsayers/url_canonicalize/compare/v0.2.1...v1.0.0

data/README.md

@@ -1,14 +1,15 @@
 # URLCanonicalize
 [![Gem Version](https://img.shields.io/gem/v/url_canonicalize.svg)](https://rubygems.org/gems/url_canonicalize)
 [![Gem downloads](https://img.shields.io/gem/dt/url_canonicalize.svg)](https://rubygems.org/gems/url_canonicalize)
-[![CircleCI](https://dl.circleci.com/status-badge/img/gh/dominicsayers/url_canonicalize/tree/main.svg?style=shield)](https://dl.circleci.com/status-badge/redirect/gh/dominicsayers/url_canonicalize/tree/main)
+[![Test](https://github.com/dominicsayers/url_canonicalize/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/dominicsayers/url_canonicalize/actions/workflows/test.yml)
 [![CodeQL](https://github.com/dominicsayers/url_canonicalize/actions/workflows/codeql-analysis.yml/badge.svg?branch=main)](https://github.com/dominicsayers/url_canonicalize/actions/workflows/codeql-analysis.yml)
+[![Quality gate status](https://sonarcloud.io/api/project_badges/measure?project=dominicsayers_url_canonicalize&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=dominicsayers_url_canonicalize)
 
 URLCanonicalize is a Ruby gem that finds the canonical version of a URL. It
-provides `canonicalize` methods for the String, URI::HTTP, URI::HTTPS and
-Addressable::URI classes.
+provides a configurable client API, and optional `canonicalize` extension
+methods for the String, URI::HTTP, URI::HTTPS and Addressable::URI classes.
 
-Ruby 3.1 or later is required.
+Ruby 3.3 or later is required.
 
 ## Installation
 
@@ -20,33 +21,123 @@ gem 'url_canonicalize'
 
 ## Usage
 
-Requiring the gem extends `String`, `URI::HTTP`, `URI::HTTPS` and
-`Addressable::URI` with a `canonicalize` method:
+The primary API is a reusable client. Its options are validated and frozen at
+construction, so one client can be shared safely between threads:
 
 ```ruby
-'http://www.twitter.com'.canonicalize # => 'https://twitter.com/'
+client = URLCanonicalize::Client.new(
+  total_timeout: 10,
+  max_redirects: 8,
+  max_body_bytes: 512_000
+)
 
-URI('http://www.twitter.com').canonicalize # => #<URI::HTTP:0x00000008767908 URL:https://twitter.com/>
+client.canonicalize('http://www.twitter.com') # => 'https://twitter.com/'
+```
 
-Addressable::URI.canonicalize('http://www.twitter.com') # => #<Addressable::URI:0x43c9 URI:https://twitter.com/>
+For one-off calls the module methods accept the same options and build a
+client internally:
+
+```ruby
+URLCanonicalize.canonicalize('http://www.twitter.com') # => 'https://twitter.com/'
+URLCanonicalize.fetch('http://www.twitter.com')        # => #<URLCanonicalize::Result ...>
 ```
 
-Options can be passed to the module API or any of the `canonicalize` extension
-methods:
+`canonicalize` returns the canonical URL as a string. `fetch` returns an
+immutable `URLCanonicalize::Result` with:
+
+- `url` — the canonical URL;
+- `response` — the `Net::HTTPResponse` that confirmed it;
+- `html` — the parsed Nokogiri document when the response was HTML;
+- `chain` — the request chain: each hop's URL and how it was discovered
+  (`:initial`, `:redirect` or `:canonical_link`);
+- `source` — how the final URL was discovered.
+
+### Configuration
+
+Beyond the security limits described below, the client accepts:
+
+- `headers` — the request headers to send (replaces the default browser-like
+  `User-Agent` and `Accept-Language`);
+- `logger` — any object responding to `debug`; each request and discovered
+  canonical URL is logged. Nothing is logged by default;
+- `transport` — anything responding to `call(uri, options)` and returning a
+  `Net::HTTP`-compatible connection. The default transport resolves and
+  validates the destination address (see below) and configures TLS and
+  timeouts. Tests can inject a fake transport to avoid the network entirely;
+  a custom resolver can be injected with
+  `URLCanonicalize::Transport.new(resolver: ...)`.
+
+### Core-class extensions
+
+The core-class extensions are opt-in. Requiring `url_canonicalize/core_ext`
+extends `String`, `URI::HTTP`, `URI::HTTPS` and `Addressable::URI` with a
+`canonicalize` method:
 
 ```ruby
-URLCanonicalize.fetch(
-  'https://example.com/article',
-  total_timeout: 10,
-  max_body_bytes: 512_000
-)
+require 'url_canonicalize/core_ext'
+
+'http://www.twitter.com'.canonicalize # => 'https://twitter.com/'
+
+URI('http://www.twitter.com').canonicalize # => #<URI::HTTP:0x00000008767908 URL:https://twitter.com/>
+
+Addressable::URI.canonicalize('http://www.twitter.com') # => #<Addressable::URI:0x43c9 URI:https://twitter.com/>
 
 'https://example.com/article'.canonicalize(total_timeout: 10)
 ```
 
-If you prefer the module API without the core-class extensions, call
-`URLCanonicalize.canonicalize(url)` (returns the canonical URL string) or
-`URLCanonicalize.fetch(url)` (returns the full response object).
+Requiring `url_canonicalize` alone no longer patches any core class: if you
+relied on the extensions, add the extra require.
+
+## Normalization and canonicalization
+
+URLCanonicalize distinguishes two operations:
+
+- **Normalization** is syntactic: rewriting a URL into its RFC 3986 normal
+  form without any network access. Parsing and reference resolution use
+  Ruby's `URI` (strict); normalization uses `Addressable::URI#normalize`.
+- **Canonicalization** is discovery: fetching the URL, following redirects
+  and server-declared canonical links, and returning the normalized final
+  URL.
+
+Every URL is processed in this order:
+
+1. The input is stripped of surrounding whitespace and parsed. A leading
+   `scheme:` token is honored; an input without one is treated as a host
+   name and given `http://`. A colon elsewhere in the URL does not suppress
+   the default scheme. Non-HTTP(S) schemes raise
+   `URLCanonicalize::Exception::URI`.
+2. The URL is normalized (see below) and fetched with a `HEAD` request,
+   retried as `GET` where the server rejects `HEAD`.
+3. Redirects (any `3xx` with a usable `Location`, temporary or permanent)
+   are followed. The `Location` value is resolved against the request URL
+   per RFC 3986, so absolute, protocol-relative, root-relative,
+   path-relative and query-only forms all work.
+4. On a successful response, a canonical link is looked for in the `Link`
+   response header first, then in the HTML `<link rel="canonical">` element;
+   **the header wins when they conflict**. HTML links resolve against the
+   document's `<base href>` when one is declared, otherwise against the
+   request URL.
+5. A declared canonical URL is not trusted blindly: it is fetched and
+   validated itself. If fetching it fails, the declared canonical URL is
+   still returned, confirmed by the response that declared it.
+6. Redirects and followed canonical links share one hop budget and one
+   visited-URL set, so cycles and over-long chains terminate
+   deterministically. The returned URL is always normalized.
+
+Normalization makes these changes and no others:
+
+| Component | Rule |
+| --- | --- |
+| Scheme and host | Lowercased |
+| Internationalized host | Converted to punycode (`xn--…`) |
+| Default port (`:80`/`:443`) | Removed; other ports preserved |
+| Dot segments (`/./`, `/../`) | Resolved |
+| Percent-encoding | Uppercase hex; unreserved characters decoded; invalid characters encoded |
+| Empty path | Replaced by `/` |
+| Trailing slash | Preserved (`/a/` and `/a` stay distinct) |
+| Query string | Preserved verbatim — parameters are never reordered or dropped |
+| Fragment (`#…`) | Stripped — fragments are client-side state |
+| Userinfo (`user:pass@`) | Rejected as a security error at fetch time |
 
 ## Security and limits
 

data/lib/url_canonicalize/client.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # A reusable canonicalization client. The options are validated and frozen
+  # at construction, so one client can be shared safely between threads
+  class Client
+    attr_reader :options
+
+    def initialize(**values)
+      @options = URLCanonicalize::Options.new(**values)
+    end
+
+    def canonicalize(url)
+      fetch(url).url
+    end
+
+    def fetch(url)
+      URLCanonicalize::HTTP.new(url, options: options).fetch
+    end
+  end
+end

data/lib/{monkey_patches/addressable/uri.rb → url_canonicalize/core_ext/addressable.rb}

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
+require 'addressable/uri'
+
 module Addressable
   # Patch for Addressable's URI class
   class URI
-    def self.canonicalize(uri, **options)
+    def self.canonicalize(uri, **)
       url = parse(uri).to_s # uri can be anything Addressable::URI can handle
-      canonical_url = URLCanonicalize.canonicalize(url, **options)
+      canonical_url = URLCanonicalize.canonicalize(url, **)
       parse(canonical_url)
     end
   end

data/lib/{monkey_patches → url_canonicalize/core_ext}/string.rb

@@ -2,7 +2,7 @@
 
 # Patch for Ruby's String class
 class String
-  def canonicalize(**options)
-    URLCanonicalize.canonicalize(self, **options)
+  def canonicalize(**)
+    URLCanonicalize.canonicalize(self, **)
   end
 end

data/lib/{monkey_patches → url_canonicalize/core_ext}/uri.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+require 'uri'
+
 module URI
   # URI having the HTTP protocol
   class HTTP
-    def canonicalize(**options)
-      new_url = URLCanonicalize.canonicalize(to_s, **options)
+    def canonicalize(**)
+      new_url = URLCanonicalize.canonicalize(to_s, **)
       ::URI.parse(new_url)
     end
   end

data/lib/url_canonicalize/core_ext.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require 'url_canonicalize'
+require 'url_canonicalize/core_ext/string'
+require 'url_canonicalize/core_ext/uri'
+require 'url_canonicalize/core_ext/addressable'

data/lib/url_canonicalize/http.rb

@@ -4,11 +4,12 @@ module URLCanonicalize
   # Persistent connection for possible repeated requests to the same host
   class HTTP
     def fetch
-      ::Timeout.timeout(
+      final = ::Timeout.timeout(
         options[:total_timeout],
         URLCanonicalize::Exception::Timeout,
         "Canonicalization exceeded #{options[:total_timeout]} seconds"
       ) { fetch_within_deadline }
+      build_result(final)
     end
 
     def uri
@@ -24,10 +25,22 @@ module URLCanonicalize
       http.request(http_request) { |response| read_response_body(http_request, response) }
     end
 
+    attr_reader :options
+
     private
 
     attr_accessor :last_known_good
-    attr_reader :options
+
+    def initialize(raw_url, options: nil, **values)
+      raise ArgumentError, 'Pass either options: or keyword options, not both' if options && !values.empty?
+
+      @raw_url = raw_url
+      @options = options || URLCanonicalize::Options.new(**values)
+    end
+
+    def build_result(final)
+      Result.new(url: final.url, response: final.response, html: final.html, chain: chain, source: @final_source)
+    end
 
     def fetch_within_deadline
       loop do
@@ -36,11 +49,6 @@ module URLCanonicalize
       end
     end
 
-    def initialize(raw_url, **options)
-      @raw_url = raw_url
-      @options = URLCanonicalize::Options.new(**options)
-    end
-
     # Fetch the response
     def response
       @response ||= fetch_response
@@ -85,47 +93,60 @@ module URLCanonicalize
 
     def handle_redirect
       reason = hop_refusal_reason
-      return follow_hop unless reason
+      return follow_hop(:redirect) unless reason
 
-      last_known_good || raise(URLCanonicalize::Exception::Redirect, reason)
+      finish_with_last_known_good || raise(URLCanonicalize::Exception::Redirect, reason)
     end
 
     def handle_canonical_found
       self.last_known_good = response.response
-      return last_known_good if hop_refusal_reason
+      return finish_with_last_known_good if hop_refusal_reason
+
+      follow_hop(:canonical_link)
+    end
 
-      follow_hop
+    # A terminal response whose URL came from a server-declared canonical link
+    def finish_with_last_known_good
+      return unless last_known_good
+
+      @final_source = :canonical_link
+      last_known_good
     end
 
     # Redirects and followed canonical links share one visited-URL set and one
     # hop budget, so any cycle or over-long chain terminates deterministically
     def hop_refusal_reason
-      return 'Redirect loop detected' if visited.include?(response_url)
+      return 'Redirect loop detected' if visited?(response_url)
       return "#{hops + 1} redirects is too many" if hops >= options[:max_redirects]
 
       nil
     end
 
-    def follow_hop
-      visited << response_url
-      @hops = hops + 1
+    def follow_hop(via)
+      chain << Result::Hop.new(url: response_url, via: via)
       self.url = response_url
       nil
     end
 
-    def visited
-      @visited ||= [url]
+    # The URLs requested so far and how each one was discovered
+    def chain
+      @chain ||= [Result::Hop.new(url: url, via: :initial)]
+    end
+
+    def visited?(candidate)
+      chain.any? { |hop| hop.url == candidate }
     end
 
     def hops
-      @hops ||= 0
+      chain.length - 1
     end
 
     def handle_failure
-      return last_known_good if last_known_good
-
-      raise URLCanonicalize::Exception::Failure, "#{response.failure_class}: #{response.message}",
-            cause: response.error
+      finish_with_last_known_good || raise(
+        URLCanonicalize::Exception::Failure,
+        "#{response.failure_class}: #{response.message}",
+        cause: response.error
+      )
     end
 
     def handle_unhandled_response
@@ -133,6 +154,7 @@ module URLCanonicalize
     end
 
     def handle_success
+      @final_source = chain.last.via
       self.last_known_good = response
     end
 
@@ -145,7 +167,7 @@ module URLCanonicalize
       return @http if same_origin? # reuse connection
 
       @previous = uri
-      @http = new_http
+      @http = options[:transport].call(uri, options)
     end
 
     def same_origin?
@@ -156,16 +178,6 @@ module URLCanonicalize
       @previous ||= Struct.new(:scheme, :host, :port).new
     end
 
-    def new_http
-      h = Net::HTTP.new uri.host, uri.port, nil
-
-      h.ipaddr = URLCanonicalize::Destination.resolve(uri, options)
-      configure_timeouts(h)
-      ssl!(h)
-
-      h
-    end
-
     def read_response_body(http_request, response)
       if buffer_response_body?(http_request, response)
         enforce_content_length!(response)
@@ -185,21 +197,5 @@ module URLCanonicalize
       raise URLCanonicalize::Exception::ResponseTooLarge,
             "Response body exceeds #{options[:max_body_bytes]} bytes"
     end
-
-    def configure_timeouts(http)
-      http.open_timeout = options[:open_timeout]
-      http.read_timeout = options[:read_timeout]
-      http.write_timeout = options[:write_timeout]
-    end
-
-    def ssl!(http)
-      if uri.scheme == 'https'
-        http.use_ssl = true # Can generate exception
-        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-        http.verify_hostname = true
-      else
-        http.use_ssl = false
-      end
-    end
   end
 end

data/lib/url_canonicalize/options.rb

@@ -6,12 +6,20 @@ module URLCanonicalize
     DEFAULTS = {
       allow_private_networks: false,
       allowed_ports: [80, 443].freeze,
+      headers: {
+        'Accept-Language' => 'en-US,en;q=0.8',
+        'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) ' \
+                        'AppleWebKit/537.36 (KHTML, like Gecko) ' \
+                        'Chrome/51.0.2704.103 Safari/537.36'
+      }.freeze,
+      logger: nil,
       max_body_bytes: 1_048_576,
       total_timeout: 30,
       open_timeout: 8,
       read_timeout: 15,
       write_timeout: 8,
-      max_redirects: 10
+      max_redirects: 10,
+      transport: Transport.new
     }.freeze
 
     POSITIVE_NUMERIC_OPTIONS = %i[total_timeout open_timeout read_timeout write_timeout].freeze
@@ -23,6 +31,7 @@ module URLCanonicalize
       @values = DEFAULTS.merge(values)
       validate!
       @values[:allowed_ports] = @values[:allowed_ports].dup.freeze
+      @values[:headers] = @values[:headers].dup.freeze
       @values.freeze
       freeze
     end
@@ -32,7 +41,7 @@ module URLCanonicalize
     end
 
     def to_h
-      @values.merge(allowed_ports: @values[:allowed_ports].dup)
+      @values.merge(allowed_ports: @values[:allowed_ports].dup, headers: @values[:headers].dup)
     end
 
     private
@@ -41,6 +50,9 @@ module URLCanonicalize
       validate_private_networks!
       validate_ports!
       validate_positive_values!
+      validate_headers!
+      validate_logger!
+      validate_transport!
     end
 
     def validate_known_options!(values)
@@ -75,5 +87,26 @@ module URLCanonicalize
         raise ArgumentError, "#{name} must be positive" unless value.is_a?(Integer) && value.positive?
       end
     end
+
+    def validate_headers!
+      headers = @values[:headers]
+      valid = headers.is_a?(Hash) && headers.all? { |key, value| key.is_a?(String) && value.is_a?(String) }
+      return if valid
+
+      raise ArgumentError, 'headers must be a Hash of String keys and String values'
+    end
+
+    def validate_logger!
+      logger = @values[:logger]
+      return if logger.nil? || logger.respond_to?(:debug)
+
+      raise ArgumentError, 'logger must respond to debug'
+    end
+
+    def validate_transport!
+      return if @values[:transport].respond_to?(:call)
+
+      raise ArgumentError, 'transport must respond to call'
+    end
   end
 end

data/lib/url_canonicalize/request.rb

@@ -85,7 +85,7 @@ module URLCanonicalize
     end
 
     def handle_success
-      @canonical_url = relative_to_absolute(URLCanonicalize::LinkHeader.canonical(response))
+      @canonical_url = normalized(relative_to_absolute(URLCanonicalize::LinkHeader.canonical(response)))
 
       return enhanced_response if canonical_url || http_method == :get
 
@@ -122,7 +122,7 @@ module URLCanonicalize
 
     def enhanced_response
       if canonical_url
-        puts "  * canonical_url:\t#{canonical_url}" if ENV.fetch('DEBUG', nil)
+        logger&.debug { "canonical_url: #{canonical_url}" }
         response_plus = URLCanonicalize::Response::Success.new(canonical_url, response, html)
         URLCanonicalize::Response::CanonicalFound.new(canonical_url, response_plus)
       else
@@ -135,7 +135,13 @@ module URLCanonicalize
     end
 
     def canonical_url
-      @canonical_url ||= relative_to_absolute(canonical_url_element&.[]('href'), document_base_url)
+      @canonical_url ||= normalized(relative_to_absolute(canonical_url_element&.[]('href'), document_base_url))
+    end
+
+    # Declared canonical URLs are returned to callers, so they get the same
+    # syntactic normalization as every requested URL
+    def normalized(url)
+      URLCanonicalize::URI.normalize(url) if url
     end
 
     # The first HTML link element whose rel tokens include "canonical", however
@@ -179,12 +185,7 @@ module URLCanonicalize
     end
 
     def headers
-      @headers ||= {
-        'Accept-Language' => 'en-US,en;q=0.8',
-        'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) ' \
-                        'AppleWebKit/537.36 (KHTML, like Gecko) ' \
-                        'Chrome/51.0.2704.103 Safari/537.36'
-      }
+      http.options[:headers]
     end
 
     def http_method=(value)
@@ -210,14 +211,11 @@ module URLCanonicalize
     end
 
     def log_response
-      debug = ENV.fetch('DEBUG', nil)
-      return unless debug
-
-      puts "#{http_method.upcase} #{url} #{response.code} #{response.message}"
-
-      return unless debug.casecmp?('headers')
+      logger&.debug { "#{http_method.upcase} #{url} #{response.code} #{response.message}" }
+    end
 
-      response.each { |k, v| puts "  #{k}:\t#{v}" }
+    def logger
+      http.options[:logger]
     end
   end
 end

data/lib/url_canonicalize/result.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # The outcome of one canonicalization: the canonical URL, the response
+  # that confirmed it, and the chain of hops that led there. Immutable
+  class Result
+    # One step in the request chain: the URL that was requested and how it
+    # was discovered (:initial, :redirect or :canonical_link)
+    Hop = Struct.new(:url, :via, keyword_init: true)
+
+    attr_reader :url, :response, :html, :chain, :source
+
+    def xml
+      Nokogiri::XML(response.body)
+    end
+
+    private
+
+    def initialize(url:, response:, html:, chain:, source:)
+      @url = url
+      @response = response
+      @html = html
+      @chain = chain.map(&:freeze).freeze
+      @source = source
+      freeze
+    end
+  end
+end

data/lib/url_canonicalize/transport.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module URLCanonicalize
+  # Builds the connection for a destination, resolving the host through the
+  # fetch security policy. Injectable via the :transport option: anything
+  # responding to call(uri, options) and returning a Net::HTTP-compatible
+  # connection can replace it
+  class Transport
+    # Resolved lazily so the destination policy can be stubbed in tests
+    DEFAULT_RESOLVER = lambda do |uri, options|
+      URLCanonicalize::Destination.resolve(uri, options)
+    end
+
+    def initialize(resolver: DEFAULT_RESOLVER)
+      @resolver = resolver
+      freeze
+    end
+
+    def call(uri, options)
+      http = Net::HTTP.new(uri.host, uri.port, nil)
+
+      http.ipaddr = @resolver.call(uri, options)
+      configure_timeouts(http, options)
+      configure_tls(http, uri)
+
+      http
+    end
+
+    private
+
+    def configure_timeouts(http, options)
+      http.open_timeout = options[:open_timeout]
+      http.read_timeout = options[:read_timeout]
+      http.write_timeout = options[:write_timeout]
+    end
+
+    def configure_tls(http, uri)
+      if uri.scheme == 'https'
+        http.use_ssl = true # Can generate exception
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        http.verify_hostname = true
+      else
+        http.use_ssl = false
+      end
+    end
+  end
+end

data/lib/url_canonicalize/uri.rb

@@ -1,20 +1,36 @@
 # frozen_string_literal: true
 
 module URLCanonicalize
-  # Manage the URL into a URI with local exception handling
+  # Parses and normalizes URLs with local exception handling
   class URI
     class << self
       VALID_CLASSES = [::URI::HTTP, ::URI::HTTPS].freeze
-      COLON = ':'
+
+      # A scheme is recognized only as a leading scheme: token; a colon
+      # anywhere else no longer suppresses the default scheme
+      SCHEME = /\A[A-Za-z][A-Za-z0-9+.-]*:/
 
       def parse(url)
-        uri = ::URI.parse decorate(url)
+        uri = ::URI.parse normalize(url)
         validate! uri
         uri
       rescue ::URI::InvalidURIError => e
         raise URLCanonicalize::Exception::URI, "#{e.class}: #{e.message}" # the original error becomes the cause
       end
 
+      # Syntactic normalization per RFC 3986, via Addressable: lowercase
+      # scheme and host, internationalized hosts to punycode, uppercase
+      # percent-encodings with unreserved characters decoded, dot segments
+      # resolved, default ports removed and the empty path replaced by "/".
+      # Fragments are client-side state, so no canonical URL carries one
+      def normalize(url)
+        addressable = Addressable::URI.parse(decorate(url.to_s.strip)).normalize
+        addressable.fragment = nil
+        addressable.to_s
+      rescue Addressable::URI::InvalidURIError, ArgumentError, Encoding::CompatibilityError => e
+        raise URLCanonicalize::Exception::URI, "#{e.class}: #{e.message}" # the original error becomes the cause
+      end
+
       private
 
       def validate!(uri)
@@ -24,7 +40,7 @@ module URLCanonicalize
       end
 
       def decorate(url)
-        return url if url.include? COLON
+        return url if url.match?(SCHEME)
 
         "http://#{url}" # Add protocol if we just receive a host name
       end

data/lib/url_canonicalize/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module URLCanonicalize
-  VERSION = '1.0.2'
+  VERSION = '2.0.0'
 end

data/lib/url_canonicalize.rb

@@ -11,6 +11,7 @@ autoload :OpenSSL, 'openssl'
 # Core methods
 module URLCanonicalize
   autoload :BoundedBody, 'url_canonicalize/bounded_body'
+  autoload :Client, 'url_canonicalize/client'
   autoload :Destination, 'url_canonicalize/destination'
   autoload :Exception, 'url_canonicalize/exception'
   autoload :HTTP, 'url_canonicalize/http'
@@ -19,20 +20,18 @@ module URLCanonicalize
   autoload :Options, 'url_canonicalize/options'
   autoload :Request, 'url_canonicalize/request'
   autoload :Response, 'url_canonicalize/response'
+  autoload :Result, 'url_canonicalize/result'
+  autoload :Transport, 'url_canonicalize/transport'
   autoload :URI, 'url_canonicalize/uri'
   autoload :VERSION, 'url_canonicalize/version'
 
   class << self
-    def canonicalize(url, **options)
-      fetch(url, **options).url
+    def canonicalize(url, **)
+      Client.new(**).canonicalize(url)
     end
 
-    def fetch(url, **options)
-      URLCanonicalize::HTTP.new(url, **options).fetch
+    def fetch(url, **)
+      Client.new(**).fetch(url)
     end
   end
 end
-
-require 'monkey_patches/uri'
-require 'monkey_patches/string'
-require 'monkey_patches/addressable/uri'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: url_canonicalize
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Dominic Sayers
@@ -49,11 +49,13 @@ files:
 - LICENSE
 - README.md
 - SECURITY.md
-- lib/monkey_patches/addressable/uri.rb
-- lib/monkey_patches/string.rb
-- lib/monkey_patches/uri.rb
 - lib/url_canonicalize.rb
 - lib/url_canonicalize/bounded_body.rb
+- lib/url_canonicalize/client.rb
+- lib/url_canonicalize/core_ext.rb
+- lib/url_canonicalize/core_ext/addressable.rb
+- lib/url_canonicalize/core_ext/string.rb
+- lib/url_canonicalize/core_ext/uri.rb
 - lib/url_canonicalize/destination.rb
 - lib/url_canonicalize/exception.rb
 - lib/url_canonicalize/http.rb
@@ -62,6 +64,8 @@ files:
 - lib/url_canonicalize/options.rb
 - lib/url_canonicalize/request.rb
 - lib/url_canonicalize/response.rb
+- lib/url_canonicalize/result.rb
+- lib/url_canonicalize/transport.rb
 - lib/url_canonicalize/uri.rb
 - lib/url_canonicalize/version.rb
 homepage: https://github.com/dominicsayers/url_canonicalize
@@ -81,7 +85,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="