twingly-url 7.0.1 → 7.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d077940af63f0646300c3438d45f3ca66370825c5ca81df32c6d1597e156071
-  data.tar.gz: 308a66eb3c281139a9f747a3538c3e53291f491eafa3bbdd3dbe30d1decb42d3
+  metadata.gz: 546297fb93a4c8c81ddeae25c27ed9cd174ac4f13df0aae03287ab94f831be6b
+  data.tar.gz: 10e53871ab61298b9d1bc219e12b593b0696b4be419d00155317d1d1f3b8e229
 SHA512:
-  metadata.gz: 0b50ee3c1e3dd3beb12b85c813e36952945fca3de296257d7e0428a1de78d866932d334f13c405f784659324a69de0b2de293c07520f94f0e9d3c9a0665a3955
-  data.tar.gz: 278af99f73d82edb9f05eed9f795fe0e0395bf45626f9fde2e09b605ce467afc90c13780020fc94b6e54f2c64daf995f061a20eddbc1fcc84d6ac1cf0a1fcd48
+  metadata.gz: 81a743b883a0c5f2f25628566002f646074fcd68f1473ccbd327dd82766bc8e645676cf9a3bfeeed23a7676ff4a1f4522d7b6407d93a55ad73ac3c6ddd1f9b4a
+  data.tar.gz: bbb6e7694dbad697653c15205ac2049c8f3ce6f90d338fe7f4634937478da54c63a1504975db4273af1fd36199896c13e232c2d9c01f908b9186f4d11f2003fe

data/README.md

@@ -12,6 +12,11 @@ Twingly URL tools.
     * `Twingly::URL::Hasher.autopingdb_hash(url)` - SHA256 64-bit signed, native endian digest
 * `twingly/url/utilities` - Utilities to work with URLs
     * `Twingly::URL::Utilities.extract_valid_urls` - Returns Array of valid `Twingly::URL`
+* `twingly/url/extended` - Extended normalization and hashing, stripping blacklisted query and matrix parameters as well as the fragment.
+    * `Twingly::URL::Extended.parse` - Like `Twingly::URL.parse`, with the extended normalization
+    * `Twingly::URL::Extended.normalize_and_calculate_urlhash(url, addressable_normalize: false)` - Returns a `HashResult` struct with `url` (original URL minus blacklisted parameters), `callable_url` (original URL canonicalized with `Addressable::URI#normalize`, minus blacklisted parameters), `normalized_url` (scheme-less), `urlhash` (`documentdb_hash` of the normalized URL, as a `String`) and `legacy_urlhash` (same digest with the scheme kept)
+        * `addressable_normalize: true` applies `Addressable::URI#normalize` (percent-encoding, case, default port and dot segments) before the extended normalization, so `normalized_url` and both hashes are calculated from the canonical form
+    * `Twingly::URL::Extended#addressable_normalized` - Returns a new instance canonicalized with `Addressable::URI#normalize`, `#normalized(addressable_normalize: true)` is the extended normalization applied to that instance
 
 ## Getting Started
 

data/lib/twingly/url/extended.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require "twingly/url"
+require "twingly/url/hasher"
+
+module Twingly
+  class URL
+    # The purpose of the twingly-url gems is to replicate the normalization happening in our legacy .NET code.
+    # As both twingly-url and the legacy .NET system are used in production, we can't change these things in the
+    # gem just yet, which is why this extension was created here.
+    class Extended < Twingly::URL
+      # Use the same expression as we use in Zambezi: https://github.com/twingly/zambezi/blob/d997871aea199256ddc56762c975b6e961d5a2d1/lib/post_document.rb#L12
+      # This way the urlhashes here will be the same as the one we have in Elasticsearch
+      PROTOCOL_EXPRESSION = /\Ahttps?:/i
+
+      HashResult = Struct.new(:url, :callable_url, :normalized_url, :urlhash, :legacy_urlhash)
+
+      # These parameters are just used for tracking, and should therefore not be included in the normalized URL
+      # See https://en.wikipedia.org/wiki/UTM_parameters
+      BLACKLISTED_QUERY_PARAMETERS = %w[
+        utm_source
+        utm_medium
+        utm_campaign
+        utm_term
+        utm_content
+        b_source
+        b_medium
+        b_campaign
+        session
+        PHPSESSID
+        cb
+      ].freeze
+
+      BLACKLISTED_MATRIX_PARAMETERS = %w[
+        jsessionid
+        jsessionid_jboss
+        JSESSIONID_B2BCH
+        JSESSIONID_kookbnagWEB
+      ].freeze
+
+      # Precompiled regex matching any blacklisted matrix parameter
+      # Matches: ;paramname=value where value ends at ; / ? or #
+      MATRIX_PARAM_REGEX = %r{;(?:#{BLACKLISTED_MATRIX_PARAMETERS.map { |p| Regexp.escape(p) }.join('|')})=[^;/?#]*}i
+
+      # Taken from twingly-url, with the addition of normalizing the query and fragment components
+      # See https://github.com/twingly/twingly-url/blob/e20f5fce077d93e89ef8520961be453c90cfec8c/lib/twingly/url.rb#L185-L193
+      def normalized(addressable_normalize: false) # rubocop:disable Metrics/AbcSize
+        return addressable_normalized.normalized if addressable_normalize
+
+        normalized_url = addressable_uri.dup
+
+        normalized_url.scheme       = normalized_scheme
+        normalized_url.host         = normalized_host
+        normalized_url.path         = normalized_path
+        normalized_url.query_values = normalized_query
+        normalized_url.fragment     = normalized_fragment
+
+        # This is a bit ugly, remove when Twingly::URL is updated to
+        # to handle this.
+        public_suffix_domain = get_public_suffix_domain(normalized_url.host)
+        self.class.send(:new, normalized_url, public_suffix_domain)
+      end
+
+      def addressable_normalized
+        addressable_normalized_url = normalize_addressable_url
+        public_suffix_domain = get_public_suffix_domain(addressable_normalized_url.host)
+        self.class.send(:new, addressable_normalized_url, public_suffix_domain)
+      end
+
+      def original_url_without_blacklisted_parameters
+        url = addressable_uri.dup
+        url.path = without_blacklisted_matrix_parameters(addressable_uri.path)
+        url.query_values = without_blacklisted_query_parameters(addressable_uri.query_values)
+        url.to_s
+      end
+
+      def query_values
+        addressable_uri.query_values
+      end
+
+      def query
+        addressable_uri.query
+      end
+
+      private
+
+      # copied the way public_suffix_domain is calculated
+      # from Twingly::URL to be able calculate it without calling the parse method.
+      def get_public_suffix_domain(host)
+        public_suffix_domain = PublicSuffix.parse(host, list: CUSTOM_PSL, default_rule: nil)
+        raise Twingly::URL::Error::ParseError if public_suffix_domain.nil?
+        raise Twingly::URL::Error::ParseError if public_suffix_domain.sld.nil?
+
+        public_suffix_domain
+      end
+
+      def normalized_path
+        addressable_uri.path = without_blacklisted_matrix_parameters(addressable_uri.path)
+
+        super
+      end
+
+      def normalized_query
+        without_blacklisted_query_parameters(addressable_uri.query_values)
+      end
+
+      def normalize_addressable_url
+        addressable_uri.normalize
+      end
+
+      def without_blacklisted_query_parameters(query_values)
+        return if query_values.nil?
+
+        values_without_blacklisted_params = query_values.except(*BLACKLISTED_QUERY_PARAMETERS)
+
+        return if values_without_blacklisted_params.empty?
+
+        values_without_blacklisted_params
+      end
+
+      def without_blacklisted_matrix_parameters(path)
+        # No need to run gsub if there are no matrix parameters
+        return path unless path.include?(";")
+
+        path.gsub(MATRIX_PARAM_REGEX, "")
+      end
+
+      def normalized_fragment
+        nil
+      end
+
+      def self.normalize_and_calculate_urlhash(url, addressable_normalize: false)
+        return empty_result if url.to_s.strip.empty?
+
+        twingly_url = if url.is_a?(Extended)
+                        url
+                      else
+                        Extended.parse(url)
+                      end
+
+        return empty_result unless twingly_url.valid?
+
+        original_url               = twingly_url.original_url_without_blacklisted_parameters
+        addressable_normalized     = twingly_url.addressable_normalized
+        callable_url               = addressable_normalized.original_url_without_blacklisted_parameters
+
+        twingly_url = addressable_normalized if addressable_normalize
+
+        normalized_url                = twingly_url.normalized.to_s
+        normalized_url_without_scheme = remove_scheme(normalized_url)
+        urlhash                       = calculate_urlhash(normalized_url_without_scheme)
+        legacy_urlhash                = calculate_urlhash(normalized_url)
+
+        HashResult.new(original_url, callable_url, normalized_url_without_scheme, urlhash, legacy_urlhash)
+      end
+
+      def self.empty_result
+        HashResult.new(nil, nil, nil, nil, nil)
+      end
+
+      def self.remove_scheme(url)
+        url.to_s.sub(PROTOCOL_EXPRESSION, "")
+      end
+
+      def self.calculate_urlhash(url)
+        Twingly::URL::Hasher.documentdb_hash(url).to_s
+      end
+
+      private_class_method :empty_result
+      private_class_method :calculate_urlhash
+    end
+  end
+end

data/lib/twingly/url/null_url.rb

@@ -12,7 +12,7 @@ module Twingly
         ""
       end
 
-      def normalized
+      def normalized(**)
         self
       end
 

data/lib/twingly/version.rb

@@ -2,6 +2,6 @@
 
 module Twingly
   class URL
-    VERSION = "7.0.1"
+    VERSION = "7.1.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twingly-url
 version: !ruby/object:Gem::Version
-  version: 7.0.1
+  version: 7.1.1
 platform: ruby
 authors:
 - Twingly AB
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-01 00:00:00.000000000 Z
+date: 2026-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -33,7 +33,7 @@ dependencies:
         version: 3.0.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -43,21 +43,21 @@ dependencies:
         version: 3.0.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -97,6 +97,7 @@ files:
 - lib/twingly/public_suffix_list.rb
 - lib/twingly/url.rb
 - lib/twingly/url/error.rb
+- lib/twingly/url/extended.rb
 - lib/twingly/url/hasher.rb
 - lib/twingly/url/null_url.rb
 - lib/twingly/url/utilities.rb
@@ -113,14 +114,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Ruby library for URL handling