normalizeurl2025 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 074cfe360aac260e52d0aa43406d0ba34be9d0d8cb38255b2f683d8d667aac99
+  data.tar.gz: 5f7c9faee5ec6add114898108cd650d8aee674d1d752e3a51a48993f3dd868ce
+SHA512:
+  metadata.gz: bdef572d7b2135a420d3e07b4fa6b4e57ac5a9b3e33dab3485d149dd2da1877bad5f15d8aac1aef6c18dbeb8a7d921eef5069e9bee6ec1f712388c4385e2277f
+  data.tar.gz: b9580b5c71d3ef21f5a3f08f2b65859433499a0637f8581be39175d3389a6132809ab232a514e59f29110c3043a361ed8a9572de654f0c6427589a47b2517e90

data/Gemfile

@@ -0,0 +1,7 @@
+source "https://rubygems.org"
+
+# Use the new gemspec name
+gemspec name: "normalizeurl2025"
+
+gem "rake", "~> 13.0"
+gem "minitest", "~> 5.0"

data/README.md

@@ -0,0 +1,278 @@
+# NormalizeURL 2025
+
+A Ruby library for normalizing URLs by removing tracking parameters, session IDs, and other extraneous elements whilst preserving important parameters.
+
+NormalizeURL 2025 creates a canonical representation of URLs that can be used for deduplication, caching keys, or comparison purposes and was developed initially to deduplicate URLs found in RSS feeds.
+
+> [!IMPORTANT]  
+> The normalized URLs are intended for creating unique representations and may not always remain functional URLs.
+
+> [!NOTE]  
+> The library has such a silly name because RubyGems refused to let me use the available name of 'normalizeurl' due to an ancient gem with a similar name. Rather than putting 'lol' or 'poo' on the end, you get 2025 instead.
+
+**Important**: 
+
+## Installation
+
+For your `Gemfile`:
+
+```ruby
+gem 'normalizeurl2025'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install normalizeurl2025
+
+## Usage
+
+### Basic Usage
+
+```ruby
+require 'normalizeurl2025'
+
+# Simple normalization
+url = "https://example.com/page?utm_source=google&utm_medium=cpc&id=123"
+normalized = Normalizeurl2025.normalize(url)
+# => "https://example.com/page?id=123"
+
+# Remove tracking parameters whilst preserving important ones
+youtube_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ&utm_source=twitter&t=42"
+normalized = Normalizeurl2025.normalize(youtube_url)
+# => "https://youtube.com/watch?t=42&v=dQw4w9WgXcQ"
+```
+
+### Configuration Options
+
+NormalizeURL 2025 accepts various options to customize the normalization behaviour:
+
+```ruby
+# Default options
+options = {
+  remove_tracking_params: true,    # Remove UTM and other tracking parameters
+  remove_trailing_slash: true,     # Remove trailing slashes from paths
+  downcase_hostname: true,         # Convert hostname to lowercase
+  remove_www: false,               # Remove 'www.' prefix from hostname
+  remove_fragment: true,           # Remove URL fragments (#section)
+  custom_tracking_params: [],      # Additional tracking parameters to remove
+  preserve_params: {}              # Domain-specific parameters to preserve
+}
+
+normalized = Normalizeurl2025.normalize(url, options)
+```
+
+### Examples
+
+#### Removing Tracking Parameters
+
+```ruby
+# UTM parameters are removed
+url = "https://example.com/article?utm_source=newsletter&utm_campaign=spring2024&id=456"
+Normalizeurl2025.normalize(url)
+# => "https://example.com/article?id=456"
+
+# Google Click ID and Facebook Click ID are removed
+url = "https://shop.example.com/product?gclid=abc123&fbclid=def456&product_id=789"
+Normalizeurl2025.normalize(url)
+# => "https://shop.example.com/product?product_id=789"
+
+# Session IDs and tracking pixels are removed
+url = "https://site.com/page?PHPSESSID=abc123&_ga=GA1.2.123456&content=article"
+Normalizeurl2025.normalize(url)
+# => "https://site.com/page?content=article"
+```
+
+#### Preserving Important Parameters
+
+```ruby
+# YouTube video parameters are preserved
+url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ&utm_source=email&t=120&list=PLxyz"
+Normalizeurl2025.normalize(url)
+# => "https://youtube.com/watch?list=PLxyz&t=120&v=dQw4w9WgXcQ"
+
+# Amazon product parameters are preserved
+url = "https://amazon.com/dp/B08N5WRWNW?tag=mysite-20&utm_source=blog&keywords=ruby"
+Normalizeurl2025.normalize(url)
+# => "https://amazon.com/dp/B08N5WRWNW?keywords=ruby&tag=mysite-20"
+
+# Search engine queries are preserved
+url = "https://google.com/search?q=ruby+gems&utm_source=referral&hl=en"
+Normalizeurl2025.normalize(url)
+# => "https://google.com/search?q=ruby+gems"
+```
+
+#### URL Structure Normalization
+
+```ruby
+# Hostname is lowercased and www is optionally removed
+url = "HTTPS://WWW.EXAMPLE.COM/Path/?utm_source=email"
+Normalizeurl2025.normalize(url)
+# => "https://www.example.com/Path"
+
+# Remove www prefix
+url = "https://www.example.com/page?id=123"
+Normalizeurl2025.normalize(url, remove_www: true)
+# => "https://example.com/page?id=123"
+
+# Trailing slashes are removed (except root)
+url = "https://example.com/path/to/page/?utm_medium=social"
+Normalizeurl2025.normalize(url)
+# => "https://example.com/path/to/page?utm_medium=social"
+
+# Fragments are removed by default
+url = "https://example.com/page#section?utm_source=twitter"
+Normalizeurl2025.normalize(url)
+# => "https://example.com/page"
+
+# Keep fragments if needed
+url = "https://example.com/page#section?utm_source=twitter"
+Normalizeurl2025.normalize(url, remove_fragment: false)
+# => "https://example.com/page#section"
+```
+
+#### Query Parameter Sorting
+
+```ruby
+# Parameters are sorted alphabetically for consistency
+url = "https://example.com/search?z=last&a=first&m=middle&utm_source=email"
+Normalizeurl2025.normalize(url)
+# => "https://example.com/search?a=first&m=middle&z=last"
+```
+
+#### Custom Configuration
+
+```ruby
+# Add custom tracking parameters to remove
+options = {
+  custom_tracking_params: ['my_tracker', 'internal_ref', 'campaign_id']
+}
+url = "https://example.com/page?my_tracker=abc&internal_ref=xyz&id=123&campaign_id=spring"
+Normalizeurl2025.normalize(url, options)
+# => "https://example.com/page?id=123"
+
+# Preserve custom parameters for specific domains
+options = {
+  preserve_params: {
+    'mysite.com' => ['special_param', 'user_pref'],
+    'shop.example.com' => ['affiliate_id', 'discount_code']
+  }
+}
+url = "https://mysite.com/page?special_param=value&utm_source=email&user_pref=dark"
+Normalizeurl2025.normalize(url, options)
+# => "https://mysite.com/page?special_param=value&user_pref=dark"
+
+# Subdomain matching works automatically
+url = "https://blog.mysite.com/post?special_param=test&utm_campaign=launch"
+Normalizeurl2025.normalize(url, options)
+# => "https://blog.mysite.com/post?special_param=test"
+```
+
+#### Error Handling
+
+```ruby
+# Invalid URLs are returned unchanged
+Normalizeurl2025.normalize("not-a-url")
+# => "not-a-url"
+
+# Nil and empty strings return nil
+Normalizeurl2025.normalize(nil)
+# => nil
+```
+
+## Default Behaviour
+
+### Tracking Parameters Removed
+
+NormalizeURL 2025 removes these common tracking parameters by default:
+
+**UTM Parameters:**
+- `utm_source`, `utm_medium`, `utm_campaign`, `utm_term`, `utm_content`
+- `utm_name`, `utm_cid`, `utm_reader`, `utm_viz_id`, `utm_pubreferrer`, `utm_swu`
+
+**Click IDs:**
+- `gclid` (Google Ads), `fbclid` (Facebook), `msclkid` (Microsoft Advertising)
+- `yclid` (Yandex), `rb_clickid` (Rakuten)
+
+**Analytics & Tracking:**
+- `_ga`, `_gl` (Google Analytics)
+- `mc_cid`, `mc_eid` (Mailchimp)
+- `s_cid` (Adobe Analytics)
+- `_openstat` (OpenStat)
+- `ck_subscriber_id` (Kit / ConvertKit)
+
+**Session & User IDs:**
+- `PHPSESSID`, `JSESSIONID`, `ASPSESSIONID`
+- `sid`, `sessionid`, `session_id`
+- `subscriber_id`, `ig_rid` (Instagram)
+- `oly_anon_id`, `oly_enc_id` (Optimizely)
+- `wickedid`, `vero_conv`, `vero_id`
+
+**Referrer & Source Tracking:**
+- `ref`, `referer`, `referrer`, `source`, `src`
+- `campaign`, `__s`
+
+### Parameters Preserved by Domain
+
+Certain parameters are automatically preserved for specific domains:
+
+- **YouTube** (`youtube.com`, `youtu.be`): `v` (video ID), `t` (timestamp), `list` (playlist), `index`
+- **Amazon** (`amazon.com`, `amazon.co.uk`): `keywords`, `tag`
+- **eBay** (`ebay.com`, `ebay.co.uk`): `hash`
+- **Google** (`google.com`, `google.co.uk`): `q` (search query)
+- **Bing** (`bing.com`): `q` (search query)
+- **DuckDuckGo** (`duckduckgo.com`): `q` (search query)
+- **Stack Overflow** (`stackoverflow.com`): `answertab`
+- **Vimeo** (`vimeo.com`): `h_original`
+
+### URL Transformations
+
+1. **Scheme**: Converted to lowercase (`HTTP` → `http`)
+2. **Hostname**: Converted to lowercase (`Example.COM` → `example.com`)
+3. **WWW prefix**: Optionally removed (`www.example.com` → `example.com`)
+4. **Path**: 
+   - Trailing slashes removed (except root path)
+   - Empty paths become `/`
+5. **Query parameters**: 
+   - Tracking parameters filtered out
+   - Remaining parameters sorted alphabetically
+   - Empty query strings removed entirely
+6. **Fragments**: Removed by default (`#section` removed)
+
+## Advanced Usage
+
+### Batch Processing
+
+```ruby
+urls = [
+  "https://example.com/page1?utm_source=email",
+  "https://example.com/page2?gclid=abc123",
+  "https://example.com/page3?fbclid=def456"
+]
+
+normalized_urls = urls.map { |url| Normalizeurl2025.normalize(url) }
+# => ["https://example.com/page1", "https://example.com/page2", "https://example.com/page3"]
+```
+
+### Creating Consistent Cache Keys
+
+```ruby
+def cache_key_for_url(url)
+  normalized = Normalizeurl2025.normalize(url)
+  Digest::MD5.hexdigest(normalized)
+end
+
+cache_key_for_url("https://example.com/article?utm_source=twitter&id=123")
+# => "a1b2c3d4e5f6..." (consistent hash regardless of tracking params)
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/peterc/normalizeurl2025.
+
+## Licence
+
+The gem is available as open source under the terms of the [MIT Licence](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+task default: :test

data/lib/normalizeurl2025/normalizer.rb

@@ -0,0 +1,165 @@
+require 'uri'
+
+module Normalizeurl2025
+  class Normalizer
+    # Parameters that are commonly used for tracking and should be removed
+    DEFAULT_TRACKING_PARAMS = %w[
+      utm_source utm_medium utm_campaign utm_term utm_content
+      utm_name utm_cid utm_reader utm_viz_id utm_pubreferrer utm_swu
+      gclid fbclid msclkid
+      ck_subscriber_id
+      ck_subscriber
+      _ga _gl
+      mc_cid mc_eid
+      PHPSESSID JSESSIONID ASPSESSIONID
+      sid sessionid session_id
+      ref referer referrer
+      source src
+      campaign
+      yclid
+      _openstat
+      rb_clickid
+      s_cid
+      vero_conv vero_id
+      wickedid
+      oly_anon_id oly_enc_id
+      __s
+      subscriber_id
+      ig_rid
+    ].freeze
+
+    # Parameters that should be preserved for certain domains
+    PRESERVE_PARAMS = {
+      'youtube.com' => %w[v t list index],
+      'youtu.be' => %w[v t],
+      'vimeo.com' => %w[h_original],
+      'amazon.com' => %w[keywords tag],
+      'amazon.co.uk' => %w[keywords tag],
+      'ebay.com' => %w[hash],
+      'ebay.co.uk' => %w[hash],
+      'stackoverflow.com' => %w[answertab],
+      'google.com' => %w[q],
+      'google.co.uk' => %w[q],
+      'bing.com' => %w[q],
+      'duckduckgo.com' => %w[q]
+    }.freeze
+
+    def initialize(options = {})
+      @remove_tracking_params = options.fetch(:remove_tracking_params, true)
+      @remove_trailing_slash = options.fetch(:remove_trailing_slash, true)
+      @downcase_hostname = options.fetch(:downcase_hostname, true)
+      @remove_www = options.fetch(:remove_www, false)
+      @remove_fragment = options.fetch(:remove_fragment, true)
+      @custom_tracking_params = options.fetch(:custom_tracking_params, [])
+      @preserve_params = options.fetch(:preserve_params, {})
+    end
+
+    def normalize(url)
+      return nil if url.nil? || url.strip.empty?
+
+      begin
+        uri = URI.parse(url.strip)
+      rescue URI::InvalidURIError
+        return url
+      end
+
+      # Only process HTTP/HTTPS URLs
+      return url unless %w[http https].include?(uri.scheme&.downcase)
+
+      normalize_scheme!(uri)
+      normalize_hostname!(uri)
+      normalize_path!(uri)
+      normalize_query!(uri)
+      normalize_fragment!(uri)
+
+      uri.to_s
+    end
+
+    private
+
+    def normalize_scheme!(uri)
+      uri.scheme = uri.scheme.downcase if uri.scheme
+    end
+
+    def normalize_hostname!(uri)
+      return unless uri.host
+
+      if @downcase_hostname
+        uri.host = uri.host.downcase
+      end
+
+      if @remove_www && uri.host.start_with?('www.')
+        uri.host = uri.host[4..-1]
+      end
+    end
+
+    def normalize_path!(uri)
+      return unless uri.path
+
+      # Remove trailing slash unless it's the root path
+      if @remove_trailing_slash && uri.path != '/' && uri.path.end_with?('/')
+        uri.path = uri.path.chomp('/')
+      end
+
+      # Ensure root path is present
+      uri.path = '/' if uri.path.empty?
+    end
+
+    def normalize_query!(uri)
+      return unless uri.query && @remove_tracking_params
+
+      params = URI.decode_www_form(uri.query)
+      
+      # Get domain-specific parameters to preserve
+      domain_preserve_params = get_preserve_params_for_domain(uri.host)
+      
+      # Filter out tracking parameters
+      filtered_params = params.reject do |key, _value|
+        should_remove_param?(key, domain_preserve_params)
+      end
+
+      if filtered_params.empty?
+        uri.query = nil
+      else
+        # Sort parameters for consistency
+        uri.query = URI.encode_www_form(filtered_params.sort)
+      end
+    end
+
+    def normalize_fragment!(uri)
+      uri.fragment = nil if @remove_fragment
+    end
+
+    def get_preserve_params_for_domain(host)
+      return [] unless host
+
+      # Check for exact domain match first
+      preserved = PRESERVE_PARAMS[host] || []
+      
+      # Check for subdomain matches
+      PRESERVE_PARAMS.each do |domain, params|
+        if host.end_with?(".#{domain}")
+          preserved.concat(params)
+        end
+      end
+
+      # Add custom preserve params for this domain
+      if @preserve_params[host]
+        preserved.concat(@preserve_params[host])
+      end
+
+      preserved.uniq
+    end
+
+    def should_remove_param?(param_name, preserve_params)
+      param_lower = param_name.downcase
+      
+      # Don't remove if it's in the preserve list
+      return false if preserve_params.any? { |p| p.downcase == param_lower }
+      
+      # Remove if it's a tracking parameter
+      tracking_params = DEFAULT_TRACKING_PARAMS + @custom_tracking_params
+      tracking_params.any? { |tp| tp.downcase == param_lower }
+    end
+  end
+end

data/lib/normalizeurl2025/version.rb

@@ -0,0 +1,3 @@
+module Normalizeurl2025
+  VERSION = "0.0.1"
+end

data/lib/normalizeurl2025.rb

@@ -0,0 +1,10 @@
+require_relative "normalizeurl2025/version"
+require_relative "normalizeurl2025/normalizer"
+
+module Normalizeurl2025
+  class Error < StandardError; end
+
+  def self.normalize(url, options = {})
+    Normalizer.new(options).normalize(url)
+  end
+end

data/normalizeurl2025.gemspec

@@ -0,0 +1,30 @@
+require_relative "lib/normalizeurl2025/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "normalizeurl2025"
+  spec.version = Normalizeurl2025::VERSION
+  spec.authors = ["Peter Cooper"]
+  spec.email = ["git@peterc.org"]
+
+  spec.summary = "A Ruby library for normalizing URLs"
+  spec.description = "Normalizes URLs by removing tracking parameters, session IDs, and other extraneous elements whilst preserving important parameters"
+  spec.homepage = "https://github.com/peterc/normalizeurl2025"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.0.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/peterc/normalizeurl2025"
+
+  # Specify which files should be added to the gem when it is released.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification
+name: normalizeurl2025
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Peter Cooper
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: Normalizes URLs by removing tracking parameters, session IDs, and other
+  extraneous elements whilst preserving important parameters
+email:
+- git@peterc.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- lib/normalizeurl2025.rb
+- lib/normalizeurl2025/normalizer.rb
+- lib/normalizeurl2025/version.rb
+- normalizeurl2025.gemspec
+homepage: https://github.com/peterc/normalizeurl2025
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/peterc/normalizeurl2025
+  source_code_uri: https://github.com/peterc/normalizeurl2025
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: A Ruby library for normalizing URLs
+test_files: []