udup 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 89618f0c392c4453c2eb610564b86b4e9cd34c590ccacd9513293f8ed2f3b95b
+  data.tar.gz: 9628469e7b5ee5932144129d47570c4242d050cc2d5fee85ce285978735570e2
+SHA512:
+  metadata.gz: c41ee00088458438c81b7109bfbee966fe1c17c872d109d3295fdcb29bf3564522b808eed87bf68dfe596487cf6ebef6ec6ec6f204189bd3a9783ebf4c98054a
+  data.tar.gz: be12f5d9d105e6287884edf006ec5d8440e55bb65d77acc2fcb118d894517999e83039c8e3e2974e52ec0a00a53d9e924b717a08d6e018270f34adf93c4038ab

data/lib/udup/main.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'set'
+
+class Udup
+  def initialize(options = {})
+    @valid_urls = {}
+    @skip_exts = options[:skip_exts] || %w[.css .png .jpg .jpeg .svg .ico .webp .ttf .otf .woff .woff2 .gif .pdf .bmp
+                                           .eot .mp3 .mp4 .avi]
+    @content_to_skip = options[:content_to_skip] || %w[blog docs post support]
+  end
+
+  def filter(urls)
+    final_urls = Set[]
+
+    urls.each do |url|
+      begin
+        uri = URI.parse(url)
+      rescue URI::InvalidURIError
+        next
+      end
+      next unless uri
+
+      uri_ext = File.extname(uri.path)
+      next if @skip_exts.include?(uri_ext) || human_content?(uri.path) || content_to_skip?(uri.path)
+
+      base_url = without_query(uri)
+      params = uri_params(uri.query)
+
+      if @valid_urls.key?(base_url)
+        @valid_urls[url] = { params: {} } if @valid_urls[base_url][:params].empty?
+        @valid_urls[base_url][:params].merge!(params)
+      else
+        @valid_urls[base_url] = { params: params }
+      end
+    end
+
+    @valid_urls.each do |url, data|
+      final_url = url
+      final_url += "?#{URI.encode_www_form(data[:params])}" unless data[:params].empty?
+      final_urls << final_url
+    end
+
+    final_urls.to_a
+  end
+
+  private
+
+  def without_query(uri)
+    uri.to_s.split('?', 2).first.to_s
+  end
+
+  def uri_params(query)
+    params = {}
+    query&.split('&')&.each do |param|
+      splitted = param.split('=')
+      params[splitted[0]] = splitted[1]
+    end
+
+    params
+  end
+
+  def human_content?(path)
+    human_content = false
+    path&.split('/')&.each { |part| human_content = true if part.count('-') >= 3 }
+
+    human_content
+  end
+
+  def content_to_skip?(path)
+    @content_to_skip.any? { |content| path.include?(content) }
+  end
+end

data/lib/udup.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative 'udup/main'

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: udup
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Joshua MARTINELLE
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-01-17 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+- contact@jomar.fr
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/udup.rb
+- lib/udup/main.rb
+homepage: https://rubygems.org/gems/udup
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: URL Deduplication
+test_files: []