wayfarer 0.4.7 → 0.4.9

data/lib/wayfarer/uri/normalization.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module URI
+    # HTTP(S) URL normalization.
+    module Normalization
+      InvalidURIError = Class.new(StandardError)
+
+      # Raised when URI is relative.
+      RelativeURIError = Class.new(InvalidURIError)
+
+      # Raised when URI scheme is not hypertext.
+      NoHypertextError = Class.new(InvalidURIError)
+
+      # Raised when URI has no host.
+      NoHostError = Class.new(InvalidURIError)
+
+      extend self
+
+      # Normalizes `uri` in-place.
+      # @param uri [Addressable::URI]
+      # @raise [InvalidURIError]
+      # @return [Addressable::URI]
+      def canonical!(uri)
+        had_no_path = uri.path.blank?
+
+        uri.normalize!
+        validate_uri!(uri)
+
+        normalize_host!(uri) if remove_www?
+
+        if remove_trailing_slash?
+          normalize_path!(uri)
+          root_path!(uri)
+        end
+
+        remove_fragment!(uri) if remove_fragment?
+        normalize_query_params!(uri)
+
+        root_path!(uri) if had_no_path && uri.query.nil?
+
+        uri
+      end
+
+    private
+
+      def validate_uri!(uri)
+        raise RelativeURIError, "URL is not absolute" unless uri.absolute?
+        raise NoHypertextError, "URL is using unsupported protocol" unless supported_protocols.include?(uri.scheme)
+        raise NoHostError, "URL misses hostname" if uri.host.blank?
+      end
+
+      def normalize_query_params!(uri)
+        return unless remove_tracking_parameters? || order_query_parameters?
+        return unless (params = uri.query_values(Array))
+
+        remove_tracking_parameters!(params) if remove_tracking_parameters?
+        order_query_parameters!(params) if order_query_parameters?
+
+        uri.query_values = params.empty? ? nil : params
+      end
+
+      def remove_tracking_parameters!(params)
+        params.reject! { |key, val| val.to_s.empty? || tracking_params.include?(key) }
+      end
+
+      def order_query_parameters!(params)
+        params.sort_by!(&:first)
+      end
+
+      def normalize_host!(uri)
+        uri.host &&= uri.host.delete_prefix("www.")
+      end
+
+      def normalize_path!(uri)
+        uri.path = uri.path.delete_suffix(File::SEPARATOR) if uri.path && uri.path.length > 1
+      end
+
+      def remove_fragment!(uri)
+        uri.fragment = nil
+      end
+
+      def root_path!(uri)
+        uri.path = "" if uri.path == File::SEPARATOR
+      end
+
+      def normalization_config
+        Wayfarer.config.fetch(:normalization)
+      end
+
+      def supported_protocols
+        normalization_config.fetch(:schemes)
+      end
+
+      def tracking_params
+        normalization_config.fetch(:tracking_params)
+      end
+
+      def remove_www?
+        normalization_config.fetch(:remove_www)
+      end
+
+      def remove_trailing_slash?
+        normalization_config.fetch(:remove_trailing_slash)
+      end
+
+      def remove_fragment?
+        normalization_config.fetch(:remove_fragment)
+      end
+
+      def remove_tracking_parameters?
+        normalization_config.fetch(:remove_tracking_parameters)
+      end
+
+      def order_query_parameters?
+        normalization_config.fetch(:order_query_parameters)
+      end
+    end
+  end
+end

data/lib/wayfarer.rb

@@ -3,13 +3,15 @@
 require "cgi"
 require "forwardable"
 require "net/http"
+require "pp"
 require "securerandom"
 require "uri"
+require "yaml"
 
 require "active_job"
+require "active_support/all"
 require "capybara"
 require "connection_pool"
-require "docile"
 require "ferrum"
 require "metainspector"
 require "mime/types"
@@ -18,7 +20,6 @@ require "mock_redis"
 require "mustermann"
 require "net/http/persistent"
 require "nokogiri"
-require "normalize_url"
 require "selenium-webdriver"
 require "redis"
 require "thor"
@@ -28,32 +29,77 @@ loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect("cli" => "CLI",
                          "dsl" => "DSL",
                          "http" => "HTTP",
+                         "uri" => "URI",
                          "url" => "URL",
                          "xml" => "XML",
                          "json" => "JSON",
-                         "gc" => "GC")
+                         "gc" => "GC",
+                         "kv" => "KV")
 loader.setup
 
 module Wayfarer
   module VERSION
     MAJOR = 0
     MINOR = 4
-    TINY = 6
+    TINY = 9
     STRING = [MAJOR, MINOR, TINY].join(".")
   end
 
   DEFAULT_CONFIG = {
     redis: {
       url: "redis://localhost:6379/0",
-      factory: ->(redis) { ::Redis.new(url: redis[:url]) }
+      factory: ->(redis_config) { ::Redis.new(url: redis_config.fetch(:url)) }
     },
     network: {
       agent: :http,
-      pool_size: 1,
-      pool_timeout: 10,
+      agents: {
+        http: Wayfarer::Networking::HTTP,
+        ferrum: Wayfarer::Networking::Ferrum,
+        selenium: Wayfarer::Networking::Selenium,
+        capybara: Wayfarer::Networking::Capybara
+      },
+      pool: {
+        size: 1,
+        timeout: 10
+      },
       http_headers: {},
       renew_on: []
     },
+    parsing: {
+      content_types: {
+        "application/json" => Wayfarer::Parsing::JSON,
+        "text/html" => [Wayfarer::Parsing::XML, :html],
+        "application/xml" => [Wayfarer::Parsing::XML, :xml]
+      }
+    },
+    normalization: {
+      remove_www: true,
+      remove_trailing_slash: true,
+      remove_fragment: true,
+      remove_tracking_parameters: true,
+      order_query_parameters: true,
+      schemes: %w[
+        http
+        https
+      ].to_set,
+      tracking_params: %w[
+        utm_source
+        utm_medium
+        utm_term
+        utm_content
+        utm_campaign
+        gclid
+        fbclid
+        msclkid
+        sms_ss
+        awesm
+        xtor
+        PHPSESSID
+      ].to_set
+    },
+    deduplication: {
+      key: ->(task) { task[:uri].to_s }
+    },
     capybara: {
       driver: nil
     },
@@ -62,12 +108,11 @@ module Wayfarer
     },
     selenium: {
       driver: :chrome,
-      options: {},
-      client_timeout: 60
+      options: {}
     }
   }.freeze
 
-  mattr_accessor :config, default: DEFAULT_CONFIG.clone
+  mattr_accessor :config, default: DEFAULT_CONFIG.deep_dup
 
   UserAgentTimeoutError = Class.new(StandardError) # TODO: Move to Networking namespace
 end

data/mise.toml

@@ -0,0 +1,2 @@
+[tools]
+ruby = "3.4.4"

data/mkdocs.yml

@@ -7,6 +7,7 @@ markdown_extensions:
   - attr_list
   - meta
   - def_list
+  - pymdownx.snippets
   - pymdownx.details
   - pymdownx.highlight
   - pymdownx.inlinehilite
@@ -19,7 +20,8 @@ markdown_extensions:
   - pymdownx.caret
   - pymdownx.mark
   - pymdownx.tilde
-  - pymdownx.tabbed
+  - pymdownx.tabbed:
+      alternate_style: true
   - pymdownx.tasklist:
       custom_checkbox: true
 
@@ -56,21 +58,14 @@ nav:
   - Home: index.md
   - Guides:
     - Tutorial: guides/tutorial.md
-    - Jobs:
-      - Overview: guides/jobs.md
-      - Error handling: guides/jobs/error_handling.md
+    - Jobs: guides/jobs.md
     - Tasks: guides/tasks.md
     - Pages: guides/pages.md
-    - Routing:
-        - Overview: guides/routing.md
-        - Matchers:
-          - URL: todo
-          - Host: todo
-          - Path: todo
-          - Query: todo
-          - Custom matchers: todo
+    - Routing: guides/routing.md
     - Callbacks: guides/callbacks.md
     - Handlers: guides/handlers.md
+    - Configuration: guides/configuration.md
+    - Command-line interface: guides/cli.md
     - Networking:
         - Introduction: guides/user_agents.md
         - User agent API: guides/networking/custom_adapters.md
@@ -80,13 +75,9 @@ nav:
           - Selenium: guides/networking/selenium.md
           - Capybara: guides/networking/capybara.md
     - Redis: guides/redis.md
-    - Design decisions: design.md
-  - Reference:
-    - Configuration: reference/configuration.md
-    - Command-line interface: reference/cli.md
+    - Development: guides/development.md
   - Cookbook:
     - Browser navigation: cookbook/navigation.md
     - Executing JavaScript: cookbook/executing_javascript.md
     - Screenhots: cookbook/screenshots.md
-    - Setting the User-Agent: cookbook/user_agent.md
   - API documentation: "https://www.rubydoc.info"

data/rake/lint.rake

@@ -1,105 +1,9 @@
 # frozen_string_literal: true
 
-require "open-uri"
-require "fileutils"
-require "rubygems/package"
-require "zlib"
-require "zip"
 require "rubocop/rake_task"
-require "net/http"
-require "uri"
-
-VALE_URL_ROOT = "https://github.com/errata-ai"
-VALE_VERSION = "2.15.4"
-VALE_STYLE_GUIDE_VERSION = "v0.3.3"
-VALE_PREFIX = File.expand_path(File.join(".github", "bin"))
-VALE_STYLE_GUIDE_PREFIX = File.expand_path(File.join(".github", "share"))
 
 namespace :lint do
   RuboCop::RakeTask.new do |task|
     task.formatters = %w[simple]
   end
-
-  vale_command = File.join(VALE_PREFIX, "vale")
-  style_guide_directory = File.join(VALE_STYLE_GUIDE_PREFIX, "vale")
-
-  desc "Lint documentation"
-  task vale: [:"lint:vale:clean", vale_command, style_guide_directory] do
-    sh "#{vale_command} docs/**/*.md"
-  end
-
-  namespace :vale do
-    directory VALE_PREFIX
-
-    desc "Install Vale to #{vale_command}"
-    file vale_command => VALE_PREFIX do
-      unless (local_command = `which vale`.strip).empty?
-        next File.symlink(local_command, vale_command)
-      end
-
-      filename = "vale_#{VALE_VERSION}_Linux_64-bit.tar.gz"
-      url = File.join(VALE_URL_ROOT, "/vale/releases/download/v#{VALE_VERSION}/#{filename}")
-
-      extract_tar_gz(url, vale_command)
-
-      FileUtils.chmod("+x", vale_command)
-    end
-
-    desc "Deletes Vale"
-    task clean: :"lint:vale:style_guide:clean" do
-      File.delete(vale_command) if File.exist?(vale_command)
-    end
-
-    namespace :style_guide do
-      directory VALE_STYLE_GUIDE_PREFIX
-
-      desc "Retrieve Vale Google style guide #{VALE_STYLE_GUIDE_VERSION}"
-      directory style_guide_directory => VALE_STYLE_GUIDE_PREFIX do
-        FileUtils.mkdir_p(style_guide_directory)
-        url = "https://github.com/errata-ai/Google/releases/download/#{VALE_STYLE_GUIDE_VERSION}/Google.zip"
-        extract_zip(url, style_guide_directory)
-      end
-
-      desc "Deletes the Vale Google style guide"
-      task :clean do
-        FileUtils.rm_rf([style_guide_directory])
-      end
-
-    private
-
-      def extract_zip(url, destination)
-        content = URI.open(url)
-
-        Zip::File.open_buffer(content) do |zip|
-          zip.each do |entry|
-            path = File.join(destination, entry.name)
-
-            if entry.directory?
-              FileUtils.mkdir_p(path)
-            else
-              entry.extract(path)
-              File.chmod(0o755, path)
-            end
-          end
-        end
-      end
-    end
-
-  private
-
-    def extract_tar_gz(url, destination)
-      URI.open(url) do |file|
-        Zlib::GzipReader.open(file) do |gz|
-          Gem::Package::TarReader.new(gz) do |tar|
-            tar.each do |entry|
-              next unless entry.file?
-
-              binary = entry.read
-              File.write(destination, binary)
-            end
-          end
-        end
-      end
-    end
-  end
 end

data/rake/release.rake

@@ -12,18 +12,6 @@ namespace :release do
 
     raise "Gem version #{gem_version} deviates from library version #{lib_version}" unless gem_version == lib_version
   end
-
-  # TODO: `GEM_HOST_API_KEY` is only supported for RubyGems 3+
-  task :write_credentials do
-    raise "RubyGems 3+ supports `GEM_HOST_API_KEY`" unless RUBY_VERSION.split(".").first.to_i == 2
-
-    key = ENV.fetch("GEM_HOST_API_KEY") { raise "`GEM_HOST_API_KEY` is unset" }
-    contents = YAML.dump(rubygems_api_key: "Basic #{key}")
-    gem_path = File.join(Dir.home, ".gem")
-    FileUtils.mkdir_p(gem_path)
-    File.write(File.join(gem_path, "credentials"), contents)
-  end
 end
 
-Rake::Task[:release].enhance(%i[release:guard_versions])
-Rake::Task[:"release:rubygem_push"].enhance(%i[release:write_credentials])
+Rake::Task[:"release:rubygem_push"].enhance(%i[release:guard_versions])

data/rake/tests.rake

@@ -8,21 +8,25 @@ RSpec::Core::RakeTask.new(:test)
 namespace :test do
   desc "Run only isolated tests"
   RSpec::Core::RakeTask.new :isolated do |task|
-    task.rspec_opts = ["--tag ~selenium --tag ~ferrum --tag ~cli"]
+    task.rspec_opts = %w[--tag ~selenium --tag ~ferrum --tag ~cli --tag ~redis]
+  end
+
+  RSpec::Core::RakeTask.new :integration do |task|
+    task.rspec_opts = %w[--tag redis --tag cli]
   end
 
   desc "Run only Selenium tests"
   RSpec::Core::RakeTask.new :selenium do |task|
-    task.rspec_opts = ["--tag selenium"]
+    task.rspec_opts = %w[--tag selenium]
   end
 
   desc "Run only Ferrum tests"
   RSpec::Core::RakeTask.new :ferrum do |task|
-    task.rspec_opts = ["--tag ferrum"]
+    task.rspec_opts = %w[--tag ferrum]
   end
 
   desc "Run only CLI tests"
   RSpec::Core::RakeTask.new :cli do |task|
-    task.rspec_opts = ["--tag cli"]
+    task.rspec_opts = %w[--tag cli]
   end
 end

data/requirements.txt

@@ -1 +1 @@
-mkdocs-material == 9.5.9
+mkdocs-material == 9.6.12

data/spec/factories/job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :job, class: ActiveJob::Base do
+    executions { 1 }
+    exception_executions { Object.new }
+  end
+end

data/spec/factories/middleware.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 FactoryBot.define do
-  factory :middleware, class: OpenStruct do
+  factory :middleware, class: Struct.new(:receiver) do
     receiver { -> {} }
 
     initialize_with do
-      new(receiver: receiver).tap do |middleware|
+      new(receiver).tap do |middleware|
         middleware.define_singleton_method(:call) do |task, &block|
           receiver.call(task, &block)
         end

data/spec/factories/path_finder.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :path_finder, class: Wayfarer::Routing::PathFinder do
+    url { "https://example.com" }
+
+    initialize_with do
+      new(build(:task, :uri, url: url))
+    end
+  end
+end

data/spec/factories/redis.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :counter, class: Wayfarer::Redis::Counter do
+    task { build(:task, :redis_pool) }
+
+    initialize_with do
+      new(task)
+    end
+  end
+
+  factory :barrier, class: Wayfarer::Redis::Barrier do
+    task { build(:task, :redis_pool) }
+
+    initialize_with do
+      new(task)
+    end
+  end
+end

data/spec/factories/task.rb

@@ -5,8 +5,24 @@ FactoryBot.define do
     url { test_app_path("/") }
     batch { "batch" }
 
+    transient do
+      job { nil }
+      controller { nil }
+      action { nil }
+    end
+
     initialize_with do
-      new(url, batch)
+      new(url, batch).tap do |task|
+        task[:job] = job
+        task[:controller] = controller
+        task[:action] = action
+      end
+    end
+
+    trait :staged_urls do
+      after(:build) do |task|
+        task[:staged_urls] = SortedSet.new
+      end
     end
 
     trait :redis_pool do
@@ -14,5 +30,27 @@ FactoryBot.define do
         task[:redis_pool] = Wayfarer::Redis::Pool.instance
       end
     end
+
+    trait :barrier do
+      redis_pool
+
+      after(:build) do |task|
+        task[:barrier] = Wayfarer::Redis::Barrier.new(task)
+      end
+    end
+
+    trait :uri do
+      after(:build) do |task|
+        task[:uri] = Addressable::URI.parse(task.url)
+      end
+    end
+
+    trait :normalized do
+      uri
+
+      after(:build) do |task|
+        task[:uri] = Wayfarer::URI::Normalization.canonical!(task[:uri])
+      end
+    end
   end
 end