wayfarer 0.4.7 → 0.4.8

data/lib/wayfarer/routing/route.rb

@@ -5,33 +5,90 @@ module Wayfarer
     class Route
       include DSL
       include Stringify
-      extend Forwardable
+      include Serializable
 
-      attr_reader :children
+      attr_reader :matcher,
+                  :parent,
+                  :children
 
-      attr_accessor :matcher,
-                    :parent,
-                    :action,
-                    :path_offset
+      stringify :matcher
 
-      stringify :matcher,
-                :action,
-                :path_offset
+      EMPTY_PARAMS = {}.freeze
 
-      def initialize(matcher = Matchers::Custom.new { children.any? }, path_offset = "/")
+      def initialize(
+        parent: nil,
+        matcher: nil,
+        action: nil,
+        **options,
+        &block
+      )
+        raise "missing parent" unless parent || is_a?(RootRoute)
+
+        @parent = parent
         @matcher = matcher
+        @action = action
+
         @children = []
-        @path_offset = path_offset
+
+        leaf = options.reduce(self) { |acc, (key, val)| acc.public_send(key, val) }
+        leaf.instance_eval(&block) if block
       end
 
-      # Accepts a visitor for in-order traversal.
+      # @return [true, false]
+      def root?
+        parent.nil?
+      end
+
+      # @return [true, false]
+      def leaf?
+        children.empty?
+      end
+
+      # @return [false]
+      def target?
+        false
+      end
+
+      # Accepts a visitor for pre-order traversal.
       def accept(visitor)
-        return unless visitor.visit(self)
+        visitor.enter(self)
+
+        return visitor.leave unless visitor.visit(self)
 
         children.each { |child| child.accept(visitor) }
+
+        visitor.leave
       end
 
-      delegate match: :matcher
+      # @param [path_finder] PathFinder
+      # @return [Hash]
+      def params(path_finder)
+        matcher&.params(path_finder) || EMPTY_PARAMS
+      end
+
+      # @param [_path_finder] PathFinder
+      # @return [nil, Symbol, Wayfarer::Handler]
+      def action(_path_finder)
+        @action
+      end
+
+      # @param [path_finder] PathFinder
+      # @return [Result::Match, Result::Mismatch, Object]
+      def match(path_finder)
+        evaluate(path_finder)
+      end
+
+      # @param [path_finder] PathFinder
+      # @return [true, false, Wayfarer::Routing::Route]
+      def evaluate(path_finder)
+        matcher.evaluate(path_finder)
+      end
+
+      def to_h
+        return {} unless matcher
+
+        { matcher.class.name.demodulize.underscore => matcher.to_h }
+      end
     end
   end
 end

data/lib/wayfarer/routing/serializable.rb

@@ -0,0 +1,28 @@
+# lib/wayfarer/routing/hash_serialisable.rb
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    module Serializable
+      def to_h
+        as_hash(self)
+      end
+
+    private
+
+      def as_hash(route)
+        {
+          matcher: matcher_name(route),
+          action: route.action(nil),
+          children: route.children.map { |child| as_hash(child) }
+        }.tap(&:compact!)
+      end
+
+      def matcher_name(route)
+        return nil unless route.matcher
+
+        route.matcher.class.name.split("::").last
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/sub_route.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class SubRoute < Route
+      def evaluate(path_finder)
+        handle(
+          path_finder,
+          match: ->(_result) { true },
+          mismatch: ->(_result) { false }
+        )
+      end
+
+      def params(path_finder)
+        handle(
+          path_finder,
+          match: lambda(&:params),
+          mismatch: ->(_result) { EMPTY_PARAMS }
+        )
+      end
+
+      def action(path_finder)
+        handle(
+          path_finder,
+          match: lambda(&:action),
+          mismatch: ->(_result) {}
+        )
+      end
+
+    private
+
+      def handle(path_finder, match:, mismatch:)
+        case root = evaluate_matcher(path_finder)
+        when Wayfarer::Routing::RootRoute
+          case result = sub_result(root, path_finder)
+          when Wayfarer::Routing::Result::Match then match.call(result)
+          when Wayfarer::Routing::Result::Mismatch then mismatch.call(result)
+          else raise "invalid result: #{result.inspect}"
+          end
+        else raise "#{route.inspect} is not a root route"
+        end
+      end
+
+      def evaluate_matcher(path_finder)
+        path_finder[matcher] ||= matcher.evaluate(path_finder)
+      end
+
+      def sub_result(route, path_finder)
+        path_finder[route] ||= Wayfarer::Routing::PathFinder.sub_result(route, path_finder)
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/target_route.rb

@@ -2,6 +2,22 @@
 
 module Wayfarer
   module Routing
-    class TargetRoute < Route; end
+    class TargetRoute < Route
+      def evaluate(_path_finder)
+        true
+      end
+
+      def target?
+        true
+      end
+
+      def to_h
+        { action: case @action
+                  when Wayfarer::Handler then { class: @action.class.name }
+                  when Array then { handler: @action.first.class.name, action: @action.second }
+                  else @action
+                  end }
+      end
+    end
   end
 end

data/lib/wayfarer/stringify.rb

@@ -25,8 +25,7 @@ module Wayfarer
       if self.class.stringified_attributes.any?
         attrs = self.class
                     .stringified_attributes
-                    .map { |attr| [attr, public_send(attr)] }
-                    .to_h
+                    .to_h { |attr| [attr, public_send(attr)] }
                     .map { |k, v| "#{k}=#{v.inspect}" }
                     .join(", ")
 

data/lib/wayfarer/task.rb

@@ -6,20 +6,18 @@ module Wayfarer
   # @!attribute [r] batch
   #   @return [String] the batch the task belongs to
   class Task
-    extend Forwardable
+    include KV
     include Stringify
 
-    attr_reader :url, :batch
+    attr_reader :url,
+                :batch
 
     stringify :url, :batch
 
-    delegate %i([] []=) => :@ephemeral
-
     # @!visibility private
     def initialize(url, batch)
       @url = url
       @batch = batch
-      @ephemeral = {}
     end
 
     # @!visibility private

data/lib/wayfarer/uri/normalization.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module URI
+    # HTTP(S) URL normalization.
+    module Normalization
+      InvalidURIError = Class.new(StandardError)
+
+      # Raised when URI is relative.
+      RelativeURIError = Class.new(InvalidURIError)
+
+      # Raised when URI scheme is not hypertext.
+      NoHypertextError = Class.new(InvalidURIError)
+
+      # Raised when URI has no host.
+      NoHostError = Class.new(InvalidURIError)
+
+      extend self
+
+      # Normalizes `uri` in-place.
+      # @param uri [Addressable::URI]
+      # @raise [InvalidURIError]
+      # @return [Addressable::URI]
+      def canonical!(uri)
+        had_no_path = uri.path.blank?
+
+        uri.normalize!
+        validate_uri!(uri)
+
+        normalize_host!(uri) if remove_www?
+
+        if remove_trailing_slash?
+          normalize_path!(uri)
+          root_path!(uri)
+        end
+
+        remove_fragment!(uri) if remove_fragment?
+        normalize_query_params!(uri)
+
+        root_path!(uri) if had_no_path && uri.query.nil?
+
+        uri
+      end
+
+    private
+
+      def validate_uri!(uri)
+        raise RelativeURIError, "URL is not absolute" unless uri.absolute?
+        raise NoHypertextError, "URL is using unsupported protocol" unless supported_protocols.include?(uri.scheme)
+        raise NoHostError, "URL misses hostname" if uri.host.blank?
+      end
+
+      def normalize_query_params!(uri)
+        return unless remove_tracking_parameters? || order_query_parameters?
+        return unless (params = uri.query_values(Array))
+
+        remove_tracking_parameters!(params) if remove_tracking_parameters?
+        order_query_parameters!(params) if order_query_parameters?
+
+        uri.query_values = params.empty? ? nil : params
+      end
+
+      def remove_tracking_parameters!(params)
+        params.reject! { |key, val| val.to_s.empty? || tracking_params.include?(key) }
+      end
+
+      def order_query_parameters!(params)
+        params.sort_by!(&:first)
+      end
+
+      def normalize_host!(uri)
+        uri.host &&= uri.host.delete_prefix("www.")
+      end
+
+      def normalize_path!(uri)
+        uri.path = uri.path.delete_suffix(File::SEPARATOR) if uri.path && uri.path.length > 1
+      end
+
+      def remove_fragment!(uri)
+        uri.fragment = nil
+      end
+
+      def root_path!(uri)
+        uri.path = "" if uri.path == File::SEPARATOR
+      end
+
+      def normalization_config
+        Wayfarer.config.fetch(:normalization)
+      end
+
+      def supported_protocols
+        normalization_config.fetch(:schemes)
+      end
+
+      def tracking_params
+        normalization_config.fetch(:tracking_params)
+      end
+
+      def remove_www?
+        normalization_config.fetch(:remove_www)
+      end
+
+      def remove_trailing_slash?
+        normalization_config.fetch(:remove_trailing_slash)
+      end
+
+      def remove_fragment?
+        normalization_config.fetch(:remove_fragment)
+      end
+
+      def remove_tracking_parameters?
+        normalization_config.fetch(:remove_tracking_parameters)
+      end
+
+      def order_query_parameters?
+        normalization_config.fetch(:order_query_parameters)
+      end
+    end
+  end
+end

data/lib/wayfarer.rb

@@ -3,13 +3,16 @@
 require "cgi"
 require "forwardable"
 require "net/http"
+require "pp"
 require "securerandom"
 require "uri"
+require "yaml"
 
 require "active_job"
+require "active_support/core_ext/array/wrap"
+require "active_support/core_ext/object/deep_dup"
 require "capybara"
 require "connection_pool"
-require "docile"
 require "ferrum"
 require "metainspector"
 require "mime/types"
@@ -18,7 +21,6 @@ require "mock_redis"
 require "mustermann"
 require "net/http/persistent"
 require "nokogiri"
-require "normalize_url"
 require "selenium-webdriver"
 require "redis"
 require "thor"
@@ -28,32 +30,71 @@ loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect("cli" => "CLI",
                          "dsl" => "DSL",
                          "http" => "HTTP",
+                         "uri" => "URI",
                          "url" => "URL",
                          "xml" => "XML",
                          "json" => "JSON",
-                         "gc" => "GC")
+                         "gc" => "GC",
+                         "kv" => "KV")
 loader.setup
 
 module Wayfarer
   module VERSION
     MAJOR = 0
     MINOR = 4
-    TINY = 6
+    TINY = 8
     STRING = [MAJOR, MINOR, TINY].join(".")
   end
 
   DEFAULT_CONFIG = {
     redis: {
       url: "redis://localhost:6379/0",
-      factory: ->(redis) { ::Redis.new(url: redis[:url]) }
+      factory: ->(redis_config) { ::Redis.new(url: redis_config.fetch(:url)) }
     },
     network: {
       agent: :http,
-      pool_size: 1,
-      pool_timeout: 10,
+      pool: {
+        size: 1,
+        timeout: 10
+      },
       http_headers: {},
       renew_on: []
     },
+    parsing: {
+      registry: {
+        "application/json" => Wayfarer::Parsing::JSON,
+        "text/html" => [Wayfarer::Parsing::XML, :html],
+        "application/xml" => [Wayfarer::Parsing::XML, :xml]
+      }
+    },
+    normalization: {
+      remove_www: true,
+      remove_trailing_slash: true,
+      remove_fragment: true,
+      remove_tracking_parameters: true,
+      order_query_parameters: true,
+      schemes: %w[
+        http
+        https
+      ].to_set,
+      tracking_params: %w[
+        utm_source
+        utm_medium
+        utm_term
+        utm_content
+        utm_campaign
+        gclid
+        fbclid
+        msclkid
+        sms_ss
+        awesm
+        xtor
+        PHPSESSID
+      ].to_set
+    },
+    deduplication: {
+      key: ->(task) { task[:uri].to_s }
+    },
     capybara: {
       driver: nil
     },
@@ -62,12 +103,11 @@ module Wayfarer
     },
     selenium: {
       driver: :chrome,
-      options: {},
-      client_timeout: 60
+      options: {}
     }
   }.freeze
 
-  mattr_accessor :config, default: DEFAULT_CONFIG.clone
+  mattr_accessor :config, default: DEFAULT_CONFIG.deep_dup
 
   UserAgentTimeoutError = Class.new(StandardError) # TODO: Move to Networking namespace
 end

data/mise.toml

@@ -0,0 +1,2 @@
+[tools]
+ruby = "3.4.4"

data/mkdocs.yml

@@ -7,6 +7,7 @@ markdown_extensions:
   - attr_list
   - meta
   - def_list
+  - pymdownx.snippets
   - pymdownx.details
   - pymdownx.highlight
   - pymdownx.inlinehilite
@@ -19,7 +20,8 @@ markdown_extensions:
   - pymdownx.caret
   - pymdownx.mark
   - pymdownx.tilde
-  - pymdownx.tabbed
+  - pymdownx.tabbed:
+      alternate_style: true
   - pymdownx.tasklist:
       custom_checkbox: true
 
@@ -56,21 +58,14 @@ nav:
   - Home: index.md
   - Guides:
     - Tutorial: guides/tutorial.md
-    - Jobs:
-      - Overview: guides/jobs.md
-      - Error handling: guides/jobs/error_handling.md
+    - Jobs: guides/jobs.md
     - Tasks: guides/tasks.md
     - Pages: guides/pages.md
-    - Routing:
-        - Overview: guides/routing.md
-        - Matchers:
-          - URL: todo
-          - Host: todo
-          - Path: todo
-          - Query: todo
-          - Custom matchers: todo
+    - Routing: guides/routing.md
     - Callbacks: guides/callbacks.md
     - Handlers: guides/handlers.md
+    - Configuration: guides/configuration.md
+    - Command-line interface: guides/cli.md
     - Networking:
         - Introduction: guides/user_agents.md
         - User agent API: guides/networking/custom_adapters.md
@@ -80,13 +75,9 @@ nav:
           - Selenium: guides/networking/selenium.md
           - Capybara: guides/networking/capybara.md
     - Redis: guides/redis.md
-    - Design decisions: design.md
-  - Reference:
-    - Configuration: reference/configuration.md
-    - Command-line interface: reference/cli.md
+    - Development: guides/development.md
   - Cookbook:
     - Browser navigation: cookbook/navigation.md
     - Executing JavaScript: cookbook/executing_javascript.md
     - Screenhots: cookbook/screenshots.md
-    - Setting the User-Agent: cookbook/user_agent.md
   - API documentation: "https://www.rubydoc.info"

data/rake/lint.rake

@@ -1,105 +1,9 @@
 # frozen_string_literal: true
 
-require "open-uri"
-require "fileutils"
-require "rubygems/package"
-require "zlib"
-require "zip"
 require "rubocop/rake_task"
-require "net/http"
-require "uri"
-
-VALE_URL_ROOT = "https://github.com/errata-ai"
-VALE_VERSION = "2.15.4"
-VALE_STYLE_GUIDE_VERSION = "v0.3.3"
-VALE_PREFIX = File.expand_path(File.join(".github", "bin"))
-VALE_STYLE_GUIDE_PREFIX = File.expand_path(File.join(".github", "share"))
 
 namespace :lint do
   RuboCop::RakeTask.new do |task|
     task.formatters = %w[simple]
   end
-
-  vale_command = File.join(VALE_PREFIX, "vale")
-  style_guide_directory = File.join(VALE_STYLE_GUIDE_PREFIX, "vale")
-
-  desc "Lint documentation"
-  task vale: [:"lint:vale:clean", vale_command, style_guide_directory] do
-    sh "#{vale_command} docs/**/*.md"
-  end
-
-  namespace :vale do
-    directory VALE_PREFIX
-
-    desc "Install Vale to #{vale_command}"
-    file vale_command => VALE_PREFIX do
-      unless (local_command = `which vale`.strip).empty?
-        next File.symlink(local_command, vale_command)
-      end
-
-      filename = "vale_#{VALE_VERSION}_Linux_64-bit.tar.gz"
-      url = File.join(VALE_URL_ROOT, "/vale/releases/download/v#{VALE_VERSION}/#{filename}")
-
-      extract_tar_gz(url, vale_command)
-
-      FileUtils.chmod("+x", vale_command)
-    end
-
-    desc "Deletes Vale"
-    task clean: :"lint:vale:style_guide:clean" do
-      File.delete(vale_command) if File.exist?(vale_command)
-    end
-
-    namespace :style_guide do
-      directory VALE_STYLE_GUIDE_PREFIX
-
-      desc "Retrieve Vale Google style guide #{VALE_STYLE_GUIDE_VERSION}"
-      directory style_guide_directory => VALE_STYLE_GUIDE_PREFIX do
-        FileUtils.mkdir_p(style_guide_directory)
-        url = "https://github.com/errata-ai/Google/releases/download/#{VALE_STYLE_GUIDE_VERSION}/Google.zip"
-        extract_zip(url, style_guide_directory)
-      end
-
-      desc "Deletes the Vale Google style guide"
-      task :clean do
-        FileUtils.rm_rf([style_guide_directory])
-      end
-
-    private
-
-      def extract_zip(url, destination)
-        content = URI.open(url)
-
-        Zip::File.open_buffer(content) do |zip|
-          zip.each do |entry|
-            path = File.join(destination, entry.name)
-
-            if entry.directory?
-              FileUtils.mkdir_p(path)
-            else
-              entry.extract(path)
-              File.chmod(0o755, path)
-            end
-          end
-        end
-      end
-    end
-
-  private
-
-    def extract_tar_gz(url, destination)
-      URI.open(url) do |file|
-        Zlib::GzipReader.open(file) do |gz|
-          Gem::Package::TarReader.new(gz) do |tar|
-            tar.each do |entry|
-              next unless entry.file?
-
-              binary = entry.read
-              File.write(destination, binary)
-            end
-          end
-        end
-      end
-    end
-  end
 end

data/rake/release.rake

@@ -13,17 +13,11 @@ namespace :release do
     raise "Gem version #{gem_version} deviates from library version #{lib_version}" unless gem_version == lib_version
   end
 
-  # TODO: `GEM_HOST_API_KEY` is only supported for RubyGems 3+
-  task :write_credentials do
-    raise "RubyGems 3+ supports `GEM_HOST_API_KEY`" unless RUBY_VERSION.split(".").first.to_i == 2
-
-    key = ENV.fetch("GEM_HOST_API_KEY") { raise "`GEM_HOST_API_KEY` is unset" }
-    contents = YAML.dump(rubygems_api_key: "Basic #{key}")
-    gem_path = File.join(Dir.home, ".gem")
-    FileUtils.mkdir_p(gem_path)
-    File.write(File.join(gem_path, "credentials"), contents)
+  task :guard_debug do
+    sh "git diff --exit-code"
+    sh "git diff-index --quiet --cached HEAD"
   end
 end
 
-Rake::Task[:release].enhance(%i[release:guard_versions])
-Rake::Task[:"release:rubygem_push"].enhance(%i[release:write_credentials])
+Rake::Task[:"release:rubygem_push"].enhance(%i[release:guard_versions])
+Rake::Task[:"release:guard_clean"].enhance(%i[release:guard_debug])