wayfarer 0.4.6 → 0.4.8

data/lib/wayfarer/routing/route.rb

@@ -5,42 +5,89 @@ module Wayfarer
     class Route
       include DSL
       include Stringify
+      include Serializable
 
-      attr_reader :children
+      attr_reader :matcher,
+                  :parent,
+                  :children
 
-      attr_accessor :matcher,
-                    :parent,
-                    :action,
-                    :path_offset
+      stringify :matcher
 
-      stringify :matcher,
-                :action,
-                :path_offset,
-                :children
+      EMPTY_PARAMS = {}.freeze
 
-      def initialize(matcher = Matchers::Custom.new { children.any? }, path_offset = "/")
+      def initialize(
+        parent: nil,
+        matcher: nil,
+        action: nil,
+        **options,
+        &block
+      )
+        raise "missing parent" unless parent || is_a?(RootRoute)
+
+        @parent = parent
         @matcher = matcher
+        @action = action
+
         @children = []
-        @path_offset = path_offset
+
+        leaf = options.reduce(self) { |acc, (key, val)| acc.public_send(key, val) }
+        leaf.instance_eval(&block) if block
       end
 
-      def match(url)
-        matcher.match(url)
+      # @return [true, false]
+      def root?
+        parent.nil?
       end
 
-      def matches?(url)
-        invoke(url).is_a?(Result::Match)
+      # @return [true, false]
+      def leaf?
+        children.empty?
       end
 
-      def invoke(url)
-        PathFinder.result(self, url)
+      # @return [false]
+      def target?
+        false
       end
 
-      # Accepts a visitor for in-order traversal.
+      # Accepts a visitor for pre-order traversal.
       def accept(visitor)
-        return unless visitor.visit(self)
+        visitor.enter(self)
+
+        return visitor.leave unless visitor.visit(self)
 
         children.each { |child| child.accept(visitor) }
+
+        visitor.leave
+      end
+
+      # @param [path_finder] PathFinder
+      # @return [Hash]
+      def params(path_finder)
+        matcher&.params(path_finder) || EMPTY_PARAMS
+      end
+
+      # @param [_path_finder] PathFinder
+      # @return [nil, Symbol, Wayfarer::Handler]
+      def action(_path_finder)
+        @action
+      end
+
+      # @param [path_finder] PathFinder
+      # @return [Result::Match, Result::Mismatch, Object]
+      def match(path_finder)
+        evaluate(path_finder)
+      end
+
+      # @param [path_finder] PathFinder
+      # @return [true, false, Wayfarer::Routing::Route]
+      def evaluate(path_finder)
+        matcher.evaluate(path_finder)
+      end
+
+      def to_h
+        return {} unless matcher
+
+        { matcher.class.name.demodulize.underscore => matcher.to_h }
       end
     end
   end

data/lib/wayfarer/routing/serializable.rb

@@ -0,0 +1,28 @@
+# lib/wayfarer/routing/hash_serialisable.rb
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    module Serializable
+      def to_h
+        as_hash(self)
+      end
+
+    private
+
+      def as_hash(route)
+        {
+          matcher: matcher_name(route),
+          action: route.action(nil),
+          children: route.children.map { |child| as_hash(child) }
+        }.tap(&:compact!)
+      end
+
+      def matcher_name(route)
+        return nil unless route.matcher
+
+        route.matcher.class.name.split("::").last
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/sub_route.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class SubRoute < Route
+      def evaluate(path_finder)
+        handle(
+          path_finder,
+          match: ->(_result) { true },
+          mismatch: ->(_result) { false }
+        )
+      end
+
+      def params(path_finder)
+        handle(
+          path_finder,
+          match: lambda(&:params),
+          mismatch: ->(_result) { EMPTY_PARAMS }
+        )
+      end
+
+      def action(path_finder)
+        handle(
+          path_finder,
+          match: lambda(&:action),
+          mismatch: ->(_result) {}
+        )
+      end
+
+    private
+
+      def handle(path_finder, match:, mismatch:)
+        case root = evaluate_matcher(path_finder)
+        when Wayfarer::Routing::RootRoute
+          case result = sub_result(root, path_finder)
+          when Wayfarer::Routing::Result::Match then match.call(result)
+          when Wayfarer::Routing::Result::Mismatch then mismatch.call(result)
+          else raise "invalid result: #{result.inspect}"
+          end
+        else raise "#{route.inspect} is not a root route"
+        end
+      end
+
+      def evaluate_matcher(path_finder)
+        path_finder[matcher] ||= matcher.evaluate(path_finder)
+      end
+
+      def sub_result(route, path_finder)
+        path_finder[route] ||= Wayfarer::Routing::PathFinder.sub_result(route, path_finder)
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/target_route.rb

@@ -2,6 +2,22 @@
 
 module Wayfarer
   module Routing
-    class TargetRoute < Route; end
+    class TargetRoute < Route
+      def evaluate(_path_finder)
+        true
+      end
+
+      def target?
+        true
+      end
+
+      def to_h
+        { action: case @action
+                  when Wayfarer::Handler then { class: @action.class.name }
+                  when Array then { handler: @action.first.class.name, action: @action.second }
+                  else @action
+                  end }
+      end
+    end
   end
 end

data/lib/wayfarer/stringify.rb

@@ -1,46 +1,37 @@
 # frozen_string_literal: true
 
-# 346325
-
 module Wayfarer
   module Stringify
-    def self.included(base)
-      base.include(InstanceMethods)
-      base.extend(ClassMethods)
-      base.instance_eval do
-        cattr_accessor :stringified_attributes do
-          []
-        end
-      end
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :stringified_attributes,
+                      default: [],
+                      instance_accessor: false,
+                      instance_predicate: false
+
+      alias_method :inspect, :to_s
     end
 
-    module ClassMethods
+    class_methods do
       def stringify(*variables)
         stringified_attributes.concat(variables)
       end
     end
 
-    module InstanceMethods
-      def to_s
-        if self.class.stringified_attributes.any?
-          "#<#{class_name} #{attributes.join(', ')}>"
-        else
-          "#<#{class_name}>"
-        end
-      end
-
-      alias inspect to_s
+    def to_s
+      class_name = self.class.name
 
-      def class_name
-        self.class.name
-      end
+      if self.class.stringified_attributes.any?
+        attrs = self.class
+                    .stringified_attributes
+                    .to_h { |attr| [attr, public_send(attr)] }
+                    .map { |k, v| "#{k}=#{v.inspect}" }
+                    .join(", ")
 
-      def attributes
-        self.class
-            .stringified_attributes
-            .map { |attr| [attr, public_send(attr)] }
-            .to_h
-            .map { |k, v| [k, "=", v.inspect].join }
+        "#<#{class_name} #{attrs}>"
+      else
+        "#<#{class_name}>"
       end
     end
   end

data/lib/wayfarer/task.rb

@@ -1,36 +1,28 @@
 # frozen_string_literal: true
 
 module Wayfarer
+  # @!attribute [r] url
+  #   @return [String] the URL to process
+  # @!attribute [r] batch
+  #   @return [String] the batch the task belongs to
   class Task
+    include KV
     include Stringify
 
     attr_reader :url,
-                :batch,
-                :metadata
+                :batch
 
-    stringify :url,
-              :batch
+    stringify :url, :batch
 
+    # @!visibility private
     def initialize(url, batch)
       @url = url
       @batch = batch
-      @metadata = OpenStruct.new
     end
 
+    # @!visibility private
     def ==(other)
       [url, batch] == [other.url, other.batch]
     end
-
-    def barrier
-      @barrier ||= Wayfarer::Redis::Barrier.new(batch)
-    end
-
-    def counter
-      @counter ||= Wayfarer::Redis::Counter.new(batch)
-    end
-
-    def gc
-      @gc ||= Wayfarer::GC.new(self)
-    end
   end
 end

data/lib/wayfarer/uri/normalization.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module URI
+    # HTTP(S) URL normalization.
+    module Normalization
+      InvalidURIError = Class.new(StandardError)
+
+      # Raised when URI is relative.
+      RelativeURIError = Class.new(InvalidURIError)
+
+      # Raised when URI scheme is not hypertext.
+      NoHypertextError = Class.new(InvalidURIError)
+
+      # Raised when URI has no host.
+      NoHostError = Class.new(InvalidURIError)
+
+      extend self
+
+      # Normalizes `uri` in-place.
+      # @param uri [Addressable::URI]
+      # @raise [InvalidURIError]
+      # @return [Addressable::URI]
+      def canonical!(uri)
+        had_no_path = uri.path.blank?
+
+        uri.normalize!
+        validate_uri!(uri)
+
+        normalize_host!(uri) if remove_www?
+
+        if remove_trailing_slash?
+          normalize_path!(uri)
+          root_path!(uri)
+        end
+
+        remove_fragment!(uri) if remove_fragment?
+        normalize_query_params!(uri)
+
+        root_path!(uri) if had_no_path && uri.query.nil?
+
+        uri
+      end
+
+    private
+
+      def validate_uri!(uri)
+        raise RelativeURIError, "URL is not absolute" unless uri.absolute?
+        raise NoHypertextError, "URL is using unsupported protocol" unless supported_protocols.include?(uri.scheme)
+        raise NoHostError, "URL misses hostname" if uri.host.blank?
+      end
+
+      def normalize_query_params!(uri)
+        return unless remove_tracking_parameters? || order_query_parameters?
+        return unless (params = uri.query_values(Array))
+
+        remove_tracking_parameters!(params) if remove_tracking_parameters?
+        order_query_parameters!(params) if order_query_parameters?
+
+        uri.query_values = params.empty? ? nil : params
+      end
+
+      def remove_tracking_parameters!(params)
+        params.reject! { |key, val| val.to_s.empty? || tracking_params.include?(key) }
+      end
+
+      def order_query_parameters!(params)
+        params.sort_by!(&:first)
+      end
+
+      def normalize_host!(uri)
+        uri.host &&= uri.host.delete_prefix("www.")
+      end
+
+      def normalize_path!(uri)
+        uri.path = uri.path.delete_suffix(File::SEPARATOR) if uri.path && uri.path.length > 1
+      end
+
+      def remove_fragment!(uri)
+        uri.fragment = nil
+      end
+
+      def root_path!(uri)
+        uri.path = "" if uri.path == File::SEPARATOR
+      end
+
+      def normalization_config
+        Wayfarer.config.fetch(:normalization)
+      end
+
+      def supported_protocols
+        normalization_config.fetch(:schemes)
+      end
+
+      def tracking_params
+        normalization_config.fetch(:tracking_params)
+      end
+
+      def remove_www?
+        normalization_config.fetch(:remove_www)
+      end
+
+      def remove_trailing_slash?
+        normalization_config.fetch(:remove_trailing_slash)
+      end
+
+      def remove_fragment?
+        normalization_config.fetch(:remove_fragment)
+      end
+
+      def remove_tracking_parameters?
+        normalization_config.fetch(:remove_tracking_parameters)
+      end
+
+      def order_query_parameters?
+        normalization_config.fetch(:order_query_parameters)
+      end
+    end
+  end
+end

data/lib/wayfarer.rb

@@ -3,13 +3,16 @@
 require "cgi"
 require "forwardable"
 require "net/http"
+require "pp"
 require "securerandom"
 require "uri"
+require "yaml"
 
 require "active_job"
+require "active_support/core_ext/array/wrap"
+require "active_support/core_ext/object/deep_dup"
 require "capybara"
 require "connection_pool"
-require "docile"
 require "ferrum"
 require "metainspector"
 require "mime/types"
@@ -18,7 +21,6 @@ require "mock_redis"
 require "mustermann"
 require "net/http/persistent"
 require "nokogiri"
-require "normalize_url"
 require "selenium-webdriver"
 require "redis"
 require "thor"
@@ -28,21 +30,84 @@ loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect("cli" => "CLI",
                          "dsl" => "DSL",
                          "http" => "HTTP",
+                         "uri" => "URI",
                          "url" => "URL",
                          "xml" => "XML",
                          "json" => "JSON",
-                         "gc" => "GC")
+                         "gc" => "GC",
+                         "kv" => "KV")
 loader.setup
 
 module Wayfarer
   module VERSION
     MAJOR = 0
     MINOR = 4
-    TINY = 6
+    TINY = 8
     STRING = [MAJOR, MINOR, TINY].join(".")
   end
 
-  mattr_accessor :config, default: Wayfarer::Config::Root.new
+  DEFAULT_CONFIG = {
+    redis: {
+      url: "redis://localhost:6379/0",
+      factory: ->(redis_config) { ::Redis.new(url: redis_config.fetch(:url)) }
+    },
+    network: {
+      agent: :http,
+      pool: {
+        size: 1,
+        timeout: 10
+      },
+      http_headers: {},
+      renew_on: []
+    },
+    parsing: {
+      registry: {
+        "application/json" => Wayfarer::Parsing::JSON,
+        "text/html" => [Wayfarer::Parsing::XML, :html],
+        "application/xml" => [Wayfarer::Parsing::XML, :xml]
+      }
+    },
+    normalization: {
+      remove_www: true,
+      remove_trailing_slash: true,
+      remove_fragment: true,
+      remove_tracking_parameters: true,
+      order_query_parameters: true,
+      schemes: %w[
+        http
+        https
+      ].to_set,
+      tracking_params: %w[
+        utm_source
+        utm_medium
+        utm_term
+        utm_content
+        utm_campaign
+        gclid
+        fbclid
+        msclkid
+        sms_ss
+        awesm
+        xtor
+        PHPSESSID
+      ].to_set
+    },
+    deduplication: {
+      key: ->(task) { task[:uri].to_s }
+    },
+    capybara: {
+      driver: nil
+    },
+    ferrum: {
+      options: {}
+    },
+    selenium: {
+      driver: :chrome,
+      options: {}
+    }
+  }.freeze
+
+  mattr_accessor :config, default: DEFAULT_CONFIG.deep_dup
 
   UserAgentTimeoutError = Class.new(StandardError) # TODO: Move to Networking namespace
 end
@@ -50,3 +115,5 @@ end
 loader.eager_load
 
 ActiveJob::Serializers.serializers << Wayfarer::Serializer
+
+Wayfarer::BatchCompletion.subscribe!

data/mise.toml

@@ -0,0 +1,2 @@
+[tools]
+ruby = "3.4.4"

data/mkdocs.yml

@@ -1,28 +1,40 @@
 repo_url: https://github.com/bauerd/wayfarer
 edit_uri: edit/develop/docs/
-repo_name: Source Code
+repo_name: Code
 site_name: Wayfarer
 markdown_extensions:
   - admonition
+  - attr_list
   - meta
   - def_list
+  - pymdownx.snippets
   - pymdownx.details
   - pymdownx.highlight
-  - pymdownx.superfences
+  - pymdownx.inlinehilite
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
   - pymdownx.critic
   - pymdownx.caret
   - pymdownx.mark
   - pymdownx.tilde
-  - pymdownx.tabbed
+  - pymdownx.tabbed:
+      alternate_style: true
   - pymdownx.tasklist:
       custom_checkbox: true
 
 theme:
+  icon:
+    logo: material/sign-direction
+    repo: fontawesome/brands/git
   name: material
   features:
     - navigation.tabs
-    - navigation.sections
-    - navigation.expand
+    # - navigation.sections
+    # - navigation.expand
+    - navigation.indexes
   font:
     text: IBM Plex Sans
     code: IBM Plex Sans Mono
@@ -41,7 +53,31 @@ theme:
       toggle:
         icon: material/lightbulb
         name: Switch to light mode
-  icon:
-    repo: fontawesome/brands/git
-
 
+nav:
+  - Home: index.md
+  - Guides:
+    - Tutorial: guides/tutorial.md
+    - Jobs: guides/jobs.md
+    - Tasks: guides/tasks.md
+    - Pages: guides/pages.md
+    - Routing: guides/routing.md
+    - Callbacks: guides/callbacks.md
+    - Handlers: guides/handlers.md
+    - Configuration: guides/configuration.md
+    - Command-line interface: guides/cli.md
+    - Networking:
+        - Introduction: guides/user_agents.md
+        - User agent API: guides/networking/custom_adapters.md
+        - Built-in user agents:
+          - Plain HTTP: guides/networking/http.md
+          - Ferrum: guides/networking/ferrum.md
+          - Selenium: guides/networking/selenium.md
+          - Capybara: guides/networking/capybara.md
+    - Redis: guides/redis.md
+    - Development: guides/development.md
+  - Cookbook:
+    - Browser navigation: cookbook/navigation.md
+    - Executing JavaScript: cookbook/executing_javascript.md
+    - Screenhots: cookbook/screenshots.md
+  - API documentation: "https://www.rubydoc.info"

data/rake/docs.rake

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "yard"
+
+namespace :yard do
+  desc "Generate documentation"
+  YARD::Rake::YardocTask.new(:generate) do |t|
+    require_relative "../lib/wayfarer"
+
+    t.options = %w[--readme docs/index.md --no-private --markup markdown]
+    t.files = [
+      Wayfarer::Base,
+      Wayfarer::Task,
+      Wayfarer::Page,
+      Wayfarer::Handler,
+      Wayfarer::Routing::DSL,
+      Wayfarer::Parsing
+    ].freeze.map { |klass| Object.const_source_location(klass.name).first }
+  end
+
+  desc "Regenerate documentation on change"
+  task :watch do
+    # The output of `yard server` differs for some reason
+    sh "rerun --wait 0.5 --dir lib bundle exec rake yard:generate"
+  end
+end

data/rake/lint.rake

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "rubocop/rake_task"
+
+namespace :lint do
+  RuboCop::RakeTask.new do |task|
+    task.formatters = %w[simple]
+  end
+end