wayfarer 0.0.3 → 0.4.0

data/lib/wayfarer/routing/path_finder.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class PathFinder
+      def self.result(route, url)
+        finder = new(url)
+        route.accept(finder)
+        return Result::Mismatch.new if finder.path.none?
+
+        Result::Match.new(finder.action, finder.params)
+      end
+
+      attr_reader :url,
+                  :path,
+                  :action,
+                  :params
+
+      def initialize(url)
+        @url = url
+        @path = []
+        @action = nil
+        @params = {}
+      end
+
+      def visit(route)
+        return false if path.any?
+        return false unless route.match(url)
+
+        follow(route) if route.children.none?
+        true
+      end
+
+    private
+
+      # Follows the route back to the root. Collects parameters and action on
+      # the way.
+      def follow(route)
+        path.unshift(route)
+        @params.merge!(route.matcher.params(url))
+        @action = route.action unless action
+        follow(route.parent) if route.parent
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/path_matcher.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class PathMatcher
+      attr_reader :path,
+                  :route,
+                  :peeking,
+                  :matcher
+
+      def initialize(path, route)
+        @path = path
+        @route = route
+        @peeking = false
+        @matcher = Mustermann.new(path, type: "sinatra")
+      end
+
+      def match(url)
+        route.accept(self)
+
+        # If the route's branch contains other path matchers in child routes,
+        # match the beginning of the path (peeking), instead of the whole path.
+        !!(if peeking
+             matcher.peek(url.path)
+           else
+             matcher.match(url.path)
+           end)
+      end
+
+      def params(url)
+        return {} unless match(url)
+
+        matcher.params(url.path) || {}
+      end
+
+      def visit(route)
+        return true if route == self.route
+
+        return unless route.matcher.is_a?(self.class)
+
+        @peeking = true
+        false
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/{query_rule.rb → query_matcher.rb}

@@ -1,29 +1,37 @@
 # frozen_string_literal: true
 
-require "cgi"
-
 module Wayfarer
   module Routing
-    # @private
-    class QueryRule < Rule
-      def initialize(field_constraints, opts = {}, &proc)
-        @field_constraints = field_constraints
-        super(opts, &proc)
+    class QueryMatcher
+      attr_reader :fields
+
+      def initialize(fields)
+        @fields = fields
+      end
+
+      def match(url)
+        query = url.query
+
+        # CGI::parse throws a NoMethodError if the query is an empty string
+        return false if query.nil? || query.empty?
+
+        CGI.parse(query).none? { |field, vals| violates?(field, vals) }
       end
 
-      private
+      def params(url)
+        return {} unless match(url)
 
-      def match!(uri)
-        CGI.parse(uri.query).none? do |field, vals| violates?(field, vals) end
-      rescue NoMethodError
-        # `CGI::parse` throws a `NoMethodError` if `uri.query` is an empty
-        # string
-        false
+        CGI.parse(url.query)
+           .select { |(k, _)| fields.keys.include?(k.to_sym) }
+           .transform_values(&:last)
       end
 
+    private
+
       # rubocop:disable Lint/AssignmentInCondition
       def violates?(field, vals)
-        return false unless constraint = @field_constraints[field.to_sym]
+        return false unless constraint = fields[field.to_sym]
+
         violates_constraint?(constraint, vals)
       end
       # rubocop:enable Lint/AssignmentInCondition
@@ -42,7 +50,7 @@ module Wayfarer
       end
 
       def violates_integer?(int, vals)
-        vals.none? do |val| int == Integer(val) end
+        vals.none? { |val| int == Integer(val) }
       rescue ArgumentError
         true
       end

data/lib/wayfarer/routing/result.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    module Result
+      # Signals that a URL matched a route.
+      # @!attribute [rw] action
+      #   @return [Page] the retrieved page.
+      Match = Struct.new(:action, :params)
+
+      # Signals that a URL matched no route.
+      Mismatch = Class.new
+    end
+  end
+end

data/lib/wayfarer/routing/root_route.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class RootRoute < Route; end
+  end
+end

data/lib/wayfarer/routing/route.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class Route
+      include DSL
+
+      attr_reader :children
+
+      attr_accessor :matcher,
+                    :parent,
+                    :action,
+                    :path_offset
+
+      def initialize(matcher = CustomMatcher.new { children.any? }, path_offset = "/")
+        @matcher = matcher
+        @children = []
+        @path_offset = path_offset
+      end
+
+      def match(url)
+        matcher.match(url)
+      end
+
+      def matches?(url)
+        invoke(url).is_a?(Result::Match)
+      end
+
+      def invoke(url)
+        PathFinder.result(self, url)
+      end
+
+      # Accepts a visitor for in-order traversal.
+      def accept(visitor)
+        return unless visitor.visit(self)
+
+        children.each { |child| child.accept(visitor) }
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/scheme_matcher.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class SchemeMatcher
+      attr_reader :scheme
+
+      def initialize(scheme)
+        @scheme = scheme
+      end
+
+      def match(url)
+        url.scheme == scheme.to_s
+      end
+
+      def params(_)
+        {}
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/suffix_matcher.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class SuffixMatcher
+      attr_reader :suffix
+
+      def initialize(suffix)
+        @suffix = suffix
+      end
+
+      def match(url)
+        url.path.end_with?(suffix)
+      end
+
+      def params(_)
+        {}
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/target_route.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class TargetRoute < Route; end
+  end
+end

data/lib/wayfarer/routing/url_matcher.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class URLMatcher
+      attr_reader :url
+
+      def initialize(url)
+        @url = url
+      end
+
+      def match(url)
+        url == URI(@url)
+      end
+
+      def params(_)
+        {}
+      end
+    end
+  end
+end

data/lib/wayfarer/serializer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class Serializer < ActiveJob::Serializers::ObjectSerializer
+    def serialize?(obj)
+      obj.is_a?(Wayfarer::Task)
+    end
+
+    def serialize(task)
+      super(url: task.url, batch: task.batch)
+    end
+
+    def deserialize(hash)
+      Wayfarer::Task.new(*hash.values_at(:url, :batch))
+    end
+  end
+end

data/lib/wayfarer/stringify.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Stringify
+    def self.included(base)
+      base.include(InstanceMethods)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def stringify(*variables)
+        stringified_attributes.concat(variables)
+      end
+
+      def stringified_attributes
+        @stringified_attributes ||= []
+      end
+    end
+
+    module InstanceMethods
+      def to_s
+        "#<#{class_name} #{attributes}>"
+      end
+
+      alias inspect to_s
+
+      def class_name
+        self.class.name.split("::").last
+      end
+
+      def attributes
+        self.class
+            .stringified_attributes
+            .map { |attr| [attr, public_send(attr)] }
+            .to_h
+            .map { |k, v| [k, "=", v.inspect].join }
+            .join(", ")
+      end
+    end
+  end
+end

data/lib/wayfarer/task.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class Task
+    include Stringify
+
+    attr_reader :url,
+                :batch,
+                :metadata
+
+    attr_accessor :job
+
+    stringify :url,
+              :batch
+
+    def initialize(url, batch)
+      @url = url
+      @batch = batch
+      @metadata = OpenStruct.new
+    end
+
+    def ==(other)
+      [url, batch] == [other.url, other.batch]
+    end
+
+    def barrier
+      @barrier ||= Wayfarer::Redis::Barrier.new(batch)
+    end
+
+    def counter
+      @counter ||= Wayfarer::Redis::Counter.new(batch)
+    end
+  end
+end

data/lib/wayfarer.rb

@@ -1,68 +1,57 @@
 # frozen_string_literal: true
 
-# rubocop:disable Style/Documentation
-require "logger"
+require "cgi"
+require "forwardable"
+require "net/http"
+require "securerandom"
 require "uri"
 
-# Plumbing
-require_relative "wayfarer/configuration"
+require "active_job"
+require "capybara"
+require "capybara/cuprite"
+require "connection_pool"
+require "docile"
+require "ferrum"
+require "metainspector"
+require "mime/types"
+require "mime-types"
+require "mock_redis"
+require "mustermann"
+require "net/http/persistent"
+require "nokogiri"
+require "normalize_url"
+require "selenium-webdriver"
+require "redis"
+require "thor"
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect("cli" => "CLI",
+                         "dsl" => "DSL",
+                         "url_matcher" => "URLMatcher",
+                         "net_http" => "NetHTTP",
+                         "xml" => "XML",
+                         "json" => "JSON",
+                         "gc" => "GC")
+loader.setup
 
-# Routing
-require_relative "wayfarer/routing/rule"
-require_relative "wayfarer/routing/uri_rule"
-require_relative "wayfarer/routing/host_rule"
-require_relative "wayfarer/routing/path_rule"
-require_relative "wayfarer/routing/query_rule"
-require_relative "wayfarer/routing/protocol_rule"
-require_relative "wayfarer/routing/filetypes_rule"
-require_relative "wayfarer/routing/custom_rule"
-require_relative "wayfarer/routing/router"
-
-# Networking
-require_relative "wayfarer/http_adapters/net_http_adapter"
-require_relative "wayfarer/http_adapters/selenium_adapter"
-require_relative "wayfarer/http_adapters/adapter_pool"
-
-# Parsers
-require_relative "wayfarer/parsers/xml_parser"
-require_relative "wayfarer/parsers/json_parser"
+module Wayfarer
+  module VERSION
+    MAJOR = 0
+    MINOR = 4
+    TINY = 0
+    STRING = [MAJOR, MINOR, TINY].join(".")
+  end
 
-# Frontiers
-require_relative "wayfarer/frontiers/frontier"
-require_relative "wayfarer/frontiers/memory_frontier"
-require_relative "wayfarer/frontiers/redis_frontier"
-require_relative "wayfarer/frontiers/normalize_uris"
+  mattr_accessor :config, default: Wayfarer::Config.from_environment
 
-unless RUBY_PLATFORM == "java"
-  require_relative "wayfarer/frontiers/memory_trie_frontier"
-  require_relative "wayfarer/frontiers/memory_bloomfilter"
-  require_relative "wayfarer/frontiers/redis_bloomfilter"
+  mattr_accessor :core_middleware, default: [Wayfarer::Middleware::Stage.new,
+                                             Wayfarer::Middleware::Dedup.new,
+                                             Wayfarer::Middleware::Normalize.new,
+                                             Wayfarer::Middleware::Router.new,
+                                             Wayfarer::Middleware::Fetch.new]
 end
 
-# Processing
-require_relative "wayfarer/crawl"
-require_relative "wayfarer/crawl_observer"
-require_relative "wayfarer/locals"
-require_relative "wayfarer/job"
-require_relative "wayfarer/finders"
-require_relative "wayfarer/page"
-require_relative "wayfarer/dispatcher"
-require_relative "wayfarer/processor"
-
-module Wayfarer
-  VERSION = "0.0.3"
+loader.eager_load
 
-  def self.logger
-    return @logger if @logger
-
-    @logger = Logger.new(STDOUT)
-    @logger.level = Logger::WARN
-    @logger
-  end
-
-  def self.config
-    @config ||= Configuration.new
-    yield(@config) if block_given?
-    @config
-  end
-end
+ActiveJob::Serializers.serializers << Wayfarer::Serializer

data/mkdocs.yml

@@ -0,0 +1,47 @@
+repo_url: https://github.com/bauerd/wayfarer
+edit_uri: edit/develop/docs/
+repo_name: Source Code
+site_name: Wayfarer
+markdown_extensions:
+  - admonition
+  - meta
+  - def_list
+  - pymdownx.details
+  - pymdownx.highlight
+  - pymdownx.superfences
+  - pymdownx.critic
+  - pymdownx.caret
+  - pymdownx.mark
+  - pymdownx.tilde
+  - pymdownx.tabbed
+  - pymdownx.tasklist:
+      custom_checkbox: true
+
+theme:
+  name: material
+  features:
+    - navigation.tabs
+    - navigation.sections
+    - navigation.expand
+  font:
+    text: IBM Plex Sans
+    code: IBM Plex Sans Mono
+  palette:
+    - scheme: default
+      primary: black
+      accent: light blue
+      media: "(prefers-color-scheme: light)"
+      toggle:
+        icon: material/lightbulb-outline
+        name: Switch to dark mode
+    - scheme: slate
+      primary: blue grey
+      accent: cyan
+      media: "(prefers-color-scheme: dark)"
+      toggle:
+        icon: material/lightbulb
+        name: Switch to light mode
+  icon:
+    repo: fontawesome/brands/git
+
+

data/requirements.txt

@@ -0,0 +1 @@
+mkdocs-material == 7.3.0