wayfarer 0.0.3 → 0.4.3

data/lib/wayfarer/routing/router.rb

@@ -1,70 +1,26 @@
 # frozen_string_literal: true
 
-require "forwardable"
-
 module Wayfarer
   module Routing
-    # A {Router} maps URIs onto a {Job}'s instance methods.
     class Router
-      extend Forwardable
+      ArgumentCountError = Class.new(StandardError)
 
-      # @!attribute [r] rule
-      # @return [Rule]
-      attr_reader :rule
+      extend Forwardable
 
-      # @!attribute [r] blacklist
-      # @return [Rule]
-      attr_reader :blacklist
+      attr_reader :root
 
       def initialize
-        @rule = Rule.new
-        @blacklist = Rule.new
-      end
-
-      delegate %i[
-        uri
-        host
-        path
-        query
-      ] => :rule
-
-      # Returns the associated instance method (action) of the first rule that
-      # matches a URI and the collected parameter hash from the rule chain.
-      # @return [[Boolean, Symbol, Hash]] if a matching rule exists.
-      # @return [false] if no matching rule exists or the URI is forbidden.
-      def route(uri)
-        return false if forbids?(uri)
-
-        # TODO: Use structs instead
-        is_matching, params, action = @rule.invoke(uri)
-        return action, params if is_matching && params
-
-        false
-      end
-
-      # Whether a route matches the URI.
-      # TODO: Test
-      def routes?(uri)
-        !!route(uri)
-      end
-
-      # Adds a {Rule} to the blacklist.
-      def forbid(opts = {}, &proc)
-        @blacklist.build_child_rule_chain_from_options(opts)
-        @blacklist.instance_eval(&proc) if block_given?
-        @blacklist
+        @blocks = []
       end
 
-      # Whether the URI is matched by the blacklist rule.
-      # @see #forbid
-      def forbids?(uri)
-        @blacklist.matches?(uri)
+      def draw(&block)
+        @blocks.push(block)
       end
 
-      # Whether the URI is allowed.
-      # @see #forbid
-      def allows?(uri)
-        !forbids?(uri)
+      def invoke(url, arguments)
+        @root = Wayfarer::Routing::RootRoute.new
+        @blocks.each { |block| Docile.dsl_eval(@root, *arguments, &block) }
+        root.invoke(url)
       end
     end
   end

data/lib/wayfarer/routing/target_route.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    class TargetRoute < Route; end
+  end
+end

data/lib/wayfarer/serializer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class Serializer < ActiveJob::Serializers::ObjectSerializer
+    def serialize?(obj)
+      obj.is_a?(Wayfarer::Task)
+    end
+
+    def serialize(task)
+      super(url: task.url, batch: task.batch)
+    end
+
+    def deserialize(hash)
+      Wayfarer::Task.new(*hash.values_at(:url, :batch))
+    end
+  end
+end

data/lib/wayfarer/stringify.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# 346325
+
+module Wayfarer
+  module Stringify
+    def self.included(base)
+      base.include(InstanceMethods)
+      base.extend(ClassMethods)
+      base.instance_eval do
+        cattr_accessor :stringified_attributes do
+          []
+        end
+      end
+    end
+
+    module ClassMethods
+      def stringify(*variables)
+        stringified_attributes.concat(variables)
+      end
+    end
+
+    module InstanceMethods
+      def to_s
+        if self.class.stringified_attributes.any?
+          "#<#{class_name} #{attributes.join(', ')}>"
+        else
+          "#<#{class_name}>"
+        end
+      end
+
+      alias inspect to_s
+
+      def class_name
+        self.class.name
+      end
+
+      def attributes
+        self.class
+            .stringified_attributes
+            .map { |attr| [attr, public_send(attr)] }
+            .to_h
+            .map { |k, v| [k, "=", v.inspect].join }
+      end
+    end
+  end
+end

data/lib/wayfarer/task.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class Task
+    include Stringify
+
+    attr_reader :url,
+                :batch,
+                :metadata
+
+    attr_accessor :job
+
+    stringify :url,
+              :batch
+
+    def initialize(url, batch)
+      @url = url
+      @batch = batch
+      @metadata = OpenStruct.new
+    end
+
+    def ==(other)
+      [url, batch] == [other.url, other.batch]
+    end
+
+    def barrier
+      @barrier ||= Wayfarer::Redis::Barrier.new(batch)
+    end
+
+    def counter
+      @counter ||= Wayfarer::Redis::Counter.new(batch)
+    end
+  end
+end

data/lib/wayfarer.rb

@@ -1,68 +1,59 @@
 # frozen_string_literal: true
 
-# rubocop:disable Style/Documentation
-require "logger"
+require "cgi"
+require "forwardable"
+require "net/http"
+require "securerandom"
 require "uri"
 
-# Plumbing
-require_relative "wayfarer/configuration"
+require "active_job"
+require "capybara"
+require "capybara/cuprite"
+require "connection_pool"
+require "docile"
+require "ferrum"
+require "metainspector"
+require "mime/types"
+require "mime-types"
+require "mock_redis"
+require "mustermann"
+require "net/http/persistent"
+require "nokogiri"
+require "normalize_url"
+require "selenium-webdriver"
+require "redis"
+require "thor"
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect("cli" => "CLI",
+                         "dsl" => "DSL",
+                         "http" => "HTTP",
+                         "url" => "URL",
+                         "xml" => "XML",
+                         "json" => "JSON",
+                         "gc" => "GC")
+loader.setup
 
-# Routing
-require_relative "wayfarer/routing/rule"
-require_relative "wayfarer/routing/uri_rule"
-require_relative "wayfarer/routing/host_rule"
-require_relative "wayfarer/routing/path_rule"
-require_relative "wayfarer/routing/query_rule"
-require_relative "wayfarer/routing/protocol_rule"
-require_relative "wayfarer/routing/filetypes_rule"
-require_relative "wayfarer/routing/custom_rule"
-require_relative "wayfarer/routing/router"
-
-# Networking
-require_relative "wayfarer/http_adapters/net_http_adapter"
-require_relative "wayfarer/http_adapters/selenium_adapter"
-require_relative "wayfarer/http_adapters/adapter_pool"
+module Wayfarer
+  module VERSION
+    MAJOR = 0
+    MINOR = 4
+    TINY = 3
+    STRING = [MAJOR, MINOR, TINY].join(".")
+  end
 
-# Parsers
-require_relative "wayfarer/parsers/xml_parser"
-require_relative "wayfarer/parsers/json_parser"
+  mattr_accessor :config, default: Wayfarer::Config::Root.new
 
-# Frontiers
-require_relative "wayfarer/frontiers/frontier"
-require_relative "wayfarer/frontiers/memory_frontier"
-require_relative "wayfarer/frontiers/redis_frontier"
-require_relative "wayfarer/frontiers/normalize_uris"
+  mattr_accessor :middleware, default: [Wayfarer::Middleware::Stage.new,
+                                        Wayfarer::Middleware::Dedup.new,
+                                        Wayfarer::Middleware::Normalize.new,
+                                        Wayfarer::Middleware::Router.new,
+                                        Wayfarer::Middleware::Fetch.new]
 
-unless RUBY_PLATFORM == "java"
-  require_relative "wayfarer/frontiers/memory_trie_frontier"
-  require_relative "wayfarer/frontiers/memory_bloomfilter"
-  require_relative "wayfarer/frontiers/redis_bloomfilter"
+  UserAgentTimeoutError = Class.new(StandardError)
 end
 
-# Processing
-require_relative "wayfarer/crawl"
-require_relative "wayfarer/crawl_observer"
-require_relative "wayfarer/locals"
-require_relative "wayfarer/job"
-require_relative "wayfarer/finders"
-require_relative "wayfarer/page"
-require_relative "wayfarer/dispatcher"
-require_relative "wayfarer/processor"
-
-module Wayfarer
-  VERSION = "0.0.3"
-
-  def self.logger
-    return @logger if @logger
-
-    @logger = Logger.new(STDOUT)
-    @logger.level = Logger::WARN
-    @logger
-  end
+loader.eager_load
 
-  def self.config
-    @config ||= Configuration.new
-    yield(@config) if block_given?
-    @config
-  end
-end
+ActiveJob::Serializers.serializers << Wayfarer::Serializer

data/mkdocs.yml

@@ -0,0 +1,47 @@
+repo_url: https://github.com/bauerd/wayfarer
+edit_uri: edit/develop/docs/
+repo_name: Source Code
+site_name: Wayfarer
+markdown_extensions:
+  - admonition
+  - meta
+  - def_list
+  - pymdownx.details
+  - pymdownx.highlight
+  - pymdownx.superfences
+  - pymdownx.critic
+  - pymdownx.caret
+  - pymdownx.mark
+  - pymdownx.tilde
+  - pymdownx.tabbed
+  - pymdownx.tasklist:
+      custom_checkbox: true
+
+theme:
+  name: material
+  features:
+    - navigation.tabs
+    - navigation.sections
+    - navigation.expand
+  font:
+    text: IBM Plex Sans
+    code: IBM Plex Sans Mono
+  palette:
+    - scheme: default
+      primary: black
+      accent: light blue
+      media: "(prefers-color-scheme: light)"
+      toggle:
+        icon: material/lightbulb-outline
+        name: Switch to dark mode
+    - scheme: slate
+      primary: blue grey
+      accent: cyan
+      media: "(prefers-color-scheme: dark)"
+      toggle:
+        icon: material/lightbulb
+        name: Switch to light mode
+  icon:
+    repo: fontawesome/brands/git
+
+

data/requirements.txt

@@ -0,0 +1 @@
+mkdocs-material == 7.3.0

data/spec/base_spec.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Base, redis: true do
+  include Wayfarer::Redis::Connection
+
+  let(:url) { "https://example.com" }
+  let(:batch) { "batch" }
+  let(:task) { build(:task, batch: batch, url: url) }
+  let(:klass) { Class.new(Wayfarer::Base) }
+
+  before { stub_const("DummyJob", klass) }
+
+  describe "::crawl_later" do
+    it "enqueues a task" do
+      expect(DummyJob).to receive(:perform_later).with(task)
+      DummyJob.crawl_later(url, batch: batch)
+    end
+  end
+
+  describe "#perform" do
+    it "assigns itself to the task" do
+      job = DummyJob.new
+      job.perform(task)
+      expect(task.job).to be(job)
+    end
+
+    it "calls the middleware chain" do
+      job = DummyJob.new
+      allow(job).to receive(:chain).and_return(spy)
+      expect(job.chain).to receive(:call).with(task)
+      job.perform(task)
+    end
+  end
+
+  describe "Callbacks" do
+    let(:counter) { task.counter }
+
+    describe "after enqueue" do
+      it "increments the counter" do
+        expect {
+          DummyJob.crawl_later(url, batch: batch)
+        }.to change { counter.value }.by(1)
+      end
+    end
+
+    describe "after perform" do
+      it "decrements the counter" do
+        DummyJob.crawl_later(url, batch: batch)
+        task.counter.increment
+        expect { perform_enqueued_jobs }.to change { task.counter.value }.by(-1)
+      end
+
+      context "when counter reaches 0" do
+        it "resets the barrier" do
+          DummyJob.crawl_later(url, batch: batch)
+          perform_enqueued_jobs
+          redis do |conn|
+            expect(conn.exists?(task.barrier.redis_key)).to be(false)
+          end
+        end
+
+        it "resets the counter" do
+          DummyJob.crawl_later(url, batch: batch)
+          perform_enqueued_jobs
+          redis do |conn|
+            expect(conn.exists?(task.counter.redis_key)).to be(false)
+          end
+        end
+
+        it "runs after batch callbacks" do
+          expect { |spy|
+            klass.after_batch(&spy)
+            DummyJob.crawl_later(url, batch: batch)
+            perform_enqueued_jobs
+          }.to yield_control
+        end
+      end
+    end
+  end
+
+  describe "Unhandled exceptions" do
+    let(:klass) { Class.new(Wayfarer::Base) }
+
+    before do
+      allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
+    end
+
+    it "does not retry the job" do
+      DummyJob.crawl_later(url, batch: batch)
+
+      expect {
+        begin
+          perform_enqueued_jobs
+        rescue StandardError
+          nil
+        end
+      }.to change { enqueued_jobs.size }.by(-1)
+    end
+
+    it "decrements the counter" do
+      3.times { task.counter.increment }
+
+      DummyJob.crawl_later(url, batch: batch)
+      begin
+        perform_enqueued_jobs
+      rescue StandardError
+        nil
+      end
+
+      expect(task.counter.value).to be(3)
+    end
+
+    # it "runs after batch callbacks" do
+    #   expect do |spy|
+    #     klass.after_batch(&spy)
+    #     DummyJob.crawl_later(url, batch: batch)
+    #     3.times { perform_enqueued_jobs }
+    #   end.to yield_control
+    # end
+  end
+
+  describe "Retries" do
+    let(:klass) do
+      Class.new(Wayfarer::Base) do
+        retry_on RuntimeError, attempts: 3 do |job, error|
+          Spy.call(job, error)
+        end
+      end
+    end
+
+    before do
+      allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
+      stub_const("Spy", spy)
+    end
+
+    it "retries the job" do
+      expect(Spy).to receive(:call).exactly(:once)
+                                   .with(kind_of(DummyJob),
+                                         kind_of(RuntimeError))
+
+      DummyJob.crawl_later(url, batch: batch)
+
+      expect {
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.last["executions"] }.by(1)
+
+      expect {
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.last["executions"] }.by(1)
+
+      expect {
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.size }.by(-1)
+    end
+
+    it "marks the URL seen" do
+      task.counter.increment # otherwise barrier gets reset
+      DummyJob.crawl_later(url, batch: batch)
+      3.times { perform_enqueued_jobs }
+      expect(task.barrier.seen?(task.url)).to be(true)
+    end
+
+    it "decrements the counter" do
+      3.times { task.counter.increment }
+
+      DummyJob.crawl_later(url, batch: batch)
+      3.times { perform_enqueued_jobs }
+
+      expect(task.counter.value).to be(3)
+    end
+
+    it "runs after batch callbacks" do
+      expect { |spy|
+        klass.after_batch(&spy)
+        DummyJob.crawl_later(url, batch: batch)
+        3.times { perform_enqueued_jobs }
+      }.to yield_control
+    end
+  end
+
+  describe "Discarding" do
+    let(:klass) do
+      Class.new(Wayfarer::Base) do
+        discard_on RuntimeError do |job, error|
+          Spy.call(job, error)
+        end
+      end
+    end
+
+    before do
+      allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
+      stub_const("Spy", spy)
+    end
+
+    it "discards the job" do
+      expect(Spy).to receive(:call).exactly(:once)
+                                   .with(kind_of(DummyJob),
+                                         kind_of(RuntimeError))
+
+      DummyJob.crawl_later(url, batch: batch)
+
+      expect {
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.size }.by(-1)
+    end
+
+    it "marks the URL seen" do
+      task.counter.increment # otherwise barrier gets reset
+      DummyJob.crawl_later(url, batch: batch)
+      perform_enqueued_jobs
+      expect(task.barrier.seen?(task.url)).to be(true)
+    end
+
+    it "decrements the counter" do
+      3.times { task.counter.increment }
+
+      DummyJob.crawl_later(url, batch: batch)
+      perform_enqueued_jobs
+
+      expect(task.counter.value).to be(3)
+    end
+
+    it "runs after batch callbacks" do
+      expect { |spy|
+        klass.after_batch(&spy)
+        DummyJob.crawl_later(url, batch: batch)
+        perform_enqueued_jobs
+      }.to yield_control
+    end
+  end
+end

data/spec/callbacks_spec.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Callbacks do
+  let(:url) { "https://alpha.com" }
+  let(:task) { build(:task, batch: "batch", url: url) }
+
+  let(:klass) do
+    Class.new(Wayfarer::Base) do
+      route { host "alpha.com", to: :alpha }
+      route { host "beta.com", to: :beta }
+
+      before_fetch do |job|
+        Spy.before_fetch(self, job)
+      end
+
+      before_fetch(except: %i[alpha beta], only: :alpha) do |job|
+        Spy.before_fetch_except_alpha_beta_only_alpha(self, job)
+      end
+
+      before_action do |job|
+        Spy.before_action(self, job)
+      end
+
+      before_action(only: :alpha) do |job|
+        Spy.before_action_only_alpha(self, job)
+      end
+
+      before_action(only: :beta) do |job|
+        Spy.before_action_only_beta(self, job)
+      end
+
+      before_action(only: %i[alpha gamma]) do |job|
+        Spy.before_action_only_alpha_gamma(self, job)
+      end
+
+      before_action(except: :alpha) do |job|
+        Spy.before_action_except_alpha(self, job)
+      end
+
+      before_action(except: %i[beta gamma]) do |job|
+        Spy.before_action_except_beta_gamma(self, job)
+      end
+
+      before_action :callback_a, only: :alpha
+
+      before_action :callback_b
+
+      before_action :callback_c, except: %i[beta]
+
+      def alpha; end
+
+      def beta; end
+
+    private
+
+      def callback_a
+        Spy.before_action_callback_a_only_alpha(self)
+      end
+
+      def callback_b
+        false
+      end
+
+      def callback_c
+        Spy.before_action_callback_c(self)
+      end
+    end
+  end
+
+  let(:job) { klass.new }
+
+  before do
+    stub_const("DummyJob", klass)
+    stub_const("Spy", spy)
+
+    allow(job).to receive(:arguments).and_return([task])
+  end
+
+  describe "before_fetch" do
+    it "fires" do
+      expect(Spy).to receive(:before_fetch).exactly(:once).with(job, job)
+      expect(Spy).not_to receive(:before_fetch_except_alpha_beta_only_alpha)
+      job.perform(task)
+    end
+  end
+
+  describe "before_action" do
+    it "fires" do
+      expect(Spy).to receive(:before_action).exactly(:once).with(job, job)
+      expect(Spy).to receive(:before_action_only_alpha).exactly(:once).with(job, job)
+      expect(Spy).not_to receive(:before_action_only_beta).with(job, job)
+      expect(Spy).to receive(:before_action_only_alpha_gamma).exactly(:once).with(job, job)
+      expect(Spy).not_to receive(:before_action_except_alpha).with(job, job)
+      expect(Spy).to receive(:before_action_except_beta_gamma).exactly(:once).with(job, job)
+      expect(Spy).to receive(:before_action_callback_a_only_alpha).exactly(:once).with(job)
+      expect(Spy).not_to receive(:before_action_callback_c).with(job)
+      job.perform(task)
+    end
+  end
+end