wayfarer 0.4.7 → 0.4.8

data/.rbenv-gemsets

@@ -1 +0,0 @@
-wayfarer

data/.ruby-version

@@ -1 +0,0 @@
-2.7.4

data/RELEASING.md

@@ -1,17 +0,0 @@
-# Release Procedure
-
-1. Ensure `Wayfarer::VERSION` was bumped appropriately.
-2. Ensure the version in wayfarer.gemspec matches.
-3. Open a release Pull Request develop -> master branch
-4. Merge the Pull Request
-5. Publish RubyGem and git tag as follows:
-
-```
-git checkout master
-git pull origin master --rebase
-bundle exec rake build
-gem push build/wayfarer-*.gem
-bundle exec rake clean
-git tag <VERSION>
-git push origin <VERSION>
-```

data/docs/cookbook/user_agent.md

@@ -1,7 +0,0 @@
-# User Agent
-
-See: [Guides: Networking: HTTP request headers](/guides/networking#http-request-headers)
-
-```ruby
-Wayfarer.config[:network][:http_headers] = { "User-Agent" => "MyCrawler ..." }
-```

data/docs/design.md

@@ -1,36 +0,0 @@
-# Design decisions
-
-## Navigate the web along URL patterns
-
-URLs are less prone to change than served markup.
-One reason for this is that changes to a URL's path can have a negative effect
-on its page ranking in search engines. Many websites also implement common
-architectural URL patterns, for example REST and its variations, that
-lend themselves to pattern matching.
-
-## Follow URLs verbatim
-
-Normalized URLs are useful for deduplication, but URLs should be followed
-as they appear in responses. Navigating to normalized versions of URLs makes
-crawlers stick out from other user agents, for example.
-
-## Tasks are version-less and don't persist metadata
-
-Tasks serialize to their URL and batch. No other data gets written to
-the message queue. Wayfarer aims to minimise job payloads.
-There is also no need for versioning persisted tasks, since there is only one
-version of a task: URL and batch.
-
-## Why depend on Redis
-
-There are two core features that depend on Redis. First, per-batch acylicity is
-achieved by maintaining the set of processed URLs per batch in Redis.
-There's no option to follow links in a cyclic manner. Second, batch completion
-requires updating an integer value in Redis, and batch completion is a very
-useful feature, since most crawls should end eventually, and often you want to
-know when.
-
-## Persistence and document mapping not included
-
-Like Active Job, Wayfarer is not concerned with persistence.
-Model <-> DOM mapping abstractions are also out of scope.

data/docs/guides/jobs/error_handling.md

@@ -1,40 +0,0 @@
-# Error handling
-
-!!! danger "Only ActiveJob error handling is supported"
-
-    Wayfarer exclusively supports ActiveJob's error handling. You cannot use
-    message queue-specific error handling, for example error handling with
-    `sidekiq_options` is unsupported. Otherwise batches get garbage-collected
-    too early as Wayfarer instruments ActiveJob.
-
-Wayfarer relies on ActiveJob's [error handling methods](https://guides.rubyonrails.org/active_job_basics.html#exceptions):
-
-* `retry_on` to retry jobs a number of times on certain errors:
-
-    ```ruby
-    class DummyJob < Wayfarer::Base
-      retry_on MyError, attempts: 3 do |job, error|
-        # This block runs once all 3 attempts have failed
-        # (1 initial attempt + 2 retries)
-      end
-    end
-    ```
-
-* `discard_on` to throw away jobs on certain errors:
-
-    ```ruby
-    class DummyJob < Wayfarer::Base
-      discard_on MyError do |job, error|
-        # This block runs once and buries the job
-      end
-    end
-    ```
-
-## Recreating user agents on certain errors
-
-You can configure a list of exception classes upon which user agents
-get recreated (see [User agent API]()):
-
-```ruby
-Wayfarer.config[:network][:renew_on] = [MyIrrecoverableError]
-```

data/docs/reference/configuration.md

@@ -1,36 +0,0 @@
----
-hide:
-  - toc
----
-
-# Configuration
-
-You can configure Wayfarer by assigning to the `Wayfarer.config` Hash
-which has the following defaults:
-
-```ruby
-{
-  redis: {
-    url: "redis://localhost:6379/0",
-    factory: ->(redis) { ::Redis.new(url: redis[:url]) }
-  },
-  network: {
-    agent: :http,
-    pool_size: 1,
-    pool_timeout: 10,
-    http_headers: {},
-    renew_on: []
-  },
-  capybara: {
-    driver: nil
-  },
-  ferrum: {
-    options: {}
-  },
-  selenium: {
-    driver: :chrome,
-    options: {},
-    client_timeout: 60
-  }
-}
-```

data/spec/batch_completion_spec.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::BatchCompletion, redis: true do
-  let(:task) { build(:task, :redis_pool) }
-  let(:job) { double(arguments: [task]) }
-
-  describe "::call" do
-    let(:name) { "foo" }
-
-    subject { described_class.call(name, nil, nil, nil, { job: job }) }
-
-    context "with Wayfarer job" do
-      before { job.extend(Wayfarer::Base) }
-
-      specify do
-        expect(Wayfarer::BatchCompletion)
-          .to receive(:handle).with(name, job, task, instance_of(Wayfarer::Redis::Counter))
-
-        subject
-      end
-
-      it "does not reassign Redis pool" do
-        expect { subject }.not_to change { task[:redis_pool] }.from(Wayfarer::Redis::Pool.instance)
-      end
-
-      context "without Redis pool" do
-        before do
-          task[:redis_pool] = nil
-        end
-
-        it "assigns Redis pool" do
-          expect { subject }.to change { task[:redis_pool] }.from(nil).to(Wayfarer::Redis::Pool.instance)
-        end
-      end
-    end
-
-    context "with other job" do
-      specify do
-        expect(Wayfarer::BatchCompletion).not_to receive(:handle)
-
-        subject
-      end
-    end
-  end
-
-  describe "::handle" do
-    let(:job) do
-      double(arguments: [task],
-             executions: executions,
-             exception_executions: exception_executions)
-    end
-
-    let(:initial_exception_executions) { {} }
-    let(:exception_executions) { initial_exception_executions }
-    let(:executions) { 0 }
-    let(:counter) { Wayfarer::Redis::Counter.new(task) }
-
-    subject(:handle) { described_class.handle(event, job, task, counter) }
-
-    before { task[:initial_exception_executions] = initial_exception_executions }
-
-    context "enqueue.active_job" do
-      let(:event) { "enqueue.active_job" }
-
-      specify do
-        expect { handle }.to change { counter.value }.by(1)
-      end
-
-      context "with retry" do
-        let(:executions) { 1 }
-
-        specify do
-          expect { handle }.not_to(change { counter.value })
-        end
-      end
-    end
-
-    context "perform.active_job" do
-      let(:event) { "perform.active_job" }
-
-      specify do
-        expect { handle }.to change { counter.value }.by(-1)
-      end
-
-      context "with exception occurred" do
-        let(:exception_executions) { { "[RuntimeError]" => 1 } }
-
-        specify do
-          expect { handle }.not_to(change { counter.value })
-        end
-      end
-    end
-
-    context "retry_stopped.active_job" do
-      let(:event) { "retry_stopped.active_job" }
-
-      specify do
-        expect { handle }.to change { counter.value }.by(-1)
-      end
-    end
-  end
-end

data/spec/cli/job_spec.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::CLI, cli: true, redis: true do
-  include Wayfarer::Redis
-
-  let(:url) { test_app_path("/hello_world") }
-  let(:batch) { "my-batch" }
-  subject(:cli) { Wayfarer::CLI }
-
-  before do
-    write_file "dummy_job.rb", <<~FILE
-      class DummyJob < ActiveJob::Base
-        include Wayfarer::Base
-      end
-    FILE
-  end
-
-  before { Wayfarer::CLI.new.send(:load_environment, "dummy_job.rb") }
-
-  describe "perform" do
-    it "performs the worker" do
-      expect_any_instance_of(DummyJob).to receive(:perform).with(kind_of(Wayfarer::Task)) do |job|
-        task = job.arguments.first
-        task[:job] = job
-      end
-
-      cli.start(["perform", "-r", "dummy_job.rb", "DummyJob", url])
-    end
-
-    context "using MockRedis" do
-      it "performs the worker using MockRedis" do
-        cli.start(["perform", "--mock-redis", "DummyJob", url])
-        expect(Wayfarer.config[:redis][:factory].call(nil)).to be_a(MockRedis)
-      end
-    end
-  end
-
-  describe "enqueue" do
-    it "enqueues the job" do
-      expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
-      cli.start(["enqueue", "DummyJob", url])
-    end
-
-    context "with batch provided" do
-      it "enqueues the job" do
-        expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: batch)
-        cli.start(["enqueue", "--batch", batch, "DummyJob", url])
-      end
-    end
-  end
-
-  describe "execute" do
-    it "executes the job" do
-      expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
-      cli.start(["execute", "DummyJob", url])
-    end
-
-    context "with batch provided" do
-      it "enqueues the job" do
-        expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: batch)
-        cli.start(["execute", "--batch", batch, "DummyJob", url])
-      end
-    end
-
-    context "using MockRedis" do
-      it "performs the worker using MockRedis" do
-        cli.start(["execute", "--mock-redis", "DummyJob", url])
-        expect(Wayfarer.config[:redis][:factory].call(nil)).to be_a(MockRedis)
-      end
-    end
-  end
-end

data/spec/cli/routing_spec.rb

@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::CLI, cli: true do
-  before { write_file("dummy_job.rb", contents) }
-
-  let(:contents) do
-    <<~RUBY
-      class DummyJob < ActiveJob::Base
-        include Wayfarer::Base
-
-        #{routes}
-      end
-    RUBY
-  end
-
-  describe "route" do
-    let(:routes) do
-      <<~RUBY
-        route.host #{test_app_host.inspect}, to: :index
-      RUBY
-    end
-
-    subject(:route) { described_class.start(["route", "-r", "dummy_job.rb", "DummyJob", url]) }
-
-    context "with matching URL" do
-      let(:url) { "http://#{test_app_host}" }
-
-      specify do
-        expect { route }.to output("Match => :index\n").to_stdout
-      end
-    end
-
-    context "with mismatching URL" do
-      let(:url) { "http://example.com" }
-
-      specify do
-        expect { route }.to output("Mismatch\n").to_stdout
-      end
-    end
-  end
-
-  describe "tree" do
-    subject(:tree) { described_class.start(["tree", "-r", "dummy_job.rb", "DummyJob", url]) }
-
-    context "without child routes" do
-      let(:url) { "http://#{test_app_host}" }
-      let(:routes) { "" }
-
-      specify do
-        expect { tree }.to output("Mismatch\n").to_stdout
-      end
-    end
-
-    context "symbol target" do
-      let(:url) { "http://#{test_app_host}" }
-      let(:routes) do
-        <<~RUBY
-          route.host #{test_app_host.inspect}, to: :index
-        RUBY
-      end
-
-      let(:expected_output) do
-        <<~OUTPUT
-          Match(:index)
-          └──Host("#{test_app_host}", match: true)
-             └──Target(match: true)
-        OUTPUT
-      end
-
-      specify do
-        expect { tree }.to output(expected_output).to_stdout
-      end
-    end
-
-    context "symbol target with params" do
-      let(:url) { "http://#{test_app_host}/barqux" }
-      let(:routes) do
-        <<~RUBY
-          route.host #{test_app_host.inspect}, to: :index do
-            path ":foobar"
-          end
-        RUBY
-      end
-
-      let(:expected_output) do
-        <<~OUTPUT
-          Match(:index, params: {:foobar=>"barqux"})
-          └──Host("#{test_app_host}", match: true)
-             └──Target(match: true)
-                └──Path("/:foobar", match: true, params: {:foobar=>"barqux"})
-        OUTPUT
-      end
-
-      specify do
-        expect { tree }.to output(expected_output).to_stdout
-      end
-    end
-  end
-end

data/spec/fixtures/dummy_job.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-class Router < ActiveJob::Base
-  include Wayfarer::Base
-
-  def hello
-    stage page.meta.links.external
-  end
-end

data/spec/gc_spec.rb

@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::GC, "::run", redis: true do
-  let(:task) { build(:task, :redis_pool) }
-  let(:barrier) { instance_double(Wayfarer::Redis::Barrier) }
-
-  subject(:run) { described_class.run(task) }
-
-  it "resets barrier and counter" do
-    expect(barrier).to receive(:reset!)
-    expect(Wayfarer::Redis::Barrier).to receive(:new).with(task).and_return(barrier)
-
-    run
-  end
-end

data/spec/integration/content_type_spec.rb

@@ -1,145 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe "Content-Type allow listing", redis: true do
-  def perform(content_type)
-    DummyJob.new.perform(
-      build(:task, url: test_app_path("response_header/Content-Type/#{content_type}"))
-    )
-  end
-
-  before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-    stub_const("DummyHandler", Class.new.include(Wayfarer::Handler))
-  end
-
-  context "with registered Content-Types" do
-    before do
-      DummyJob.class_eval do
-        route.to :index
-
-        content_type "foo", "bar"
-
-        def index
-          :ok
-        end
-      end
-    end
-
-    specify do
-      expect(perform("foo")).to be(:ok)
-      expect(perform("bar")).to be(:ok)
-      expect(perform("qux")).to be(nil)
-    end
-  end
-
-  context "with registered Content-Type media type" do
-    before do
-      DummyJob.class_eval do
-        route.to :index
-
-        content_type(/text/)
-        content_type "application/vnd.scenario.custom+json"
-
-        def index
-          :ok
-        end
-      end
-    end
-
-    specify do
-      expect(perform("text/xml")).to be(:ok)
-      expect(perform("application/xml")).to be(nil)
-      expect(perform("application/rss+xml")).to be(nil)
-      expect(perform("image/svg+xml")).to be(nil)
-      expect(perform("application/vnd.scenario.custom+json")).to be(:ok)
-    end
-  end
-
-  context "without registered Content-Types" do
-    before do
-      DummyJob.class_eval do
-        route.to :index
-
-        def index
-          :ok
-        end
-      end
-    end
-
-    specify do
-      expect(perform("foo")).to be(:ok)
-    end
-  end
-
-  context "with regular expression Content-Type" do
-    before do
-      DummyJob.class_eval do
-        route.to :index
-
-        content_type(/xml/)
-
-        def index
-          :ok
-        end
-      end
-    end
-
-    specify do
-      expect(perform("text/xml")).to be(:ok)
-      expect(perform("application/xml")).to be(:ok)
-      expect(perform("application/rss+xml")).to be(:ok)
-      expect(perform("image/svg+xml")).to be(:ok)
-      expect(perform("application/vnd.scenario.custom+json")).to be(nil)
-    end
-  end
-
-  context "with registered Content-Types" do
-    before do
-      DummyJob.class_eval do
-        route.to :index
-
-        content_type(/text/, "foo")
-
-        def index
-          :ok
-        end
-      end
-    end
-
-    specify do
-      expect(perform("text/application")).to be(:ok)
-      expect(perform("foo")).to be(:ok)
-      expect(perform("bar")).to be(nil)
-    end
-  end
-
-  context "with handler" do
-    before do
-      DummyJob.class_eval do
-        route.to DummyHandler
-
-        content_type(/text/, "foo")
-      end
-    end
-
-    before do
-      DummyHandler.class_eval do
-        route.to :index
-
-        content_type "foo"
-
-        def index
-          :handler
-        end
-      end
-    end
-
-    specify do
-      expect(perform("text/application")).to be(nil)
-      expect(perform("foo")).to be(:handler)
-      expect(perform("bar")).to be(nil)
-    end
-  end
-end

data/spec/integration/routing_spec.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe "Routing" do
-  let(:task) { build(:task) }
-
-  before do
-    stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
-  end
-
-  describe "custom routing" do
-    before do
-      DummyJob.class_eval do
-      end
-    end
-  end
-end

data/spec/middleware/dedup_spec.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Middleware::Dedup, redis: true do
-  let(:task) { build(:task, :redis_pool) }
-  let(:uri) { Addressable::URI.parse(task.url) }
-  let(:executions) { 1 }
-  let(:job) { double(executions: executions) }
-  subject { described_class.new }
-
-  before do
-    task[:job] = job
-    task[:controller] = job
-    task[:normalized_url] = task.url
-  end
-
-  describe "#call" do
-    it "assigns barrier" do
-      expect { subject.call(task) }.to change { task[:barrier] }.from(nil).to(instance_of(Wayfarer::Redis::Barrier))
-    end
-
-    it "assigns barrier for batch" do
-      subject.call(task)
-
-      expect(task[:barrier].task).to be(task)
-    end
-
-    context "with retry" do
-      let(:executions) { 2 }
-
-      specify do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
-      end
-
-      specify do
-        expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::DEBUG, "Not deduplicating retry")
-
-        subject.call(task)
-      end
-    end
-
-    context "when rerouted" do
-      before { task[:controller] = Object.new }
-
-      specify do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
-      end
-
-      specify do
-        expect(Wayfarer::Logging.logger).to receive(:add).with(Logger::DEBUG, "Not deduplicating rerouted task")
-
-        subject.call(task)
-      end
-    end
-
-    context "with unchecked URL" do
-      specify do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
-      end
-    end
-
-    context "with checked URL" do
-      before { Wayfarer::Redis::Barrier.new(task).check!(task[:normalized_url]) }
-
-      specify do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
-      end
-    end
-  end
-end