wayfarer 0.4.6 → 0.4.8

data/spec/base_spec.rb

@@ -1,224 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Base, redis: true do
-  include Wayfarer::Redis::Connection
-
-  let(:url) { "https://example.com" }
-  let(:batch) { "batch" }
-  let(:task) { build(:task, batch: batch, url: url) }
-  let(:klass) { Class.new(Wayfarer::Base) }
-
-  before { stub_const("DummyJob", klass) }
-
-  describe "::crawl" do
-    it "enqueues a task" do
-      expect(DummyJob).to receive(:perform_later).with(task)
-      DummyJob.crawl(url, batch: batch)
-    end
-
-    it "returns a task" do
-      expect(DummyJob.crawl(url)).to be_a(Wayfarer::Task)
-    end
-  end
-
-  describe "Callbacks" do
-    let(:counter) { task.counter }
-
-    describe "after enqueue" do
-      it "increments the counter" do
-        expect {
-          DummyJob.crawl(url, batch: batch)
-        }.to change { counter.value }.by(1)
-      end
-    end
-
-    describe "after perform" do
-      it "decrements the counter" do
-        DummyJob.crawl(url, batch: batch)
-        task.counter.increment
-        expect { perform_enqueued_jobs }.to change { task.counter.value }.by(-1)
-      end
-
-      context "when counter reaches 0" do
-        it "resets the barrier" do
-          DummyJob.crawl(url, batch: batch)
-          perform_enqueued_jobs
-          redis do |conn|
-            expect(conn.exists?(task.barrier.redis_key)).to be(false)
-          end
-        end
-
-        it "resets the counter" do
-          DummyJob.crawl(url, batch: batch)
-          perform_enqueued_jobs
-          redis do |conn|
-            expect(conn.exists?(task.counter.redis_key)).to be(false)
-          end
-        end
-
-        it "runs after batch callbacks" do
-          expect { |spy|
-            klass.after_batch(&spy)
-            DummyJob.crawl(url, batch: batch)
-            perform_enqueued_jobs
-          }.to yield_control
-        end
-      end
-    end
-  end
-
-  describe "Unhandled exceptions" do
-    let(:klass) { Class.new(Wayfarer::Base) }
-
-    before do
-      allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
-    end
-
-    it "does not retry the job" do
-      DummyJob.crawl(url, batch: batch)
-
-      expect {
-        begin
-          perform_enqueued_jobs
-        rescue StandardError
-          nil
-        end
-      }.to change { enqueued_jobs.size }.by(-1)
-    end
-
-    it "decrements the counter" do
-      3.times { task.counter.increment }
-
-      DummyJob.crawl(url, batch: batch)
-      begin
-        perform_enqueued_jobs
-      rescue StandardError
-        nil
-      end
-
-      expect(task.counter.value).to be(3)
-    end
-  end
-
-  describe "Retries" do
-    let(:klass) do
-      Class.new(Wayfarer::Base) do
-        retry_on RuntimeError, attempts: 3 do |job, error|
-          Spy.call(job, error)
-        end
-      end
-    end
-
-    before do
-      allow_any_instance_of(DummyJob).to receive(:perform) do |job|
-        task = job.arguments.first
-        task.metadata.job = job
-
-        raise RuntimeError
-      end
-
-      stub_const("Spy", spy)
-    end
-
-    it "retries the job" do
-      expect(Spy).to receive(:call).exactly(:once)
-                                   .with(kind_of(DummyJob),
-                                         kind_of(RuntimeError))
-
-      DummyJob.crawl(url, batch: batch)
-
-      expect {
-        perform_enqueued_jobs
-      }.to change { enqueued_jobs.last["executions"] }.by(1)
-
-      expect {
-        perform_enqueued_jobs
-      }.to change { enqueued_jobs.last["executions"] }.by(1)
-
-      expect {
-        perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(-1)
-    end
-
-    it "marks the URL seen" do
-      task.counter.increment # otherwise barrier gets reset
-      DummyJob.crawl(url, batch: batch)
-      3.times { perform_enqueued_jobs }
-      expect(task.barrier.seen?(task.url)).to be(true)
-    end
-
-    it "decrements the counter" do
-      3.times { task.counter.increment }
-
-      DummyJob.crawl(url, batch: batch)
-      3.times { perform_enqueued_jobs }
-
-      expect(task.counter.value).to be(3)
-    end
-
-    it "runs after batch callbacks" do
-      expect { |spy|
-        klass.after_batch(&spy)
-        DummyJob.crawl(url, batch: batch)
-        3.times { perform_enqueued_jobs }
-      }.to yield_control
-    end
-  end
-
-  describe "Discarding" do
-    let(:klass) do
-      Class.new(Wayfarer::Base) do
-        discard_on RuntimeError do |job, error|
-          Spy.call(job, error)
-        end
-      end
-    end
-
-    before do
-      allow_any_instance_of(DummyJob).to receive(:perform) do |job|
-        task = job.arguments.first
-        task.metadata.job = job
-
-        raise RuntimeError
-      end
-
-      stub_const("Spy", spy)
-    end
-
-    it "discards the job" do
-      expect(Spy).to receive(:call).exactly(:once)
-                                   .with(kind_of(DummyJob),
-                                         kind_of(RuntimeError))
-
-      DummyJob.crawl(url, batch: batch)
-
-      expect {
-        perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(-1)
-    end
-
-    it "marks the URL seen" do
-      task.counter.increment # otherwise barrier gets reset
-      DummyJob.crawl(url, batch: batch)
-      perform_enqueued_jobs
-      expect(task.barrier.seen?(task.url)).to be(true)
-    end
-
-    it "decrements the counter" do
-      3.times { task.counter.increment }
-      DummyJob.crawl(url, batch: batch)
-      perform_enqueued_jobs
-      expect(task.counter.value).to be(3)
-    end
-
-    it "runs after batch callbacks" do
-      expect { |spy|
-        klass.after_batch(&spy)
-        DummyJob.crawl(url, batch: batch)
-        perform_enqueued_jobs
-      }.to yield_control
-    end
-  end
-end

data/spec/callbacks_spec.rb

@@ -1,102 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Callbacks do
-  let(:url) { "https://alpha.com" }
-  let(:task) { build(:task, batch: "batch", url: url) }
-
-  let(:klass) do
-    Class.new(Wayfarer::Base) do
-      route { host "alpha.com", to: :alpha }
-      route { host "beta.com", to: :beta }
-
-      before_fetch do |job|
-        Spy.before_fetch(self, job)
-      end
-
-      before_fetch(except: %i[alpha beta], only: :alpha) do |job|
-        Spy.before_fetch_except_alpha_beta_only_alpha(self, job)
-      end
-
-      before_action do |job|
-        Spy.before_action(self, job)
-      end
-
-      before_action(only: :alpha) do |job|
-        Spy.before_action_only_alpha(self, job)
-      end
-
-      before_action(only: :beta) do |job|
-        Spy.before_action_only_beta(self, job)
-      end
-
-      before_action(only: %i[alpha gamma]) do |job|
-        Spy.before_action_only_alpha_gamma(self, job)
-      end
-
-      before_action(except: :alpha) do |job|
-        Spy.before_action_except_alpha(self, job)
-      end
-
-      before_action(except: %i[beta gamma]) do |job|
-        Spy.before_action_except_beta_gamma(self, job)
-      end
-
-      before_action :callback_a, only: :alpha
-
-      before_action :callback_b
-
-      before_action :callback_c, except: %i[beta]
-
-      def alpha; end
-
-      def beta; end
-
-    private
-
-      def callback_a
-        Spy.before_action_callback_a_only_alpha(self)
-      end
-
-      def callback_b
-        false
-      end
-
-      def callback_c
-        Spy.before_action_callback_c(self)
-      end
-    end
-  end
-
-  let(:job) { klass.new }
-
-  before do
-    stub_const("DummyJob", klass)
-    stub_const("Spy", spy)
-
-    allow(job).to receive(:arguments).and_return([task])
-  end
-
-  describe "before_fetch" do
-    it "fires" do
-      expect(Spy).to receive(:before_fetch).exactly(:once).with(job, job)
-      expect(Spy).not_to receive(:before_fetch_except_alpha_beta_only_alpha)
-      job.perform(task)
-    end
-  end
-
-  describe "before_action" do
-    it "fires" do
-      expect(Spy).to receive(:before_action).exactly(:once).with(job, job)
-      expect(Spy).to receive(:before_action_only_alpha).exactly(:once).with(job, job)
-      expect(Spy).not_to receive(:before_action_only_beta).with(job, job)
-      expect(Spy).to receive(:before_action_only_alpha_gamma).exactly(:once).with(job, job)
-      expect(Spy).not_to receive(:before_action_except_alpha).with(job, job)
-      expect(Spy).to receive(:before_action_except_beta_gamma).exactly(:once).with(job, job)
-      expect(Spy).to receive(:before_action_callback_a_only_alpha).exactly(:once).with(job)
-      expect(Spy).not_to receive(:before_action_callback_c).with(job)
-      job.perform(task)
-    end
-  end
-end

data/spec/cli/generate_spec.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::CLI::Generate, cli: true do
-  subject(:cli) { Wayfarer::CLI::Runner }
-
-  describe "generate project" do
-    it "outputs" do
-      expected_output = <<-OUT
-      create  foobar
-      create  foobar/Gemfile
-      create  foobar/app/jobs/foobar.rb
-      OUT
-
-      expect {
-        cli.start(%w[generate project foobar])
-      }.to output(expected_output).to_stdout
-    end
-
-    it "creates the project directory" do
-      expect {
-        cli.start(%w[generate project foobar])
-      }.to change { File.directory?("foobar") }.to(true)
-    end
-
-    it "creates the Gemfile" do
-      expect {
-        cli.start(%w[generate project foobar])
-      }.to change { File.file?("foobar/Gemfile") }.to(true)
-    end
-
-    it "creates the job" do
-      expect {
-        cli.start(%w[generate project foobar])
-      }.to change { File.file?("foobar/app/jobs/foobar.rb") }.to(true)
-    end
-  end
-end

data/spec/cli/job_spec.rb

@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::CLI::Job, cli: true, redis: true do
-  include Wayfarer::Redis
-
-  let(:url) { test_app_path("/hello_world") }
-  let(:batch) { "my-batch" }
-  subject(:cli) { Wayfarer::CLI::Runner }
-
-  before do
-    write_file "app/jobs/dummy_job.rb", <<~FILE
-      class DummyJob < Wayfarer::Base
-      end
-    FILE
-  end
-
-  before { Wayfarer::CLI::Base.new.send(:load_environment) }
-
-  describe "job perform" do
-    it "performs the worker" do
-      expect_any_instance_of(DummyJob).to receive(:perform).with(kind_of(Wayfarer::Task)) do |job|
-        task = job.arguments.first
-        task.metadata.job = job
-      end
-
-      cli.start(["job", "perform", "DummyJob", url])
-    end
-
-    it "collects garbage" do
-      expect_any_instance_of(Wayfarer::GC).to receive(:run).exactly(:once)
-      cli.start(["job", "perform", "DummyJob", url])
-    end
-
-    context "using MockRedis" do
-      it "performs the worker using MockRedis" do
-        cli.start(["job", "perform", "--mock-redis", "DummyJob", url])
-        expect(Wayfarer.config.redis.factory.call(nil)).to be_a(MockRedis)
-      end
-    end
-  end
-
-  describe "job enqueue" do
-    it "enqueues the job" do
-      expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
-      cli.start(["job", "enqueue", "DummyJob", url])
-    end
-
-    context "with batch provided" do
-      it "enqueues the job" do
-        expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: batch)
-        cli.start(["job", "enqueue", "--batch", batch, "DummyJob", url])
-      end
-    end
-  end
-
-  describe "job execute" do
-    it "executes the job" do
-      expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
-      cli.start(["job", "execute", "DummyJob", url])
-    end
-
-    context "with batch provided" do
-      it "enqueues the job" do
-        expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: batch)
-        cli.start(["job", "execute", "--batch", batch, "DummyJob", url])
-      end
-    end
-
-    context "using MockRedis" do
-      it "performs the worker using MockRedis" do
-        cli.start(["job", "execute", "--mock-redis", "DummyJob", url])
-        expect(Wayfarer.config.redis.factory.call(nil)).to be_a(MockRedis)
-      end
-    end
-  end
-end

data/spec/config/capybara_spec.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Config::Capybara do
-  let(:env) { {} }
-  subject(:capybara) { Wayfarer::Config::Capybara.new(env) }
-
-  describe "#driver" do
-    context "with env var set" do
-      before { env["WAYFARER_CAPYBARA_DRIVER"] = "cuprite" }
-
-      it "parses the env var" do
-        expect(capybara.driver).to be(:cuprite)
-      end
-    end
-  end
-end

data/spec/config/ferrum_spec.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Config::Ferrum do
-  let(:env) { {} }
-  subject(:ferrum) { Wayfarer::Config::Ferrum.new(env) }
-
-  describe "#options" do
-    context "by default" do
-      it "is {}" do
-        expect(ferrum.options).to eq({})
-      end
-    end
-
-    context "with env var set" do
-      before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
-
-      it "parses the env var" do
-        expect(ferrum.options).to eq(url: "http://chrome:3000", headless: false)
-      end
-    end
-  end
-end

data/spec/config/networking_spec.rb

@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Config::Networking do
-  let(:env) { {} }
-  subject(:network) { Wayfarer::Config::Networking.new(env) }
-
-  describe "#agent" do
-    context "by default" do
-      it "is :http" do
-        expect(network.agent).to be(:http)
-      end
-    end
-
-    context "with env var set" do
-      before { env["WAYFARER_NETWORK_AGENT"] = "ferrum" }
-
-      it "parses the env var" do
-        expect(network.agent).to be(:ferrum)
-      end
-    end
-  end
-
-  describe "#pool_size" do
-    context "by default" do
-      it "is 1" do
-        expect(network.pool_size).to be(1)
-      end
-    end
-
-    context "with env var set" do
-      before { env["WAYFARER_NETWORK_POOL_SIZE"] = "42" }
-
-      it "parses the env var" do
-        expect(network.pool_size).to be(42)
-      end
-    end
-  end
-
-  describe "#pool_timeout" do
-    context "by default" do
-      it "is 10" do
-        expect(network.pool_timeout).to be(10)
-      end
-    end
-
-    context "with env var set" do
-      before { env["WAYFARER_NETWORK_POOL_SIZE"] = "1337" }
-
-      it "parses the env var" do
-        expect(network.pool_size).to be(1337)
-      end
-    end
-  end
-
-  describe "#http_headers" do
-    context "by default" do
-      it "is {}" do
-        expect(network.http_headers).to eq({})
-      end
-    end
-
-    context "with env var set" do
-      before { env["WAYFARER_NETWORK_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
-
-      it "parses the env var" do
-        expect(network.http_headers).to eq("user-agent": "foo",
-                                           authorization: "bar")
-      end
-    end
-  end
-end

data/spec/config/redis_spec.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Config::Redis do
-  let(:env) { {} }
-  subject(:redis) { Wayfarer::Config::Redis.new(env) }
-
-  describe "#url" do
-    context "by default" do
-      it "is redis://localhost:6379" do
-        expect(redis.url).to eq("redis://localhost:6379")
-      end
-    end
-
-    context "with env var set" do
-      before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
-
-      it "parses the env var" do
-        expect(redis.url).to eq("redis://redis:6379")
-      end
-    end
-  end
-
-  describe "#factory" do
-    context "by default" do
-      it "instantiates Redis" do
-        expect(redis.factory.call(redis)).to be_a(::Redis)
-      end
-    end
-  end
-end

data/spec/config/root_spec.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Config::Root do
-  subject(:config) { Wayfarer::Config::Root.new }
-
-  describe "#ferrum" do
-    it "returns a Ferrum config" do
-      expect(config.ferrum).to be_a(Wayfarer::Config::Ferrum)
-    end
-  end
-
-  describe "#network" do
-    it "returns a network config" do
-      expect(config.network).to be_a(Wayfarer::Config::Networking)
-    end
-  end
-
-  describe "#redis" do
-    it "returns a Redis config" do
-      expect(config.redis).to be_a(Wayfarer::Config::Redis)
-    end
-  end
-
-  describe "#selenium" do
-    it "returns a Selenium config" do
-      expect(config.selenium).to be_a(Wayfarer::Config::Selenium)
-    end
-  end
-end

data/spec/config/selenium_spec.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Config::Selenium do
-  let(:env) { {} }
-  subject(:selenium) { Wayfarer::Config::Selenium.new(env) }
-
-  describe "#driver" do
-    context "by default" do
-      it "is :chrome" do
-        expect(selenium.driver).to be(:chrome)
-      end
-    end
-
-    context "with env var set" do
-      before { env["WAYFARER_SELENIUM_DRIVER"] = "firefox" }
-
-      it "parses the env var" do
-        expect(selenium.driver).to be(:firefox)
-      end
-    end
-  end
-
-  describe "#options" do
-    context "by default" do
-      it "is {}" do
-        expect(selenium.options).to eq({})
-      end
-    end
-
-    context "with env var set" do
-      before { env["WAYFARER_SELENIUM_OPTIONS"] = "url:http://firefox" }
-
-      it "parses the env var" do
-        expect(selenium.options).to eq(url: "http://firefox")
-      end
-    end
-  end
-
-  describe "#client_timeout" do
-    context "by default" do
-      it "is 60" do
-        expect(selenium.client_timeout).to be(60)
-      end
-    end
-
-    context "with env var set" do
-      before { env["WAYFARER_SELENIUM_CLIENT_TIMEOUT"] = "10" }
-
-      it "parses the env var" do
-        expect(selenium.client_timeout).to be(10)
-      end
-    end
-  end
-end