wayfarer 0.0.3 → 0.4.0

data/spec/base_spec.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Base, redis: true do
+  include Wayfarer::Redis::Connection
+
+  let(:url) { "https://example.com" }
+  let(:batch) { "batch" }
+  let(:task) { build(:task, batch: batch, url: url) }
+  let(:klass) { Class.new(Wayfarer::Base) }
+
+  before { stub_const("DummyJob", klass) }
+
+  describe "::crawl_later" do
+    it "enqueues a task" do
+      expect(DummyJob).to receive(:perform_later).with(task)
+      DummyJob.crawl_later(url, batch: batch)
+    end
+  end
+
+  describe "#perform" do
+    it "assigns itself to the task" do
+      job = DummyJob.new
+      job.perform(task)
+      expect(task.job).to be(job)
+    end
+
+    it "calls the middleware chain" do
+      job = DummyJob.new
+      allow(job).to receive(:chain).and_return(spy)
+      expect(job.chain).to receive(:call).with(task)
+      job.perform(task)
+    end
+  end
+
+  describe "Callbacks" do
+    let(:counter) { task.counter }
+
+    describe "after enqueue" do
+      it "increments the counter" do
+        expect {
+          DummyJob.crawl_later(url, batch: batch)
+        }.to change { counter.value }.by(1)
+      end
+    end
+
+    describe "after perform" do
+      it "decrements the counter" do
+        DummyJob.crawl_later(url, batch: batch)
+        task.counter.increment
+        expect { perform_enqueued_jobs }.to change { task.counter.value }.by(-1)
+      end
+
+      context "when counter reaches 0" do
+        it "resets the barrier" do
+          DummyJob.crawl_later(url, batch: batch)
+          perform_enqueued_jobs
+          redis do |conn|
+            expect(conn.exists?(task.barrier.redis_key)).to be(false)
+          end
+        end
+
+        it "resets the counter" do
+          DummyJob.crawl_later(url, batch: batch)
+          perform_enqueued_jobs
+          redis do |conn|
+            expect(conn.exists?(task.counter.redis_key)).to be(false)
+          end
+        end
+
+        it "runs after batch callbacks" do
+          expect { |spy|
+            klass.after_batch(&spy)
+            DummyJob.crawl_later(url, batch: batch)
+            perform_enqueued_jobs
+          }.to yield_control
+        end
+      end
+    end
+  end
+
+  describe "Unhandled exceptions" do
+    let(:klass) { Class.new(Wayfarer::Base) }
+
+    before do
+      allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
+    end
+
+    it "does not retry the job" do
+      DummyJob.crawl_later(url, batch: batch)
+
+      expect {
+        begin
+          perform_enqueued_jobs
+        rescue StandardError
+          nil
+        end
+      }.to change { enqueued_jobs.size }.by(-1)
+    end
+
+    it "decrements the counter" do
+      3.times { task.counter.increment }
+
+      DummyJob.crawl_later(url, batch: batch)
+      begin
+        perform_enqueued_jobs
+      rescue StandardError
+        nil
+      end
+
+      expect(task.counter.value).to be(3)
+    end
+
+    # it "runs after batch callbacks" do
+    #   expect do |spy|
+    #     klass.after_batch(&spy)
+    #     DummyJob.crawl_later(url, batch: batch)
+    #     3.times { perform_enqueued_jobs }
+    #   end.to yield_control
+    # end
+  end
+
+  describe "Retries" do
+    let(:klass) do
+      Class.new(Wayfarer::Base) do
+        retry_on RuntimeError, attempts: 3 do |job, error|
+          Spy.call(job, error)
+        end
+      end
+    end
+
+    before do
+      allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
+      stub_const("Spy", spy)
+    end
+
+    it "retries the job" do
+      expect(Spy).to receive(:call).exactly(:once)
+                                   .with(kind_of(DummyJob),
+                                         kind_of(RuntimeError))
+
+      DummyJob.crawl_later(url, batch: batch)
+
+      expect {
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.last["executions"] }.by(1)
+
+      expect {
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.last["executions"] }.by(1)
+
+      expect {
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.size }.by(-1)
+    end
+
+    it "decrements the counter" do
+      3.times { task.counter.increment }
+
+      DummyJob.crawl_later(url, batch: batch)
+      3.times { perform_enqueued_jobs }
+
+      expect(task.counter.value).to be(3)
+    end
+
+    it "runs after batch callbacks" do
+      expect { |spy|
+        klass.after_batch(&spy)
+        DummyJob.crawl_later(url, batch: batch)
+        3.times { perform_enqueued_jobs }
+      }.to yield_control
+    end
+  end
+
+  describe "Discarding" do
+    let(:klass) do
+      Class.new(Wayfarer::Base) do
+        discard_on RuntimeError do |job, error|
+          Spy.call(job, error)
+        end
+      end
+    end
+
+    before do
+      allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
+      stub_const("Spy", spy)
+    end
+
+    it "discards the job" do
+      expect(Spy).to receive(:call).exactly(:once)
+                                   .with(kind_of(DummyJob),
+                                         kind_of(RuntimeError))
+
+      DummyJob.crawl_later(url, batch: batch)
+
+      expect {
+        perform_enqueued_jobs
+      }.to change { enqueued_jobs.size }.by(-1)
+    end
+
+    it "decrements the counter" do
+      3.times { task.counter.increment }
+
+      DummyJob.crawl_later(url, batch: batch)
+      perform_enqueued_jobs
+
+      expect(task.counter.value).to be(3)
+    end
+
+    it "runs after batch callbacks" do
+      expect { |spy|
+        klass.after_batch(&spy)
+        DummyJob.crawl_later(url, batch: batch)
+        perform_enqueued_jobs
+      }.to yield_control
+    end
+  end
+end

data/spec/cli/generate_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::CLI::Generate, cli: true do
+  subject(:cli) { Wayfarer::CLI::Runner }
+
+  describe "generate project" do
+    it "outputs" do
+      expected_output = <<-OUT
+      create  foobar
+      create  foobar/Gemfile
+      create  foobar/app/jobs/foobar.rb
+      OUT
+
+      expect {
+        cli.start(%w[generate project foobar])
+      }.to output(expected_output).to_stdout
+    end
+
+    it "creates the project directory" do
+      expect {
+        cli.start(%w[generate project foobar])
+      }.to change { File.directory?("foobar") }.to(true)
+    end
+
+    it "creates the Gemfile" do
+      expect {
+        cli.start(%w[generate project foobar])
+      }.to change { File.file?("foobar/Gemfile") }.to(true)
+    end
+
+    it "creates the job" do
+      expect {
+        cli.start(%w[generate project foobar])
+      }.to change { File.file?("foobar/app/jobs/foobar.rb") }.to(true)
+    end
+  end
+end

data/spec/cli/job_spec.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::CLI::Job, cli: true, redis: true do
+  include Wayfarer::Redis
+
+  let(:url) { test_app_path("/hello_world") }
+  let(:batch) { "my-batch" }
+  subject(:cli) { Wayfarer::CLI::Runner }
+
+  before do
+    write_file "app/jobs/dummy_job.rb", <<~FILE
+      class DummyJob < Wayfarer::Base
+      end
+    FILE
+  end
+
+  before { Wayfarer::CLI::Base.new.send(:load_environment) }
+
+  describe "job perform" do
+    it "performs the worker" do
+      expect_any_instance_of(DummyJob).to receive(:perform).with(kind_of(Wayfarer::Task))
+      cli.start(["job", "perform", "DummyJob", url])
+    end
+
+    it "collects garbage" do
+      expect_any_instance_of(Wayfarer::GC).to receive(:run).exactly(:once)
+      cli.start(["job", "perform", "DummyJob", url])
+    end
+
+    context "using MockRedis" do
+      it "performs the worker using MockRedis" do
+        cli.start(["job", "perform", "--mock-redis", "DummyJob", url])
+        expect(Wayfarer.config.redis_factory.call).to be_a(MockRedis)
+      end
+    end
+  end
+
+  describe "job enqueue" do
+    it "enqueues the job" do
+      expect(DummyJob).to receive(:crawl_later).with(URI(url), batch: kind_of(String))
+      cli.start(["job", "enqueue", "DummyJob", url])
+    end
+
+    context "with batch provided" do
+      it "enqueues the job" do
+        expect(DummyJob).to receive(:crawl_later).with(URI(url), batch: batch)
+        cli.start(["job", "enqueue", "--batch", batch, "DummyJob", url])
+      end
+    end
+  end
+
+  describe "job execute" do
+    it "executes the job" do
+      expect(DummyJob).to receive(:crawl_later).with(URI(url), batch: kind_of(String))
+      cli.start(["job", "execute", "DummyJob", url])
+    end
+
+    context "with batch provided" do
+      it "enqueues the job" do
+        expect(DummyJob).to receive(:crawl_later).with(URI(url), batch: batch)
+        cli.start(["job", "execute", "--batch", batch, "DummyJob", url])
+      end
+    end
+
+    context "using MockRedis" do
+      it "performs the worker using MockRedis" do
+        cli.start(["job", "execute", "--mock-redis", "DummyJob", url])
+        expect(Wayfarer.config.redis_factory.call).to be_a(MockRedis)
+      end
+    end
+  end
+end

data/spec/cli/version_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::CLI::Runner, cli: true do
+  describe "version" do
+    it "prints the version" do
+      expect {
+        described_class.start(%w[version])
+      }.to output(Wayfarer::VERSION::STRING << "\n").to_stdout
+    end
+  end
+end

data/spec/config_spec.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config do
+  describe "::defaults" do
+    it "returns the default config" do
+      config = Wayfarer::Config.default
+      expect(config.adapter).to be(:net_http)
+      expect(config.adapter_pool_size).to be(3)
+      expect(config.adapter_pool_timeout).to eq(ConnectionPool::DEFAULTS[:timeout])
+      expect(config.ferrum_options).to eq({})
+      expect(config.selenium_argv).to eq(%i[chrome])
+      expect(config.redis_url).to eq("redis://localhost:6379")
+      expect(config.redis_factory).not_to be(nil)
+    end
+  end
+
+  describe "::from_environment" do
+    let(:env) { {} }
+    subject(:config) { Wayfarer::Config.from_environment(env) }
+
+    describe "#adapter" do
+      context "by default" do
+        it "returns :net_http" do
+          expect(config.adapter).to be(:net_http)
+        end
+      end
+
+      context "with env var set" do
+        before { env["WAYFARER_ADAPTER"] = "ferrum" }
+
+        it "symbolizes the env var" do
+          expect(config.adapter).to be(:ferrum)
+        end
+      end
+    end
+
+    describe "#adapter_pool_size" do
+      context "by default" do
+        it "returns the default config value" do
+          expect(config.adapter_pool_size).to be(3)
+        end
+      end
+
+      context "with env var set" do
+        before { env["WAYFARER_POOL_SIZE"] = "1234" }
+
+        it "parses the env var to a number" do
+          expect(config.adapter_pool_size).to be(1234)
+        end
+      end
+    end
+
+    describe "#adapter_pool_timeout" do
+      context "by default" do
+        it "returns the default config value" do
+          expect(config.adapter_pool_timeout).to be(ConnectionPool::DEFAULTS[:timeout])
+        end
+      end
+
+      context "with env var set" do
+        let(:env) { { "WAYFARER_POOL_TIMEOUT" => "1000" } }
+
+        it "parses the env var to a number" do
+          expect(config.adapter_pool_timeout).to be(1000)
+        end
+      end
+    end
+
+    describe "#ferrum_options" do
+      context "by default" do
+        it "is empty" do
+          expect(config.ferrum_options).to eq({})
+        end
+      end
+
+      context "with env var set" do
+        before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
+
+        it "parses the env var to a Hash" do
+          expect(config.ferrum_options).to eq(url: "http://chrome:3000", headless: false)
+        end
+      end
+    end
+
+    describe "#selenium_argv" do
+      context "by default" do
+        it "is [:chrome]" do
+          expect(config.selenium_argv).to eq(%i[chrome])
+        end
+      end
+
+      context "with env var set" do
+        before { env["WAYFARER_SELENIUM_ARGV"] = "firefox" }
+
+        it "parses the env var to an array" do
+          expect(config.selenium_argv).to eq(%w[firefox])
+        end
+      end
+    end
+
+    describe "#redis_url" do
+      context "by default" do
+        it "is redis://localhost:6379" do
+          expect(config.redis_url).to eq("redis://localhost:6379")
+        end
+      end
+
+      context "with env var set" do
+        before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
+
+        it "reads the env var" do
+          expect(config.redis_url).to eq("redis://redis:6379")
+        end
+      end
+    end
+
+    describe "#http_headers" do
+      context "by default" do
+        it "is {}" do
+          expect(config.http_headers).to eq({})
+        end
+      end
+
+      context "with env var set" do
+        before { env["WAYFARER_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
+
+        it "reads the env var" do
+          expect(config.http_headers).to eq("user-agent": "foo",
+                                            authorization: "bar")
+        end
+      end
+    end
+  end
+
+  describe "#redis_factory" do
+    let(:config) { Wayfarer::Config.default }
+
+    it "instantiates Redis clients" do
+      expect(config.redis_factory.call).to be_a(Redis)
+    end
+  end
+end

data/spec/factories/queue/chain.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :chain, class: Wayfarer::Middleware::Chain do
+    middlewares { [] }
+
+    initialize_with do
+      new(middlewares)
+    end
+  end
+end

data/spec/factories/queue/middleware.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :middleware, class: OpenStruct do
+    callee { -> {} }
+
+    initialize_with do
+      new(callee: callee).tap do |middleware|
+        middleware.define_singleton_method(:call) do |task, &block|
+          callee.call(task, &block)
+        end
+      end
+    end
+  end
+end

data/spec/factories/queue/page.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :page, class: Wayfarer::Page do
+    url { "https://example.com" }
+    status_code { 200 }
+    headers { {} }
+    body { "" }
+
+    trait :html do
+      headers { { "Content-Type" => %w[text/html] } }
+      body do
+        <<~HTML
+          <!doctype html>
+          <html lang=en>
+            <head>
+              <meta charset=utf-8>
+              <title>Title</title>
+            </head>
+            <body></body>
+          </html>
+        HTML
+      end
+    end
+
+    trait :xml do
+      headers { { "Content-Type" => %w[application/xml] } }
+      body do
+        <<~XML
+          <menu id="file" value="File">
+            <popup>
+              <menuitem value="New" onclick="CreateNewDoc()" />
+              <menuitem value="Open" onclick="OpenDoc()" />
+              <menuitem value="Close" onclick="CloseDoc()" />
+            </popup>
+          </menu>
+        XML
+      end
+    end
+
+    trait :json do
+      headers { { "Content-Type" => %w[application/json] } }
+      body do
+        <<~JSON
+          {
+            "menu":{
+              "id":"file",
+              "value":"File",
+              "popup":{
+                "menuitem":[
+                  {
+                    "value":"New",
+                    "onclick":"CreateNewDoc()"
+                  },
+                  {
+                    "value":"Open",
+                    "onclick":"OpenDoc()"
+                  },
+                  {
+                    "value":"Close",
+                    "onclick":"CloseDoc()"
+                  }
+                ]
+              }
+            }
+          }
+        JSON
+      end
+    end
+
+    initialize_with do
+      new(url: url,
+          status_code: status_code,
+          body: body,
+          headers: headers)
+    end
+  end
+end

data/spec/factories/queue/task.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :task, class: Wayfarer::Task do
+    url { "https://example.com" }
+    batch { "batch" }
+
+    initialize_with do
+      new(url, batch)
+    end
+  end
+end

data/spec/fixtures/dummy_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class DummyJob < Wayfarer::Base
+  route.to :index
+
+  def hello; end
+end

data/spec/gc_spec.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::GC, redis: true do
+  include Wayfarer::Redis::Connection
+
+  let(:task) { build(:task) }
+  let(:klass) { Struct.new(:arguments) }
+  let(:job) { klass.new([task]) }
+  subject(:gc) { Wayfarer::GC.new(job) }
+
+  before do
+    allow(klass).to receive(:run_after_batch_callbacks)
+    task.barrier.seen?("https://example.com")
+  end
+
+  describe "#run" do
+    context "when counter reaches 0" do
+      before { task.counter.increment }
+
+      it "resets the barrier" do
+        expect {
+          gc.run
+        }.to change { redis { |conn| conn.exists?(task.barrier.redis_key) } }.to(false)
+      end
+
+      it "resets the counter" do
+        expect {
+          gc.run
+        }.to change { redis { |conn| conn.exists?(task.counter.redis_key) } }.to(false)
+      end
+
+      it "runs after batch callbacks" do
+        expect(klass).to receive(:run_after_batch_callbacks).exactly(:once)
+        gc.run
+      end
+    end
+
+    context "when counter does not reach 0" do
+      before { 2.times { task.counter.increment } }
+
+      it "does not reset the barrier" do
+        expect {
+          gc.run
+        }.not_to(change { redis { |conn| conn.exists?(task.barrier.redis_key) } })
+      end
+
+      it "does not reset the counter" do
+        expect {
+          gc.run
+        }.not_to(change { redis { |conn| conn.exists?(task.counter.redis_key) } })
+      end
+
+      it "does not run after batch callbacks" do
+        expect(klass).not_to receive(:run_after_batch_callbacks)
+        gc.run
+      end
+    end
+  end
+end