wayfarer 0.0.3 → 0.4.3

data/spec/cli/generate_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::CLI::Generate, cli: true do
+  subject(:cli) { Wayfarer::CLI::Runner }
+
+  describe "generate project" do
+    it "outputs" do
+      expected_output = <<-OUT
+      create  foobar
+      create  foobar/Gemfile
+      create  foobar/app/jobs/foobar.rb
+      OUT
+
+      expect {
+        cli.start(%w[generate project foobar])
+      }.to output(expected_output).to_stdout
+    end
+
+    it "creates the project directory" do
+      expect {
+        cli.start(%w[generate project foobar])
+      }.to change { File.directory?("foobar") }.to(true)
+    end
+
+    it "creates the Gemfile" do
+      expect {
+        cli.start(%w[generate project foobar])
+      }.to change { File.file?("foobar/Gemfile") }.to(true)
+    end
+
+    it "creates the job" do
+      expect {
+        cli.start(%w[generate project foobar])
+      }.to change { File.file?("foobar/app/jobs/foobar.rb") }.to(true)
+    end
+  end
+end

data/spec/cli/job_spec.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::CLI::Job, cli: true, redis: true do
+  include Wayfarer::Redis
+
+  let(:url) { test_app_path("/hello_world") }
+  let(:batch) { "my-batch" }
+  subject(:cli) { Wayfarer::CLI::Runner }
+
+  before do
+    write_file "app/jobs/dummy_job.rb", <<~FILE
+      class DummyJob < Wayfarer::Base
+      end
+    FILE
+  end
+
+  before { Wayfarer::CLI::Base.new.send(:load_environment) }
+
+  describe "job perform" do
+    it "performs the worker" do
+      expect_any_instance_of(DummyJob).to receive(:perform).with(kind_of(Wayfarer::Task))
+      cli.start(["job", "perform", "DummyJob", url])
+    end
+
+    it "collects garbage" do
+      expect_any_instance_of(Wayfarer::GC).to receive(:run).exactly(:once)
+      cli.start(["job", "perform", "DummyJob", url])
+    end
+
+    context "using MockRedis" do
+      it "performs the worker using MockRedis" do
+        cli.start(["job", "perform", "--mock-redis", "DummyJob", url])
+        expect(Wayfarer.config.redis.factory.call(nil)).to be_a(MockRedis)
+      end
+    end
+  end
+
+  describe "job enqueue" do
+    it "enqueues the job" do
+      expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: kind_of(String))
+      cli.start(["job", "enqueue", "DummyJob", url])
+    end
+
+    context "with batch provided" do
+      it "enqueues the job" do
+        expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: batch)
+        cli.start(["job", "enqueue", "--batch", batch, "DummyJob", url])
+      end
+    end
+  end
+
+  describe "job execute" do
+    it "executes the job" do
+      expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: kind_of(String))
+      cli.start(["job", "execute", "DummyJob", url])
+    end
+
+    context "with batch provided" do
+      it "enqueues the job" do
+        expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: batch)
+        cli.start(["job", "execute", "--batch", batch, "DummyJob", url])
+      end
+    end
+
+    context "using MockRedis" do
+      it "performs the worker using MockRedis" do
+        cli.start(["job", "execute", "--mock-redis", "DummyJob", url])
+        expect(Wayfarer.config.redis.factory.call(nil)).to be_a(MockRedis)
+      end
+    end
+  end
+end

data/spec/cli/version_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::CLI::Runner, cli: true do
+  describe "version" do
+    it "prints the version" do
+      expect {
+        described_class.start(%w[version])
+      }.to output(Wayfarer::VERSION::STRING << "\n").to_stdout
+    end
+  end
+end

data/spec/config/capybara_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Capybara do
+  let(:env) { {} }
+  subject(:capybara) { Wayfarer::Config::Capybara.new(env) }
+
+  describe "#driver" do
+    context "with env var set" do
+      before { env["WAYFARER_CAPYBARA_DRIVER"] = "cuprite" }
+
+      it "parses the env var" do
+        expect(capybara.driver).to be(:cuprite)
+      end
+    end
+  end
+end

data/spec/config/ferrum_spec.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Ferrum do
+  let(:env) { {} }
+  subject(:ferrum) { Wayfarer::Config::Ferrum.new(env) }
+
+  describe "#options" do
+    context "by default" do
+      it "is {}" do
+        expect(ferrum.options).to eq({})
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
+
+      it "parses the env var" do
+        expect(ferrum.options).to eq(url: "http://chrome:3000", headless: false)
+      end
+    end
+  end
+end

data/spec/config/networking_spec.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Networking do
+  let(:env) { {} }
+  subject(:network) { Wayfarer::Config::Networking.new(env) }
+
+  describe "#agent" do
+    context "by default" do
+      it "is :http" do
+        expect(network.agent).to be(:http)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_AGENT"] = "ferrum" }
+
+      it "parses the env var" do
+        expect(network.agent).to be(:ferrum)
+      end
+    end
+  end
+
+  describe "#pool_size" do
+    context "by default" do
+      it "is 1" do
+        expect(network.pool_size).to be(1)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_POOL_SIZE"] = "42" }
+
+      it "parses the env var" do
+        expect(network.pool_size).to be(42)
+      end
+    end
+  end
+
+  describe "#pool_timeout" do
+    context "by default" do
+      it "is 10" do
+        expect(network.pool_timeout).to be(10)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_POOL_SIZE"] = "1337" }
+
+      it "parses the env var" do
+        expect(network.pool_size).to be(1337)
+      end
+    end
+  end
+
+  describe "#http_headers" do
+    context "by default" do
+      it "is {}" do
+        expect(network.http_headers).to eq({})
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
+
+      it "parses the env var" do
+        expect(network.http_headers).to eq("user-agent": "foo",
+                                           authorization: "bar")
+      end
+    end
+  end
+end

data/spec/config/redis_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Redis do
+  let(:env) { {} }
+  subject(:redis) { Wayfarer::Config::Redis.new(env) }
+
+  describe "#url" do
+    context "by default" do
+      it "is redis://localhost:6379" do
+        expect(redis.url).to eq("redis://localhost:6379")
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
+
+      it "parses the env var" do
+        expect(redis.url).to eq("redis://redis:6379")
+      end
+    end
+  end
+
+  describe "#factory" do
+    context "by default" do
+      it "instantiates Redis" do
+        expect(redis.factory.call(redis)).to be_a(::Redis)
+      end
+    end
+  end
+end

data/spec/config/root_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Root do
+  subject(:config) { Wayfarer::Config::Root.new }
+
+  describe "#ferrum" do
+    it "returns a Ferrum config" do
+      expect(config.ferrum).to be_a(Wayfarer::Config::Ferrum)
+    end
+  end
+
+  describe "#network" do
+    it "returns a network config" do
+      expect(config.network).to be_a(Wayfarer::Config::Networking)
+    end
+  end
+
+  describe "#redis" do
+    it "returns a Redis config" do
+      expect(config.redis).to be_a(Wayfarer::Config::Redis)
+    end
+  end
+
+  describe "#selenium" do
+    it "returns a Selenium config" do
+      expect(config.selenium).to be_a(Wayfarer::Config::Selenium)
+    end
+  end
+end

data/spec/config/selenium_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Selenium do
+  let(:env) { {} }
+  subject(:selenium) { Wayfarer::Config::Selenium.new(env) }
+
+  describe "#driver" do
+    context "by default" do
+      it "is :chrome" do
+        expect(selenium.driver).to be(:chrome)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_SELENIUM_DRIVER"] = "firefox" }
+
+      it "parses the env var" do
+        expect(selenium.driver).to be(:firefox)
+      end
+    end
+  end
+
+  describe "#options" do
+    context "by default" do
+      it "is {}" do
+        expect(selenium.options).to eq({})
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_SELENIUM_OPTIONS"] = "url:http://firefox" }
+
+      it "parses the env var" do
+        expect(selenium.options).to eq(url: "http://firefox")
+      end
+    end
+  end
+
+  describe "#client_timeout" do
+    context "by default" do
+      it "is 60" do
+        expect(selenium.client_timeout).to be(60)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_SELENIUM_CLIENT_TIMEOUT"] = "10" }
+
+      it "parses the env var" do
+        expect(selenium.client_timeout).to be(10)
+      end
+    end
+  end
+end

data/spec/config/strconv_spec.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Strconv do
+  subject(:strconv) { Wayfarer::Config::Strconv }
+
+  describe "::parse" do
+    describe Hash do
+      it "parses" do
+        input = "alpha:1,beta:two,gamma:true"
+        output = strconv.parse(input, Hash)
+        expect(output).to eq(alpha: 1, beta: "two", gamma: true)
+      end
+    end
+
+    describe Array do
+      it "parses" do
+        input = "alpha, beta , gamma"
+        output = strconv.parse(input, Array)
+        expect(output).to eq(%w[alpha beta gamma])
+      end
+    end
+
+    describe Symbol do
+      it "parses" do
+        expect(strconv.parse("foobar", Symbol)).to be(:foobar)
+      end
+    end
+
+    describe Integer do
+      it "parses" do
+        expect(strconv.parse("42", Integer)).to be(42)
+      end
+    end
+
+    describe "Primitives" do
+      context "Booleans" do
+        it "parses" do
+          expect(strconv.parse("true")).to be(true)
+          expect(strconv.parse("false")).to be(false)
+        end
+      end
+
+      context "Numbers" do
+        it "parses" do
+          expect(strconv.parse("42")).to be(42)
+        end
+      end
+
+      context "Strings" do
+        it "parses" do
+          expect(strconv.parse("foobar")).to be("foobar")
+        end
+      end
+    end
+  end
+end

data/spec/config/struct_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Struct do
+  let(:struct) do
+    Wayfarer::Config::Struct.new(members)
+  end
+
+  let(:members) { { foo: options } }
+  let(:options) { {} }
+  let(:env) { {} }
+  subject { struct.new(env) }
+
+  describe "Reader" do
+    context "without environment values and default" do
+      let(:options) { {} }
+
+      it "returns nil" do
+        expect(subject.foo).to be(nil)
+      end
+    end
+
+    context "with default only" do
+      let(:options) { { default: 42 } }
+
+      it "returns the default" do
+        expect(subject.foo).to be(42)
+      end
+    end
+
+    context "with environment key specified" do
+      let(:options) { { env_key: "FOO" } }
+
+      context "with environment value" do
+        let(:env) { { "FOO" => "hello" } }
+
+        it "returns the value" do
+          expect(subject.foo).to eq("hello")
+        end
+
+        context "with type specified" do
+          let(:options) { { env_key: "FOO", type: Symbol } }
+
+          it "parses the value" do
+            expect(subject.foo).to be(:hello)
+          end
+        end
+      end
+
+      context "without environment value" do
+        it "returns nil" do
+          expect(subject.foo).to be(nil)
+        end
+      end
+    end
+  end
+
+  describe "Writer" do
+    it "allows overriding environment values and defaults" do
+      expect {
+        subject.foo = 3
+      }.to change { subject.foo }.from(nil).to(3)
+    end
+  end
+end

data/spec/factories/middleware.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :middleware, class: OpenStruct do
+    receiver { -> {} }
+
+    initialize_with do
+      new(receiver: receiver).tap do |middleware|
+        middleware.define_singleton_method(:call) do |task, &block|
+          receiver.call(task, &block)
+        end
+      end
+    end
+  end
+end

data/spec/factories/page.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :page, class: Wayfarer::Page do
+    url { "https://example.com" }
+    status_code { 200 }
+    headers { {} }
+    body { "" }
+
+    trait :html do
+      headers { { "Content-Type" => "text/html" } }
+      body do
+        <<~HTML
+          <!doctype html>
+          <html lang=en>
+            <head>
+              <meta charset=utf-8>
+              <title>Title</title>
+            </head>
+            <body></body>
+          </html>
+        HTML
+      end
+    end
+
+    trait :xml do
+      headers { { "Content-Type" => "application/xml" } }
+      body do
+        <<~XML
+          <menu id="file" value="File">
+            <popup>
+              <menuitem value="New" onclick="CreateNewDoc()" />
+              <menuitem value="Open" onclick="OpenDoc()" />
+              <menuitem value="Close" onclick="CloseDoc()" />
+            </popup>
+          </menu>
+        XML
+      end
+    end
+
+    trait :json do
+      headers { { "Content-Type" => "application/json" } }
+      body do
+        <<~JSON
+          {
+            "menu":{
+              "id":"file",
+              "value":"File",
+              "popup":{
+                "menuitem":[
+                  {
+                    "value":"New",
+                    "onclick":"CreateNewDoc()"
+                  },
+                  {
+                    "value":"Open",
+                    "onclick":"OpenDoc()"
+                  },
+                  {
+                    "value":"Close",
+                    "onclick":"CloseDoc()"
+                  }
+                ]
+              }
+            }
+          }
+        JSON
+      end
+    end
+
+    initialize_with do
+      new(url: url,
+          status_code: status_code,
+          body: body,
+          headers: headers)
+    end
+  end
+end

data/spec/factories/task.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :task, class: Wayfarer::Task do
+    url { "https://example.com" }
+    batch { "batch" }
+
+    initialize_with do
+      new(url, batch)
+    end
+  end
+end

data/spec/fixtures/dummy_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class DummyJob < Wayfarer::Base
+  route { to :index }
+
+  def hello; end
+end

data/spec/gc_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::GC, redis: true do
+  include Wayfarer::Redis::Connection
+
+  let(:task) { build(:task) }
+  let(:klass) { Class.new(Wayfarer::Base) }
+  let(:job) { klass.new }
+  subject(:gc) { Wayfarer::GC.new(job) }
+
+  before do
+    job.arguments = [task]
+    task.barrier.seen?(task.url)
+  end
+
+  describe "#run" do
+    context "when counter reaches 0" do
+      before { task.counter.increment }
+
+      it "resets the barrier" do
+        expect {
+          gc.run
+        }.to change { redis { |conn| conn.exists?(task.barrier.redis_key) } }.to(false)
+      end
+
+      it "resets the counter" do
+        expect {
+          gc.run
+        }.to change { redis { |conn| conn.exists?(task.counter.redis_key) } }.to(false)
+      end
+
+      it "runs after batch callbacks" do
+        expect { |spy|
+          klass.after_batch(&spy)
+          gc.run
+        }.to yield_control
+      end
+    end
+
+    context "when counter does not reach 0" do
+      before { 2.times { task.counter.increment } }
+
+      it "does not reset the barrier" do
+        expect {
+          gc.run
+        }.not_to(change { redis { |conn| conn.exists?(task.barrier.redis_key) } })
+      end
+
+      it "does not reset the counter" do
+        expect {
+          gc.run
+        }.not_to(change { redis { |conn| conn.exists?(task.counter.redis_key) } })
+      end
+
+      it "does not run after batch callbacks" do
+        expect(klass).not_to receive(:run_after_batch_callbacks)
+        gc.run
+      end
+    end
+  end
+end