wayfarer 0.4.0 → 0.4.4

data/spec/config/ferrum_spec.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Ferrum do
+  let(:env) { {} }
+  subject(:ferrum) { Wayfarer::Config::Ferrum.new(env) }
+
+  describe "#options" do
+    context "by default" do
+      it "is {}" do
+        expect(ferrum.options).to eq({})
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
+
+      it "parses the env var" do
+        expect(ferrum.options).to eq(url: "http://chrome:3000", headless: false)
+      end
+    end
+  end
+end

data/spec/config/networking_spec.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Networking do
+  let(:env) { {} }
+  subject(:network) { Wayfarer::Config::Networking.new(env) }
+
+  describe "#agent" do
+    context "by default" do
+      it "is :http" do
+        expect(network.agent).to be(:http)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_AGENT"] = "ferrum" }
+
+      it "parses the env var" do
+        expect(network.agent).to be(:ferrum)
+      end
+    end
+  end
+
+  describe "#pool_size" do
+    context "by default" do
+      it "is 1" do
+        expect(network.pool_size).to be(1)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_POOL_SIZE"] = "42" }
+
+      it "parses the env var" do
+        expect(network.pool_size).to be(42)
+      end
+    end
+  end
+
+  describe "#pool_timeout" do
+    context "by default" do
+      it "is 10" do
+        expect(network.pool_timeout).to be(10)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_POOL_SIZE"] = "1337" }
+
+      it "parses the env var" do
+        expect(network.pool_size).to be(1337)
+      end
+    end
+  end
+
+  describe "#http_headers" do
+    context "by default" do
+      it "is {}" do
+        expect(network.http_headers).to eq({})
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
+
+      it "parses the env var" do
+        expect(network.http_headers).to eq("user-agent": "foo",
+                                           authorization: "bar")
+      end
+    end
+  end
+end

data/spec/config/redis_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Redis do
+  let(:env) { {} }
+  subject(:redis) { Wayfarer::Config::Redis.new(env) }
+
+  describe "#url" do
+    context "by default" do
+      it "is redis://localhost:6379" do
+        expect(redis.url).to eq("redis://localhost:6379")
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
+
+      it "parses the env var" do
+        expect(redis.url).to eq("redis://redis:6379")
+      end
+    end
+  end
+
+  describe "#factory" do
+    context "by default" do
+      it "instantiates Redis" do
+        expect(redis.factory.call(redis)).to be_a(::Redis)
+      end
+    end
+  end
+end

data/spec/config/root_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Root do
+  subject(:config) { Wayfarer::Config::Root.new }
+
+  describe "#ferrum" do
+    it "returns a Ferrum config" do
+      expect(config.ferrum).to be_a(Wayfarer::Config::Ferrum)
+    end
+  end
+
+  describe "#network" do
+    it "returns a network config" do
+      expect(config.network).to be_a(Wayfarer::Config::Networking)
+    end
+  end
+
+  describe "#redis" do
+    it "returns a Redis config" do
+      expect(config.redis).to be_a(Wayfarer::Config::Redis)
+    end
+  end
+
+  describe "#selenium" do
+    it "returns a Selenium config" do
+      expect(config.selenium).to be_a(Wayfarer::Config::Selenium)
+    end
+  end
+end

data/spec/config/selenium_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Selenium do
+  let(:env) { {} }
+  subject(:selenium) { Wayfarer::Config::Selenium.new(env) }
+
+  describe "#driver" do
+    context "by default" do
+      it "is :chrome" do
+        expect(selenium.driver).to be(:chrome)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_SELENIUM_DRIVER"] = "firefox" }
+
+      it "parses the env var" do
+        expect(selenium.driver).to be(:firefox)
+      end
+    end
+  end
+
+  describe "#options" do
+    context "by default" do
+      it "is {}" do
+        expect(selenium.options).to eq({})
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_SELENIUM_OPTIONS"] = "url:http://firefox" }
+
+      it "parses the env var" do
+        expect(selenium.options).to eq(url: "http://firefox")
+      end
+    end
+  end
+
+  describe "#client_timeout" do
+    context "by default" do
+      it "is 60" do
+        expect(selenium.client_timeout).to be(60)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_SELENIUM_CLIENT_TIMEOUT"] = "10" }
+
+      it "parses the env var" do
+        expect(selenium.client_timeout).to be(10)
+      end
+    end
+  end
+end

data/spec/config/strconv_spec.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Strconv do
+  subject(:strconv) { Wayfarer::Config::Strconv }
+
+  describe "::parse" do
+    describe Hash do
+      it "parses" do
+        input = "alpha:1,beta:two,gamma:true"
+        output = strconv.parse(input, Hash)
+        expect(output).to eq(alpha: 1, beta: "two", gamma: true)
+      end
+    end
+
+    describe Array do
+      it "parses" do
+        input = "alpha, beta , gamma"
+        output = strconv.parse(input, Array)
+        expect(output).to eq(%w[alpha beta gamma])
+      end
+    end
+
+    describe Symbol do
+      it "parses" do
+        expect(strconv.parse("foobar", Symbol)).to be(:foobar)
+      end
+    end
+
+    describe Integer do
+      it "parses" do
+        expect(strconv.parse("42", Integer)).to be(42)
+      end
+    end
+
+    describe "Primitives" do
+      context "Booleans" do
+        it "parses" do
+          expect(strconv.parse("true")).to be(true)
+          expect(strconv.parse("false")).to be(false)
+        end
+      end
+
+      context "Numbers" do
+        it "parses" do
+          expect(strconv.parse("42")).to be(42)
+        end
+      end
+
+      context "Strings" do
+        it "parses" do
+          expect(strconv.parse("foobar")).to be("foobar")
+        end
+      end
+    end
+  end
+end

data/spec/config/struct_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Struct do
+  let(:struct) do
+    Wayfarer::Config::Struct.new(members)
+  end
+
+  let(:members) { { foo: options } }
+  let(:options) { {} }
+  let(:env) { {} }
+  subject { struct.new(env) }
+
+  describe "Reader" do
+    context "without environment values and default" do
+      let(:options) { {} }
+
+      it "returns nil" do
+        expect(subject.foo).to be(nil)
+      end
+    end
+
+    context "with default only" do
+      let(:options) { { default: 42 } }
+
+      it "returns the default" do
+        expect(subject.foo).to be(42)
+      end
+    end
+
+    context "with environment key specified" do
+      let(:options) { { env_key: "FOO" } }
+
+      context "with environment value" do
+        let(:env) { { "FOO" => "hello" } }
+
+        it "returns the value" do
+          expect(subject.foo).to eq("hello")
+        end
+
+        context "with type specified" do
+          let(:options) { { env_key: "FOO", type: Symbol } }
+
+          it "parses the value" do
+            expect(subject.foo).to be(:hello)
+          end
+        end
+      end
+
+      context "without environment value" do
+        it "returns nil" do
+          expect(subject.foo).to be(nil)
+        end
+      end
+    end
+  end
+
+  describe "Writer" do
+    it "allows overriding environment values and defaults" do
+      expect {
+        subject.foo = 3
+      }.to change { subject.foo }.from(nil).to(3)
+    end
+  end
+end

data/spec/factories/{queue/middleware.rb → middleware.rb}

@@ -2,12 +2,12 @@
 
 FactoryBot.define do
   factory :middleware, class: OpenStruct do
-    callee { -> {} }
+    receiver { -> {} }
 
     initialize_with do
-      new(callee: callee).tap do |middleware|
+      new(receiver: receiver).tap do |middleware|
         middleware.define_singleton_method(:call) do |task, &block|
-          callee.call(task, &block)
+          receiver.call(task, &block)
         end
       end
     end

data/spec/factories/{queue/page.rb → page.rb}

@@ -8,7 +8,7 @@ FactoryBot.define do
     body { "" }
 
     trait :html do
-      headers { { "Content-Type" => %w[text/html] } }
+      headers { { "Content-Type" => "text/html" } }
       body do
         <<~HTML
           <!doctype html>
@@ -24,7 +24,7 @@ FactoryBot.define do
     end
 
     trait :xml do
-      headers { { "Content-Type" => %w[application/xml] } }
+      headers { { "Content-Type" => "application/xml" } }
       body do
         <<~XML
           <menu id="file" value="File">
@@ -39,7 +39,7 @@ FactoryBot.define do
     end
 
     trait :json do
-      headers { { "Content-Type" => %w[application/json] } }
+      headers { { "Content-Type" => "application/json" } }
       body do
         <<~JSON
           {

data/spec/fixtures/dummy_job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class DummyJob < Wayfarer::Base
-  route.to :index
+  route { to :index }
 
   def hello; end
 end

data/spec/gc_spec.rb

@@ -6,13 +6,11 @@ describe Wayfarer::GC, redis: true do
   include Wayfarer::Redis::Connection
 
   let(:task) { build(:task) }
-  let(:klass) { Struct.new(:arguments) }
-  let(:job) { klass.new([task]) }
-  subject(:gc) { Wayfarer::GC.new(job) }
+  subject(:gc) { described_class.new(task) }
 
   before do
-    allow(klass).to receive(:run_after_batch_callbacks)
-    task.barrier.seen?("https://example.com")
+    task.metadata.job = spy
+    task.barrier.seen?(task.url)
   end
 
   describe "#run" do
@@ -32,7 +30,7 @@ describe Wayfarer::GC, redis: true do
       end
 
       it "runs after batch callbacks" do
-        expect(klass).to receive(:run_after_batch_callbacks).exactly(:once)
+        expect(task.metadata.job).to receive(:run_callbacks).with(:batch).exactly(:once)
         gc.run
       end
     end
@@ -53,7 +51,7 @@ describe Wayfarer::GC, redis: true do
       end
 
       it "does not run after batch callbacks" do
-        expect(klass).not_to receive(:run_after_batch_callbacks)
+        expect(task.metadata.job).not_to receive(:run_callbacks).with(:batch)
         gc.run
       end
     end

data/spec/handler_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Handler do
+  subject { Class.new(described_class) }
+
+  it "undefines ::after_batch" do
+    expect(subject).not_to respond_to(:after_batch)
+  end
+end

data/spec/integration/callbacks_spec.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Callbacks" do
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
+  describe Wayfarer::Base do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+        include RSpec::Matchers
+
+        route { host test_app_host, to: :index }
+
+        attr_accessor :callbacks_fired
+
+        before_fetch do
+          self.callbacks_fired = %i[before_fetch]
+        end
+
+        before_action do
+          callbacks_fired.push(:before_action)
+        end
+
+        after_batch do
+          expect(callbacks_fired).to eq(%i[before_fetch before_action])
+        end
+
+        def index; end
+      end
+
+      self.class::DummyJob.crawl(url)
+      perform_enqueued_jobs
+    end
+  end
+
+  describe Wayfarer::Handler do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+        include RSpec::Matchers
+
+        route { host test_app_host, to: DummyHandler }
+
+        attr_accessor :callbacks_fired
+
+        before_fetch do
+          self.callbacks_fired = %i[before_fetch_job]
+        end
+
+        before_action do
+          callbacks_fired.push(:before_action_job)
+        end
+
+        after_batch do
+          expect(callbacks_fired).to eq(%i[before_fetch_job before_action_job])
+        end
+
+        class DummyHandler < Wayfarer::Handler
+          include RSpec::Matchers
+
+          route { to :index }
+
+          attr_accessor :callbacks_fired
+
+          before_fetch do
+            raise "before_fetch ran in handler"
+          end
+
+          before_action do
+            self.callbacks_fired = %i[before_action_handler]
+          end
+
+          def index
+            expect(callbacks_fired).to eq(%i[before_action_handler])
+          end
+        end
+      end
+
+      self.class::DummyJob.crawl(url)
+      perform_enqueued_jobs
+    end
+  end
+end

data/spec/integration/page_spec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "Pages" do
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
+  describe Wayfarer::Base do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+        include RSpec::Matchers
+
+        route { host test_app_host, to: :index }
+
+        def index
+          expect(page.url).to eq("http://test:9876/git-scm.com/book/en/v2.html")
+          expect(page.status_code).to be(200)
+          expect(page.body).not_to be_empty
+          expect(page.headers.count).to be(9)
+
+          expect(page.meta.links.all.count).to be(157)
+          expect(page.meta.links.internal.count).to be(102)
+          expect(page.meta.links.external.count).to be(55)
+        end
+      end
+
+      self.class::DummyJob.crawl(url)
+      perform_enqueued_jobs
+    end
+  end
+
+  describe Wayfarer::Handler do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+
+        route { host test_app_host, to: DummyHandler }
+
+        class DummyHandler < Wayfarer::Handler
+          include RSpec::Matchers
+
+          route { to :index }
+
+          def index
+            expect(page.url).to eq("http://test:9876/git-scm.com/book/en/v2.html")
+            expect(page.status_code).to be(200)
+            expect(page.body).not_to be_empty
+            expect(page.headers.count).to be(9)
+
+            expect(page.meta.links.all.count).to be(157)
+            expect(page.meta.links.internal.count).to be(102)
+            expect(page.meta.links.external.count).to be(55)
+          end
+        end
+      end
+
+      self.class::DummyJob.crawl(url)
+      perform_enqueued_jobs
+    end
+  end
+end

data/spec/integration/params_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe "URL parameters" do
+  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
+
+  describe Wayfarer::Base do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+        include RSpec::Matchers
+
+        route do
+          to :index, host: test_app_host do
+            path "git-scm.com/book/:lang/:file"
+          end
+        end
+
+        def index
+          expect(params).to eq("lang" => "en", "file" => "v2.html")
+        end
+      end
+
+      self.class::DummyJob.crawl(url)
+      perform_enqueued_jobs
+    end
+  end
+
+  describe Wayfarer::Handler do
+    specify do
+      class self.class::DummyJob < Wayfarer::Base
+        extend SpecHelpers
+
+        route do
+          to DummyHandler, host: test_app_host do
+            path "git-scm.com/book/:lang/:file"
+          end
+        end
+
+        class DummyHandler < Wayfarer::Handler
+          include RSpec::Matchers
+
+          route { to :index }
+
+          def index
+            expect(params).to eq("lang" => "en", "file" => "v2.html")
+          end
+        end
+      end
+
+      self.class::DummyJob.crawl(url)
+      perform_enqueued_jobs
+    end
+  end
+end