wayfarer 0.4.0 → 0.4.1

data/lib/wayfarer/routing/matchers/url.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  module Routing
+    module Matchers
+      URL = Struct.new(:url) do
+        def match(url)
+          url == Addressable::URI.parse(self.url)
+        end
+
+        def params(_)
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/route.rb

@@ -12,7 +12,7 @@ module Wayfarer
                     :action,
                     :path_offset
 
-      def initialize(matcher = CustomMatcher.new { children.any? }, path_offset = "/")
+      def initialize(matcher = Matchers::Custom.new { children.any? }, path_offset = "/")
         @matcher = matcher
         @children = []
         @path_offset = path_offset

data/lib/wayfarer.rb

@@ -28,8 +28,8 @@ require "zeitwerk"
 loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect("cli" => "CLI",
                          "dsl" => "DSL",
-                         "url_matcher" => "URLMatcher",
-                         "net_http" => "NetHTTP",
+                         "http" => "HTTP",
+                         "url" => "URL",
                          "xml" => "XML",
                          "json" => "JSON",
                          "gc" => "GC")
@@ -39,17 +39,17 @@ module Wayfarer
   module VERSION
     MAJOR = 0
     MINOR = 4
-    TINY = 0
+    TINY = 1
     STRING = [MAJOR, MINOR, TINY].join(".")
   end
 
-  mattr_accessor :config, default: Wayfarer::Config.from_environment
+  mattr_accessor :config, default: Wayfarer::Config::Root.new
 
-  mattr_accessor :core_middleware, default: [Wayfarer::Middleware::Stage.new,
-                                             Wayfarer::Middleware::Dedup.new,
-                                             Wayfarer::Middleware::Normalize.new,
-                                             Wayfarer::Middleware::Router.new,
-                                             Wayfarer::Middleware::Fetch.new]
+  mattr_accessor :middleware, default: [Wayfarer::Middleware::Stage.new,
+                                        Wayfarer::Middleware::Dedup.new,
+                                        Wayfarer::Middleware::Normalize.new,
+                                        Wayfarer::Middleware::Router.new,
+                                        Wayfarer::Middleware::Fetch.new]
 end
 
 loader.eager_load

data/spec/base_spec.rb

@@ -155,6 +155,13 @@ describe Wayfarer::Base, redis: true do
       }.to change { enqueued_jobs.size }.by(-1)
     end
 
+    it "marks the URL seen" do
+      task.counter.increment # otherwise barrier gets reset
+      DummyJob.crawl_later(url, batch: batch)
+      3.times { perform_enqueued_jobs }
+      expect(task.barrier.seen?(task.url)).to be(true)
+    end
+
     it "decrements the counter" do
       3.times { task.counter.increment }
 
@@ -199,6 +206,13 @@ describe Wayfarer::Base, redis: true do
       }.to change { enqueued_jobs.size }.by(-1)
     end
 
+    it "marks the URL seen" do
+      task.counter.increment # otherwise barrier gets reset
+      DummyJob.crawl_later(url, batch: batch)
+      perform_enqueued_jobs
+      expect(task.barrier.seen?(task.url)).to be(true)
+    end
+
     it "decrements the counter" do
       3.times { task.counter.increment }
 

data/spec/callbacks_spec.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Callbacks do
+  let(:url) { "https://alpha.com" }
+  let(:task) { build(:task, batch: "batch", url: url) }
+
+  let(:klass) do
+    Class.new(Wayfarer::Base) do
+      route.host "alpha.com", to: :alpha
+      route.host "beta.com", to: :beta
+
+      before_fetch do |job|
+        Spy.before_fetch(self, job)
+      end
+
+      before_fetch(except: %i[alpha beta], only: :alpha) do |job|
+        Spy.before_fetch_except_alpha_beta_only_alpha(self, job)
+      end
+
+      before_action do |job|
+        Spy.before_action(self, job)
+      end
+
+      before_action(only: :alpha) do |job|
+        Spy.before_action_only_alpha(self, job)
+      end
+
+      before_action(only: :beta) do |job|
+        Spy.before_action_only_beta(self, job)
+      end
+
+      before_action(only: %i[alpha gamma]) do |job|
+        Spy.before_action_only_alpha_gamma(self, job)
+      end
+
+      before_action(except: :alpha) do |job|
+        Spy.before_action_except_alpha(self, job)
+      end
+
+      before_action(except: %i[beta gamma]) do |job|
+        Spy.before_action_except_beta_gamma(self, job)
+      end
+
+      before_action :callback_a, only: :alpha
+
+      before_action :callback_b
+
+      before_action :callback_c, except: %i[beta]
+
+      def alpha; end
+
+      def beta; end
+
+    private
+
+      def callback_a
+        Spy.before_action_callback_a_only_alpha(self)
+      end
+
+      def callback_b
+        false
+      end
+
+      def callback_c
+        Spy.before_action_callback_c(self)
+      end
+    end
+  end
+
+  let(:job) { klass.new }
+
+  before do
+    stub_const("DummyJob", klass)
+    stub_const("Spy", spy)
+
+    allow(job).to receive(:arguments).and_return([task])
+  end
+
+  describe "before_fetch" do
+    it "fires" do
+      expect(Spy).to receive(:before_fetch).exactly(:once).with(job, job)
+      expect(Spy).not_to receive(:before_fetch_except_alpha_beta_only_alpha)
+      job.perform(task)
+    end
+  end
+
+  describe "before_action" do
+    it "fires" do
+      expect(Spy).to receive(:before_action).exactly(:once).with(job, job)
+      expect(Spy).to receive(:before_action_only_alpha).exactly(:once).with(job, job)
+      expect(Spy).not_to receive(:before_action_only_beta).with(job, job)
+      expect(Spy).to receive(:before_action_only_alpha_gamma).exactly(:once).with(job, job)
+      expect(Spy).not_to receive(:before_action_except_alpha).with(job, job)
+      expect(Spy).to receive(:before_action_except_beta_gamma).exactly(:once).with(job, job)
+      expect(Spy).to receive(:before_action_callback_a_only_alpha).exactly(:once).with(job)
+      expect(Spy).not_to receive(:before_action_callback_c).with(job)
+      job.perform(task)
+    end
+  end
+end

data/spec/cli/job_spec.rb

@@ -32,20 +32,20 @@ describe Wayfarer::CLI::Job, cli: true, redis: true do
     context "using MockRedis" do
       it "performs the worker using MockRedis" do
         cli.start(["job", "perform", "--mock-redis", "DummyJob", url])
-        expect(Wayfarer.config.redis_factory.call).to be_a(MockRedis)
+        expect(Wayfarer.config.redis.factory.call(nil)).to be_a(MockRedis)
       end
     end
   end
 
   describe "job enqueue" do
     it "enqueues the job" do
-      expect(DummyJob).to receive(:crawl_later).with(URI(url), batch: kind_of(String))
+      expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: kind_of(String))
       cli.start(["job", "enqueue", "DummyJob", url])
     end
 
     context "with batch provided" do
       it "enqueues the job" do
-        expect(DummyJob).to receive(:crawl_later).with(URI(url), batch: batch)
+        expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: batch)
         cli.start(["job", "enqueue", "--batch", batch, "DummyJob", url])
       end
     end
@@ -53,13 +53,13 @@ describe Wayfarer::CLI::Job, cli: true, redis: true do
 
   describe "job execute" do
     it "executes the job" do
-      expect(DummyJob).to receive(:crawl_later).with(URI(url), batch: kind_of(String))
+      expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: kind_of(String))
       cli.start(["job", "execute", "DummyJob", url])
     end
 
     context "with batch provided" do
       it "enqueues the job" do
-        expect(DummyJob).to receive(:crawl_later).with(URI(url), batch: batch)
+        expect(DummyJob).to receive(:crawl_later).with(Addressable::URI.parse(url), batch: batch)
         cli.start(["job", "execute", "--batch", batch, "DummyJob", url])
       end
     end
@@ -67,7 +67,7 @@ describe Wayfarer::CLI::Job, cli: true, redis: true do
     context "using MockRedis" do
       it "performs the worker using MockRedis" do
         cli.start(["job", "execute", "--mock-redis", "DummyJob", url])
-        expect(Wayfarer.config.redis_factory.call).to be_a(MockRedis)
+        expect(Wayfarer.config.redis.factory.call(nil)).to be_a(MockRedis)
       end
     end
   end

data/spec/config/capybara_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Capybara do
+  let(:env) { {} }
+  subject(:capybara) { Wayfarer::Config::Capybara.new(env) }
+
+  describe "#driver" do
+    context "with env var set" do
+      before { env["WAYFARER_CAPYBARA_DRIVER"] = "cuprite" }
+
+      it "parses the env var" do
+        expect(capybara.driver).to be(:cuprite)
+      end
+    end
+  end
+end

data/spec/config/ferrum_spec.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Ferrum do
+  let(:env) { {} }
+  subject(:ferrum) { Wayfarer::Config::Ferrum.new(env) }
+
+  describe "#options" do
+    context "by default" do
+      it "is {}" do
+        expect(ferrum.options).to eq({})
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_FERRUM_OPTIONS"] = "url:http://chrome:3000,headless:false" }
+
+      it "parses the env var" do
+        expect(ferrum.options).to eq(url: "http://chrome:3000", headless: false)
+      end
+    end
+  end
+end

data/spec/config/networking_spec.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Networking do
+  let(:env) { {} }
+  subject(:network) { Wayfarer::Config::Networking.new(env) }
+
+  describe "#agent" do
+    context "by default" do
+      it "is :http" do
+        expect(network.agent).to be(:http)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_AGENT"] = "ferrum" }
+
+      it "parses the env var" do
+        expect(network.agent).to be(:ferrum)
+      end
+    end
+  end
+
+  describe "#pool_size" do
+    context "by default" do
+      it "is 3" do
+        expect(network.pool_size).to be(3)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_POOL_SIZE"] = "42" }
+
+      it "parses the env var" do
+        expect(network.pool_size).to be(42)
+      end
+    end
+  end
+
+  describe "#pool_timeout" do
+    context "by default" do
+      it "is 10" do
+        expect(network.pool_timeout).to be(10)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_POOL_SIZE"] = "1337" }
+
+      it "parses the env var" do
+        expect(network.pool_size).to be(1337)
+      end
+    end
+  end
+
+  describe "#http_headers" do
+    context "by default" do
+      it "is {}" do
+        expect(network.http_headers).to eq({})
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_NETWORK_HTTP_HEADERS"] = "user-agent:foo,authorization:bar" }
+
+      it "parses the env var" do
+        expect(network.http_headers).to eq("user-agent": "foo",
+                                           authorization: "bar")
+      end
+    end
+  end
+end

data/spec/config/redis_spec.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Redis do
+  let(:env) { {} }
+  subject(:redis) { Wayfarer::Config::Redis.new(env) }
+
+  describe "#url" do
+    context "by default" do
+      it "is redis://localhost:6379" do
+        expect(redis.url).to eq("redis://localhost:6379")
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_REDIS_URL"] = "redis://redis:6379" }
+
+      it "parses the env var" do
+        expect(redis.url).to eq("redis://redis:6379")
+      end
+    end
+  end
+
+  describe "#factory" do
+    context "by default" do
+      it "instantiates Redis" do
+        expect(redis.factory.call(redis)).to be_a(::Redis)
+      end
+    end
+  end
+end

data/spec/config/root_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Root do
+  subject(:config) { Wayfarer::Config::Root.new }
+
+  describe "#ferrum" do
+    it "returns a Ferrum config" do
+      expect(config.ferrum).to be_a(Wayfarer::Config::Ferrum)
+    end
+  end
+
+  describe "#network" do
+    it "returns a network config" do
+      expect(config.network).to be_a(Wayfarer::Config::Networking)
+    end
+  end
+
+  describe "#redis" do
+    it "returns a Redis config" do
+      expect(config.redis).to be_a(Wayfarer::Config::Redis)
+    end
+  end
+
+  describe "#selenium" do
+    it "returns a Selenium config" do
+      expect(config.selenium).to be_a(Wayfarer::Config::Selenium)
+    end
+  end
+end

data/spec/config/selenium_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Selenium do
+  let(:env) { {} }
+  subject(:selenium) { Wayfarer::Config::Selenium.new(env) }
+
+  describe "#driver" do
+    context "by default" do
+      it "is :chrome" do
+        expect(selenium.driver).to be(:chrome)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_SELENIUM_DRIVER"] = "firefox" }
+
+      it "parses the env var" do
+        expect(selenium.driver).to be(:firefox)
+      end
+    end
+  end
+
+  describe "#options" do
+    context "by default" do
+      it "is {}" do
+        expect(selenium.options).to eq({})
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_SELENIUM_OPTIONS"] = "url:http://firefox" }
+
+      it "parses the env var" do
+        expect(selenium.options).to eq(url: "http://firefox")
+      end
+    end
+  end
+
+  describe "#client_timeout" do
+    context "by default" do
+      it "is 60" do
+        expect(selenium.client_timeout).to be(60)
+      end
+    end
+
+    context "with env var set" do
+      before { env["WAYFARER_SELENIUM_CLIENT_TIMEOUT"] = "10" }
+
+      it "parses the env var" do
+        expect(selenium.client_timeout).to be(10)
+      end
+    end
+  end
+end

data/spec/config/strconv_spec.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Strconv do
+  subject(:strconv) { Wayfarer::Config::Strconv }
+
+  describe "::parse" do
+    describe Hash do
+      it "parses" do
+        input = "alpha:1,beta:two,gamma:true"
+        output = strconv.parse(input, Hash)
+        expect(output).to eq(alpha: 1, beta: "two", gamma: true)
+      end
+    end
+
+    describe Array do
+      it "parses" do
+        input = "alpha, beta , gamma"
+        output = strconv.parse(input, Array)
+        expect(output).to eq(%w[alpha beta gamma])
+      end
+    end
+
+    describe Symbol do
+      it "parses" do
+        expect(strconv.parse("foobar", Symbol)).to be(:foobar)
+      end
+    end
+
+    describe Integer do
+      it "parses" do
+        expect(strconv.parse("42", Integer)).to be(42)
+      end
+    end
+
+    describe "Primitives" do
+      context "Booleans" do
+        it "parses" do
+          expect(strconv.parse("true")).to be(true)
+          expect(strconv.parse("false")).to be(false)
+        end
+      end
+
+      context "Numbers" do
+        it "parses" do
+          expect(strconv.parse("42")).to be(42)
+        end
+      end
+
+      context "Strings" do
+        it "parses" do
+          expect(strconv.parse("foobar")).to be("foobar")
+        end
+      end
+    end
+  end
+end

data/spec/config/struct_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Config::Struct do
+  let(:struct) do
+    Wayfarer::Config::Struct.new(members)
+  end
+
+  let(:members) { { foo: options } }
+  let(:options) { {} }
+  let(:env) { {} }
+  subject { struct.new(env) }
+
+  describe "Reader" do
+    context "without environment values and default" do
+      let(:options) { {} }
+
+      it "returns nil" do
+        expect(subject.foo).to be(nil)
+      end
+    end
+
+    context "with default only" do
+      let(:options) { { default: 42 } }
+
+      it "returns the default" do
+        expect(subject.foo).to be(42)
+      end
+    end
+
+    context "with environment key specified" do
+      let(:options) { { env_key: "FOO" } }
+
+      context "with environment value" do
+        let(:env) { { "FOO" => "hello" } }
+
+        it "returns the value" do
+          expect(subject.foo).to eq("hello")
+        end
+
+        context "with type specified" do
+          let(:options) { { env_key: "FOO", type: Symbol } }
+
+          it "parses the value" do
+            expect(subject.foo).to be(:hello)
+          end
+        end
+      end
+
+      context "without environment value" do
+        it "returns nil" do
+          expect(subject.foo).to be(nil)
+        end
+      end
+    end
+  end
+
+  describe "Writer" do
+    it "allows overriding environment values and defaults" do
+      expect {
+        subject.foo = 3
+      }.to change { subject.foo }.from(nil).to(3)
+    end
+  end
+end

data/spec/gc_spec.rb

@@ -6,13 +6,13 @@ describe Wayfarer::GC, redis: true do
   include Wayfarer::Redis::Connection
 
   let(:task) { build(:task) }
-  let(:klass) { Struct.new(:arguments) }
-  let(:job) { klass.new([task]) }
+  let(:klass) { Class.new(Wayfarer::Base) }
+  let(:job) { klass.new }
   subject(:gc) { Wayfarer::GC.new(job) }
 
   before do
-    allow(klass).to receive(:run_after_batch_callbacks)
-    task.barrier.seen?("https://example.com")
+    job.arguments = [task]
+    task.barrier.seen?(task.url)
   end
 
   describe "#run" do
@@ -32,8 +32,10 @@ describe Wayfarer::GC, redis: true do
       end
 
       it "runs after batch callbacks" do
-        expect(klass).to receive(:run_after_batch_callbacks).exactly(:once)
-        gc.run
+        expect { |spy|
+          klass.after_batch(&spy)
+          gc.run
+        }.to yield_control
       end
     end