wayfarer 0.4.6 → 0.4.8

data/spec/config/strconv_spec.rb

@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Config::Strconv do
-  subject(:strconv) { Wayfarer::Config::Strconv }
-
-  describe "::parse" do
-    describe Hash do
-      it "parses" do
-        input = "alpha:1,beta:two,gamma:true"
-        output = strconv.parse(input, Hash)
-        expect(output).to eq(alpha: 1, beta: "two", gamma: true)
-      end
-    end
-
-    describe Array do
-      it "parses" do
-        input = "alpha, beta , gamma"
-        output = strconv.parse(input, Array)
-        expect(output).to eq(%w[alpha beta gamma])
-      end
-    end
-
-    describe Symbol do
-      it "parses" do
-        expect(strconv.parse("foobar", Symbol)).to be(:foobar)
-      end
-    end
-
-    describe Integer do
-      it "parses" do
-        expect(strconv.parse("42", Integer)).to be(42)
-      end
-    end
-
-    describe "Primitives" do
-      context "Booleans" do
-        it "parses" do
-          expect(strconv.parse("true")).to be(true)
-          expect(strconv.parse("false")).to be(false)
-        end
-      end
-
-      context "Numbers" do
-        it "parses" do
-          expect(strconv.parse("42")).to be(42)
-        end
-      end
-
-      context "Strings" do
-        it "parses" do
-          expect(strconv.parse("foobar")).to be("foobar")
-        end
-      end
-    end
-  end
-end

data/spec/config/struct_spec.rb

@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Config::Struct do
-  let(:struct) do
-    Wayfarer::Config::Struct.new(members)
-  end
-
-  let(:members) { { foo: options } }
-  let(:options) { {} }
-  let(:env) { {} }
-  subject { struct.new(env) }
-
-  describe "Reader" do
-    context "without environment values and default" do
-      let(:options) { {} }
-
-      it "returns nil" do
-        expect(subject.foo).to be(nil)
-      end
-    end
-
-    context "with default only" do
-      let(:options) { { default: 42 } }
-
-      it "returns the default" do
-        expect(subject.foo).to be(42)
-      end
-    end
-
-    context "with environment key specified" do
-      let(:options) { { env_key: "FOO" } }
-
-      context "with environment value" do
-        let(:env) { { "FOO" => "hello" } }
-
-        it "returns the value" do
-          expect(subject.foo).to eq("hello")
-        end
-
-        context "with type specified" do
-          let(:options) { { env_key: "FOO", type: Symbol } }
-
-          it "parses the value" do
-            expect(subject.foo).to be(:hello)
-          end
-        end
-      end
-
-      context "without environment value" do
-        it "returns nil" do
-          expect(subject.foo).to be(nil)
-        end
-      end
-    end
-  end
-
-  describe "Writer" do
-    it "allows overriding environment values and defaults" do
-      expect {
-        subject.foo = 3
-      }.to change { subject.foo }.from(nil).to(3)
-    end
-  end
-end

data/spec/fixtures/dummy_job.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-class DummyJob < Wayfarer::Base
-  route { to :index }
-
-  def hello; end
-end

data/spec/gc_spec.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::GC, redis: true do
-  include Wayfarer::Redis::Connection
-
-  let(:task) { build(:task) }
-  subject(:gc) { described_class.new(task) }
-
-  before do
-    task.metadata.job = spy
-    task.barrier.seen?(task.url)
-  end
-
-  describe "#run" do
-    context "when counter reaches 0" do
-      before { task.counter.increment }
-
-      it "resets the barrier" do
-        expect {
-          gc.run
-        }.to change { redis { |conn| conn.exists?(task.barrier.redis_key) } }.to(false)
-      end
-
-      it "resets the counter" do
-        expect {
-          gc.run
-        }.to change { redis { |conn| conn.exists?(task.counter.redis_key) } }.to(false)
-      end
-
-      it "runs after batch callbacks" do
-        expect(task.metadata.job).to receive(:run_callbacks).with(:batch).exactly(:once)
-        gc.run
-      end
-    end
-
-    context "when counter does not reach 0" do
-      before { 2.times { task.counter.increment } }
-
-      it "does not reset the barrier" do
-        expect {
-          gc.run
-        }.not_to(change { redis { |conn| conn.exists?(task.barrier.redis_key) } })
-      end
-
-      it "does not reset the counter" do
-        expect {
-          gc.run
-        }.not_to(change { redis { |conn| conn.exists?(task.counter.redis_key) } })
-      end
-
-      it "does not run after batch callbacks" do
-        expect(task.metadata.job).not_to receive(:run_callbacks).with(:batch)
-        gc.run
-      end
-    end
-  end
-end

data/spec/handler_spec.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Handler do
-  subject { Class.new(described_class) }
-
-  it "undefines ::after_batch" do
-    expect(subject).not_to respond_to(:after_batch)
-  end
-end

data/spec/integration/callbacks_spec.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe "Callbacks" do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
-
-  describe Wayfarer::Base do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-        include RSpec::Matchers
-
-        route { host test_app_host, to: :index }
-
-        attr_accessor :callbacks_fired
-
-        before_fetch do
-          self.callbacks_fired = %i[before_fetch]
-        end
-
-        before_action do
-          callbacks_fired.push(:before_action)
-        end
-
-        after_batch do
-          expect(callbacks_fired).to eq(%i[before_fetch before_action])
-        end
-
-        def index; end
-      end
-
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
-    end
-  end
-
-  describe Wayfarer::Handler do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-        include RSpec::Matchers
-
-        route { host test_app_host, to: DummyHandler }
-
-        attr_accessor :callbacks_fired
-
-        before_fetch do
-          self.callbacks_fired = %i[before_fetch_job]
-        end
-
-        before_action do
-          callbacks_fired.push(:before_action_job)
-        end
-
-        after_batch do
-          expect(callbacks_fired).to eq(%i[before_fetch_job before_action_job])
-        end
-
-        class DummyHandler < Wayfarer::Handler
-          include RSpec::Matchers
-
-          route { to :index }
-
-          attr_accessor :callbacks_fired
-
-          before_fetch do
-            raise "before_fetch ran in handler"
-          end
-
-          before_action do
-            self.callbacks_fired = %i[before_action_handler]
-          end
-
-          def index
-            expect(callbacks_fired).to eq(%i[before_action_handler])
-          end
-        end
-      end
-
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
-    end
-  end
-end

data/spec/integration/page_spec.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe "Pages" do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
-
-  describe Wayfarer::Base do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-        include RSpec::Matchers
-
-        route { host test_app_host, to: :index }
-
-        def index
-          expect(page.url).to eq("http://test:9876/git-scm.com/book/en/v2.html")
-          expect(page.status_code).to be(200)
-          expect(page.body).not_to be_empty
-          expect(page.headers.count).to be(9)
-
-          expect(page.meta.links.all.count).to be(157)
-          expect(page.meta.links.internal.count).to be(102)
-          expect(page.meta.links.external.count).to be(55)
-        end
-      end
-
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
-    end
-  end
-
-  describe Wayfarer::Handler do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-
-        route { host test_app_host, to: DummyHandler }
-
-        class DummyHandler < Wayfarer::Handler
-          include RSpec::Matchers
-
-          route { to :index }
-
-          def index
-            expect(page.url).to eq("http://test:9876/git-scm.com/book/en/v2.html")
-            expect(page.status_code).to be(200)
-            expect(page.body).not_to be_empty
-            expect(page.headers.count).to be(9)
-
-            expect(page.meta.links.all.count).to be(157)
-            expect(page.meta.links.internal.count).to be(102)
-            expect(page.meta.links.external.count).to be(55)
-          end
-        end
-      end
-
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
-    end
-  end
-end

data/spec/integration/params_spec.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe "URL parameters" do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
-
-  describe Wayfarer::Base do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-        include RSpec::Matchers
-
-        route do
-          to :index, host: test_app_host do
-            path "git-scm.com/book/:lang/:file"
-          end
-        end
-
-        def index
-          expect(params).to eq("lang" => "en", "file" => "v2.html")
-        end
-      end
-
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
-    end
-  end
-
-  describe Wayfarer::Handler do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-
-        route do
-          to DummyHandler, host: test_app_host do
-            path "git-scm.com/book/:lang/:file"
-          end
-        end
-
-        class DummyHandler < Wayfarer::Handler
-          include RSpec::Matchers
-
-          route { to :index }
-
-          def index
-            expect(params).to eq("lang" => "en", "file" => "v2.html")
-          end
-        end
-      end
-
-      self.class::DummyJob.crawl(url)
-      perform_enqueued_jobs
-    end
-  end
-end

data/spec/integration/stage_spec.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe "Staging" do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
-
-  describe Wayfarer::Base do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-
-        route { host test_app_host, to: :index }
-
-        def index
-          stage page.meta.links.all
-        end
-      end
-
-      expect {
-        self.class::DummyJob.crawl(url)
-        perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(156)
-    end
-  end
-
-  describe Wayfarer::Handler do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend SpecHelpers
-
-        route do
-          host test_app_host, to: DummyHandler
-        end
-
-        class DummyHandler < Wayfarer::Handler
-          route { to :index }
-
-          def index
-            stage page.meta.links.all
-          end
-        end
-      end
-
-      expect {
-        self.class::DummyJob.crawl(url)
-        perform_enqueued_jobs
-      }.to change { enqueued_jobs.size }.by(156)
-    end
-  end
-end

data/spec/integration/steering_spec.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe "Steering" do
-  let(:url) { test_app_path("git-scm.com/book/en/v2.html") }
-
-  describe Wayfarer::Base do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        extend RSpec::Matchers
-
-        route do |a, b|
-          expect(a).to eq("foobar")
-          expect(b).to eq("barqux")
-        end
-
-        steer do |task|
-          [task.batch, "barqux"]
-        end
-      end
-
-      self.class::DummyJob.crawl(url, batch: "foobar")
-      perform_enqueued_jobs
-    end
-  end
-
-  describe Wayfarer::Handler do
-    specify do
-      class self.class::DummyJob < Wayfarer::Base
-        route do |_a, _b|
-          to DummyHandler
-        end
-
-        steer do |_task|
-          [123, "barqux"]
-        end
-
-        class DummyHandler < Wayfarer::Handler
-          extend RSpec::Matchers
-
-          route do |a, b|
-            expect(a).to eq("foobar")
-            expect(b).to eq("fooz")
-          end
-
-          steer do |task|
-            [task.batch, "fooz"]
-          end
-        end
-      end
-
-      self.class::DummyJob.crawl(url, batch: "foobar")
-      perform_enqueued_jobs
-    end
-  end
-end

data/spec/middleware/dedup_spec.rb

@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Middleware::Dedup, redis: true do
-  let(:task) { build(:task) }
-  subject { described_class.new }
-
-  before { task.metadata.staged_urls = SortedSet.new }
-
-  describe "#call" do
-    context "if already routed" do
-      before { task.metadata.action = :action }
-
-      it "does not call the barrier" do
-        expect(task.barrier).not_to receive(:seen?)
-      end
-
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
-      end
-    end
-
-    context "without staged URLs" do
-      it "does not raise" do
-        expect { subject.call(task) }.not_to raise_error
-      end
-    end
-
-    context "with unseen URL" do
-      it "marks the URL as seen" do
-        expect {
-          subject.call(task)
-        }.to change { task.barrier.seen?(task.url) }.to(true)
-      end
-
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
-      end
-    end
-
-    context "with seen URL" do
-      before { task.barrier.seen?(task.url) }
-
-      it "does not yield" do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
-      end
-    end
-
-    context "with exception raised" do
-      it "does not mark the URL as seen" do
-        begin
-          subject.call(task) { raise }
-        rescue StandardError
-          nil
-        end
-        expect(task.barrier.seen?(task.url)).to be(false)
-      end
-
-      it "re-raises the exception" do
-        expect {
-          subject.call(task) { raise }
-        }.to raise_error(RuntimeError)
-      end
-    end
-
-    describe "staged URL filtering" do
-      let(:seen_urls) { %w[https://yahoo.com https://google.com] }
-      let(:unseen_urls) { %w[https://w3c.org https://nasa.gov] }
-
-      before do
-        seen_urls.each do |url|
-          task.barrier.seen?(url)
-        end
-
-        [*seen_urls, *unseen_urls].each do |url|
-          task.metadata.staged_urls.add(url)
-        end
-      end
-
-      it "filters seen staged URLs" do
-        expect {
-          subject.call(task)
-        }.to change { task.metadata.staged_urls }.to(SortedSet.new(unseen_urls))
-      end
-    end
-  end
-end

data/spec/middleware/dispatch_spec.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Middleware::Dispatch do
-  let(:task) { build(:task) }
-  let(:action) { :action }
-  subject(:chain) { described_class.new }
-
-  before do
-    task.metadata.controller = spy
-    task.metadata.action = action
-
-    allow(task.metadata.controller).to receive(:run_callbacks).and_yield
-  end
-
-  describe "#call" do
-    it "runs callbacks" do
-      expect(task.metadata.controller).to receive(:run_callbacks).with(action)
-      subject.call(task)
-    end
-
-    context "when action is a Symbol" do
-      it "calls the method" do
-        expect(task.metadata.controller).to receive(action)
-        subject.call(task)
-      end
-    end
-
-    context "with other action" do
-      let(:action) { Class.new }
-
-      it "instantiates and calls" do
-        expect_any_instance_of(action).to receive(:call).with(task)
-        subject.call(task)
-      end
-    end
-
-    it "yields" do
-      expect { |spy| subject.call(task, &spy) }.to yield_control
-    end
-  end
-end