wayfarer 0.0.3 → 0.4.0

data/spec/middleware/chain_spec.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Chain do
+  let(:task) { build(:task) }
+  subject(:chain) { build(:chain, middlewares: middlewares) }
+
+  describe "#call" do
+    context "without middlewares" do
+      let(:middlewares) { [] }
+
+      it "does not raise" do
+        expect { chain.call(task) }.not_to raise_error
+      end
+
+      it "yields" do
+        expect { |spy| chain.call(task, &spy) }.to yield_control
+      end
+    end
+
+    context "when middleware yields" do
+      let(:middlewares) { [spy, spy] }
+
+      it "yields" do
+        allow(middlewares.first).to receive(:call).and_yield
+        expect(middlewares.last).to receive(:call).with(task).and_yield
+        expect { |spy| chain.call(task, &spy) }.to yield_control
+      end
+    end
+
+    context "when middleware does not yield" do
+      let(:middlewares) { [spy, spy] }
+
+      it "does not yield" do
+        allow(middlewares.first).to receive(:call)
+        expect(middlewares.last).not_to receive(:call)
+        expect { |spy| chain.call(task, &spy) }.not_to yield_control
+      end
+    end
+
+    describe "Immutability" do
+      let(:middlewares) { [spy, spy] }
+
+      it "does not alter its middleware" do
+        allow(middlewares.first).to receive(:call).and_yield
+        allow(middlewares.second).to receive(:call).and_yield
+        expect { chain.call(task) }.not_to(change { chain.middlewares })
+      end
+    end
+
+    describe "Return value" do
+      let(:middlewares) do
+        [build(:middleware, callee: lambda do |task, &block|
+                                      task.metadata.foobar = 42
+                                      block.call
+                                    end),
+         build(:middleware, callee: lambda do |task|
+                                      task.metadata.foobar *= 1337
+                                    end)]
+      end
+
+      it "is the last middleware's return value" do
+        expect(chain.call(task)).to be(56_154)
+      end
+    end
+
+    describe "Metadata" do
+      let(:first) do
+        build(:middleware, callee: lambda do |task, &block|
+                                     task.metadata.foobar = 42
+                                     block.call
+                                     task.metadata.barqux = 1337
+                                   end)
+      end
+
+      let(:last) do
+        build(:middleware, callee: lambda do |task|
+                                     raise unless task.metadata.foobar == 42
+                                     raise if task.metadata.barqux
+                                   end)
+      end
+
+      let(:middlewares) { [first, last] }
+
+      it "is accessible across middleware" do
+        expect {
+          chain.call(task)
+        }.not_to raise_error
+
+        expect(task.metadata.foobar).to be(42)
+        expect(task.metadata.barqux).to be(1337)
+      end
+    end
+  end
+end

data/spec/middleware/dedup_spec.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Dedup, redis: true do
+  let(:task) { build(:task) }
+  subject(:dedup) { Wayfarer::Middleware::Dedup.new }
+
+  before { task.metadata.staged_urls = SortedSet.new }
+
+  describe "#call" do
+    context "without staged URLs" do
+      it "does not raise" do
+        expect { dedup.call(task) }.not_to raise_error
+      end
+    end
+
+    context "with unseen URL" do
+      it "marks the URL as seen" do
+        expect {
+          dedup.call(task)
+        }.to change { task.barrier.seen?(task.url) }.to(true)
+      end
+
+      it "yields" do
+        expect { |spy| dedup.call(task, &spy) }.to yield_control
+      end
+    end
+
+    context "with seen URL" do
+      before { task.barrier.seen?(task.url) }
+
+      it "does not yield" do
+        expect { |spy| dedup.call(task, &spy) }.not_to yield_control
+      end
+    end
+
+    context "with exception raised" do
+      it "does not mark the URL as seen" do
+        begin
+          dedup.call(task) { raise }
+        rescue StandardError
+          nil
+        end
+        expect(task.barrier.seen?(task.url)).to be(false)
+      end
+
+      it "re-raises the exception" do
+        expect {
+          dedup.call(task) { raise }
+        }.to raise_error(RuntimeError)
+      end
+    end
+
+    describe "staged URL filtering" do
+      let(:seen_urls) { %w[https://yahoo.com https://google.com] }
+      let(:unseen_urls) { %w[https://w3c.org https://nasa.gov] }
+
+      before do
+        seen_urls.each do |url|
+          task.barrier.seen?(url)
+        end
+
+        [*seen_urls, *unseen_urls].each do |url|
+          task.metadata.staged_urls.add(url)
+        end
+      end
+
+      it "filters seen staged URLs" do
+        expect {
+          dedup.call(task)
+        }.to change { task.metadata.staged_urls }.to(SortedSet.new(unseen_urls))
+      end
+    end
+  end
+end

data/spec/middleware/fetch_spec.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Fetch do
+  let(:task) { build(:task) }
+  let(:adapter) { spy }
+  subject(:fetch) { Wayfarer::Middleware::Fetch.new }
+
+  before { allow(fetch.pool).to receive(:with).and_yield(adapter) }
+
+  describe "#call" do
+    it "assigns the task" do
+      expect {
+        fetch.call(task)
+      }.to change { fetch.task }.to(task)
+    end
+
+    it "assigns the adapter" do
+      expect {
+        fetch.call(task)
+      }.to change { task.metadata.adapter }.to(adapter)
+    end
+
+    it "retrieves the page" do
+      expect(adapter).to receive(:fetch).with(task.url)
+      fetch.call(task)
+    end
+
+    context "when encountering an HTTP redirect" do
+      let(:redirect_url) { "https://redirect.com" }
+
+      let(:result) do
+        Wayfarer::Networking::Result::Redirect.new(task.url, redirect_url)
+      end
+
+      before do
+        task.metadata.staged_urls = Set.new
+        allow(adapter).to receive(:fetch).with(task.url).and_return(result)
+      end
+
+      it "stages the redirect URL" do
+        expect {
+          fetch.call(task)
+        }.to change { task.metadata.staged_urls }.to([redirect_url])
+      end
+
+      it "does not yield" do
+        expect { |spy| fetch.call(task, &spy) }.not_to yield_control
+      end
+    end
+
+    context "when retrieval succeeds" do
+      let(:page) { build(:page) }
+      let(:result) { Wayfarer::Networking::Result::Success.new(page) }
+
+      before do
+        allow(adapter).to receive(:fetch).with(task.url).and_return(result)
+      end
+
+      it "assigns the page" do
+        expect {
+          fetch.call(task)
+        }.to change { task.metadata.page }.to(page)
+      end
+
+      it "yields" do
+        expect { |spy| fetch.call(task, &spy) }.to yield_control
+      end
+    end
+  end
+end

data/spec/middleware/normalize_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Normalize do
+  let(:task) { build(:task) }
+  subject(:normalize) { Wayfarer::Middleware::Normalize.new }
+
+  describe "#call" do
+    let(:urls) do
+      ["http://example.com/products?product_id=123",
+       "HTTP://EXAMPLE.COM/products/?product_id=123",
+       "http://example.com/products/?product_id=123",
+       "http://example.com/foo/../products?product_id=123"]
+    end
+
+    before { task.metadata.staged_urls = SortedSet.new(urls) }
+
+    it "yields" do
+      expect { |spy| normalize.call(task, &spy) }.to yield_control
+    end
+
+    it "normalizes and compacts URLs" do
+      normalize.call(task)
+      expect(task.metadata.staged_urls).to eq(SortedSet[urls.first])
+    end
+  end
+end

data/spec/middleware/router_spec.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Router do
+  let(:task) { build(:task, url: url) }
+  let(:klass) do
+    Class.new(Wayfarer::Base) do
+      route.to :index, host: "example.com", path: "/foobar"
+    end
+  end
+
+  subject(:router) { Wayfarer::Middleware::Router.new }
+
+  before { task.job = klass.new }
+
+  describe "#call" do
+    context "when URL matches a route" do
+      let(:url) { "https://example.com/foobar" }
+
+      it "yields" do
+        expect { |spy| router.call(task, &spy) }.to yield_control
+      end
+
+      it "assigns metadata" do
+        router.call(task)
+        expect(task.metadata.action).to be(:index)
+        expect(task.metadata.params).to eq({})
+      end
+    end
+
+    context "when URL does not match a route" do
+      let(:url) { "https://yahoo.com/foobar" }
+
+      it "does not yield" do
+        expect { |spy| router.call(task, &spy) }.not_to yield_control
+      end
+
+      it "does not assign metadata" do
+        router.call(task) {}
+        expect(task.metadata.action).to be(nil)
+        expect(task.metadata.params).to be(nil)
+      end
+    end
+  end
+end

data/spec/middleware/stage_spec.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Stage do
+  include ActiveJob::TestHelper
+
+  let(:task) { build(:task) }
+  subject(:stage) { Wayfarer::Middleware::Stage.new }
+
+  describe "#call" do
+    it "assigns staged URLs" do
+      stage.call(task) do
+        expect(task.metadata.staged_urls).to eq(SortedSet.new)
+      end
+    end
+
+    it "yields" do
+      expect { |spy| stage.call(task, &spy) }.to yield_control
+    end
+
+    describe "Task queuing" do
+      let(:klass) { Class.new(Wayfarer::Base) }
+      let(:urls) { %w[https://example.com https://w3c.org] }
+
+      before { task.job = klass.new }
+
+      it "enqueues tasks" do
+        expect(klass).to receive(:crawl_later).with(urls.first, batch: task.batch)
+        expect(klass).to receive(:crawl_later).with(urls.second, batch: task.batch)
+
+        stage.call(task) do
+          extend Wayfarer::Middleware::Stage::API
+          stage(urls)
+        end
+      end
+    end
+  end
+end

data/spec/middleware/worker_spec.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Middleware::Worker do
+  let(:task) { build(:task) }
+
+  before { task.metadata.action = :index }
+
+  let(:job) do
+    Class.new(Wayfarer::Base) do
+      def index
+        :foobar
+      end
+    end
+  end
+
+  describe "::route" do
+    it "returns a route" do
+      expect(job.route).to be_a(Wayfarer::Routing::Route)
+    end
+  end
+
+  describe "#call" do
+    it "assigns the task" do
+      worker = job.new
+      expect { worker.call(task) }.to change { worker.task }.to(task)
+    end
+
+    it "calls the action method" do
+      worker = job.new
+      expect(worker).to receive(:index)
+      worker.call(task)
+    end
+
+    it "yields" do
+      expect { |spy| job.new.call(task, &spy) }.to yield_control
+    end
+  end
+
+  describe "#chain" do
+    it "returns its middleware chain" do
+      worker = job.new
+      expect(worker.chain.middlewares).to eq([*Wayfarer.core_middleware, worker])
+    end
+  end
+
+  describe "#redis" do
+    it "yields a Redis connection" do
+      job.new.redis do |conn|
+        expect(conn).to be_a(Redis)
+      end
+    end
+  end
+
+  describe "#page" do
+    let(:page) { Object.new }
+    let(:worker) { job.new }
+
+    before do
+      task.metadata.page = page
+      worker.task = task
+    end
+
+    it "returns the page" do
+      expect(worker.page).to be(page)
+    end
+
+    context "with live keyword" do
+      let(:adapter) { spy }
+      let(:body) { "foobar" }
+      let(:live) { Object.new }
+
+      before do
+        task.metadata.adapter = adapter
+        allow(adapter).to receive(:live).with(page).and_return(live)
+      end
+
+      it "returns a live page" do
+        expect(worker.page(live: true)).to be(live)
+      end
+
+      it "replaces the existing page" do
+        expect {
+          worker.page(live: true)
+        }.to change { task.metadata.page }.to(live)
+      end
+    end
+  end
+end

data/spec/networking/adapter.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+RSpec.shared_examples "Network adapter" do |options|
+  describe "::renew_on" do
+    it "returns an Array" do
+      expect(adapter.class.renew_on).to be_an(Array)
+    end
+  end
+
+  describe "#fetch" do
+    it "returns Success" do
+      url = test_app_path("/hello_world")
+      result = adapter.fetch(url)
+      expect(result).to be_a(Wayfarer::Networking::Result::Success)
+      expect(result.page).to be_a(Wayfarer::Page)
+    end
+
+    it "sets the URL" do
+      url = test_app_path("/status_code/404")
+      page = adapter.fetch(url).page
+      expect(page.url).to eq test_app_path("/status_code/404")
+    end
+
+    it "sets the response body" do
+      url = test_app_path("/hello_world")
+      page = adapter.fetch(url).page
+      expect(page.body).to match(/Hello world!/)
+    end
+  end
+
+  describe "#live" do
+    context "when automating a browser" do
+      before { skip unless options[:browser] }
+
+      it "returns a live page" do
+        adapter.fetch(test_app_path("/hello_world"))
+
+        url = test_app_path("/body/foobar")
+        case adapter
+        when Wayfarer::Networking::Selenium then adapter.browser.navigate.to(url)
+        when Wayfarer::Networking::Ferrum then adapter.browser.goto(url)
+        end
+
+        page = adapter.live(nil)
+        expect(page.url).to eq(url)
+        expect(page.body).to match(/foobar/)
+        # Response headers and status code cannot be tested here, because
+        # Selenium does not support them.
+      end
+    end
+
+    context "when using plain HTTP" do
+      before { skip if options[:browser] }
+
+      it "returns the same page" do
+        page = build(:page)
+        expect(adapter.live(page)).to be(page)
+      end
+    end
+  end
+
+  describe "#body" do
+    it "returns the body" do
+      url = test_app_path("/hello_world")
+      adapter.fetch(url).page
+      expect(adapter.body).to match(/Hello world!/)
+    end
+  end
+
+  describe "#free" do
+    it "frees the browser" do
+      skip unless options[:browser]
+
+      expect {
+        adapter.free
+      }.to change { adapter.browser }.to(nil)
+    end
+
+    it "frees the Capybara session" do
+      skip unless options[:capybara]
+
+      adapter.capybara
+
+      expect {
+        adapter.free
+      }.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
+    end
+  end
+
+  describe "#renew" do
+    it "does not raise" do
+      expect { adapter.renew }.not_to raise_error
+    end
+
+    it "renews the browser" do
+      skip unless options[:browser]
+
+      expect {
+        adapter.renew
+      }.to(change { adapter.browser })
+    end
+
+    it "frees the Capybara session" do
+      skip unless options[:capybara]
+
+      adapter.capybara
+
+      expect {
+        adapter.renew
+      }.to change { adapter.instance_variable_get(:@capybara) }.to(nil)
+    end
+  end
+
+  describe "#capybara" do
+    before { skip unless options[:capybara] }
+
+    it "returns a Capybara driver" do
+      expect(adapter.capybara).to be_a(Capybara::Session)
+    end
+  end
+
+  describe "HTTP headers" do
+    before { Wayfarer.config.http_headers = { "User-Agent" => "Foobar" } }
+
+    it "uses configured HTTP headers" do
+      skip unless options[:request_headers]
+
+      url = test_app_path("/headers/HTTP_USER_AGENT")
+      page = adapter.fetch(url).page
+      expect(page.body).to match(/Foobar/)
+    end
+  end
+end

data/spec/networking/ferrum_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+require_relative "adapter"
+
+describe Wayfarer::Networking::Ferrum, ferrum: true do
+  subject(:adapter) { Wayfarer::Networking::Ferrum.new }
+
+  after { adapter.free }
+
+  include_examples "Network adapter", browser: true,
+                                      capybara: true,
+                                      request_headers: true
+
+  describe "#fetch" do
+    it "sets the status code" do
+      url = test_app_path("/status_code/418")
+      page = adapter.fetch(url).page
+      expect(page.status_code).to be(418)
+    end
+
+    it "sets response headers" do
+      url = test_app_path("/hello_world")
+      page = adapter.fetch(url).page
+      expect(page.headers["hello"]).to eq("world")
+    end
+  end
+end

data/spec/networking/healer_spec.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+
+describe Wayfarer::Networking::Healer do
+  let(:error) { Class.new(StandardError) }
+  let(:renew_on) { [error] }
+
+  let(:adapter) do
+    double.tap do |adapter|
+      allow(adapter.class).to receive(:renew_on).and_return([error])
+    end
+  end
+
+  subject(:healer) do
+    Wayfarer::Networking::Healer.new(adapter)
+  end
+
+  describe "#fetch" do
+    context "when proxied call raises renewing error" do
+      before do
+        allow(adapter).to receive(:fetch).and_raise(error)
+      end
+
+      it "renews its adapter and re-raises" do
+        expect(adapter).to receive(:renew)
+        expect {
+          healer.fetch(URI("https://example.com"))
+        }.to raise_error(error)
+      end
+    end
+
+    context "when proxied call raises non-renewing error" do
+      before do
+        allow(adapter).to receive(:fetch).and_raise(StandardError.new)
+      end
+
+      it "does not renew its adapter and intercept" do
+        expect(adapter).not_to receive(:renew)
+        expect {
+          healer.fetch(URI("https://example.com"))
+        }.to raise_error(StandardError)
+      end
+    end
+  end
+end

data/spec/networking/net_http_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "spec_helpers"
+require_relative "adapter"
+
+describe Wayfarer::Networking::NetHTTP do
+  subject(:adapter) { Wayfarer::Networking::NetHTTP.send(:new) }
+
+  after { adapter.free }
+
+  include_examples "Network adapter", browser: false,
+                                      capybara: false,
+                                      request_headers: true
+
+  describe "#fetch" do
+    it "sets the status code" do
+      url = test_app_path("/status_code/404")
+      page = adapter.fetch(url).page
+      expect(page.status_code).to be 404
+    end
+
+    it "sets response headers" do
+      url = test_app_path("/hello_world")
+      page = adapter.fetch(url).page
+      expect(page.headers["hello"]).to eq ["world"]
+    end
+
+    context "when encountering a redirect" do
+      it "returns a Redirect" do
+        url = test_app_path("/redirect?times=3")
+        result = adapter.fetch(url)
+        expect(result).to be_a Wayfarer::Networking::Result::Redirect
+        expect(result.redirect_url).to eq test_app_path("/redirect?times=2")
+      end
+    end
+  end
+end