wayfarer 0.4.6 → 0.4.8

data/spec/middleware/fetch_spec.rb

@@ -1,155 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Middleware::Fetch do
-  let(:task) { build(:task) }
-  let(:page) { Object.new }
-  let(:agent) { Object.new }
-  let(:result) { Wayfarer::Networking::Result::Success.new(page) }
-  let(:context) do
-    double(instance: agent).tap do |context|
-      allow(context).to receive(:fetch).with(task.url).and_return(result)
-    end
-  end
-
-  subject { described_class.new }
-
-  describe "#call" do
-    let(:controller) do
-      Struct.new(:task).include(Wayfarer::Middleware::Stage::API).new(task)
-    end
-
-    before do
-      allow(controller).to receive(:run_callbacks).with(:fetch).and_yield
-
-      spy.tap do |pool|
-        allow(pool).to receive(:with).and_yield(context)
-        allow(subject).to receive(:pool).and_return(pool)
-      end
-
-      task.metadata.staged_urls = SortedSet.new
-      task.metadata.controller = controller
-    end
-
-    context "with page assigned" do
-      before { task.metadata.page = page }
-
-      it "does not alter the page" do
-        expect {
-          subject.call(task)
-        }.not_to(change { task.metadata.page })
-      end
-
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
-      end
-    end
-
-    it "runs callbacks" do
-      expect(controller).to receive(:run_callbacks).with(:fetch)
-      subject.call(task)
-    end
-
-    it "fetches the URL" do
-      expect(context).to receive(:fetch).with(task.url)
-      subject.call(task)
-    end
-
-    context "with Redirect" do
-      let(:redirect_url) { test_app_path("/foobar") }
-      let(:result) { Wayfarer::Networking::Result::Redirect.new(page) }
-
-      it "stages the redirect URL" do
-        expect {
-          subject.call(task)
-        }.to change { task.metadata.staged_urls.count }.by(1)
-      end
-
-      it "does not yield" do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
-      end
-    end
-
-    context "with Success" do
-      it "assigns the context" do
-        expect {
-          subject.call(task)
-        }.to change { task.metadata.context }.to(context)
-      end
-
-      it "assigns the page" do
-        expect {
-          subject.call(task)
-        }.to change { task.metadata.page }.to(result.page)
-      end
-
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
-      end
-    end
-  end
-
-  describe described_class::API do
-    subject(:controller) do
-      Struct.new(:task).include(described_class).new(task)
-    end
-
-    describe "#agent" do
-      before { task.metadata.context = context }
-
-      it "returns the agent" do
-        expect(controller.agent).to be(context.instance)
-      end
-    end
-
-    describe "#context" do
-      before { task.metadata.context = context }
-
-      it "returns the context" do
-        expect(controller.context).to be(task.metadata.context)
-      end
-    end
-
-    describe "#page" do
-      before { task.metadata.page = page }
-
-      it "returns the page" do
-        expect(controller.page).to be(task.metadata.page)
-      end
-
-      context "with live keyword" do
-        before { task.metadata.context = context }
-
-        context "with stateful agent" do
-          before do
-            result.page = Object.new
-            allow(context).to receive(:live).and_return(result)
-          end
-
-          it "replaces the page" do
-            expect {
-              controller.page(live: true)
-            }.to change { task.metadata.page }.to(result.page)
-          end
-        end
-
-        context "with stateless agent" do
-          before { allow(context).to receive(:live).and_return(nil) }
-
-          it "does not alter the page" do
-            expect {
-              controller.page(live: true)
-            }.not_to(change { task.metadata.page })
-          end
-        end
-      end
-    end
-
-    describe "#http" do
-      it "returns a redirect-following HTTP agent" do
-        expect(controller.http).to be_a(Wayfarer::Networking::Follow)
-      end
-    end
-  end
-end

data/spec/middleware/normalize_spec.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Middleware::Normalize do
-  let(:task) { build(:task) }
-  subject { described_class.new }
-
-  describe "#call" do
-    let(:urls) do
-      ["http://example.com/products?product_id=123",
-       "HTTP://EXAMPLE.COM/products/?product_id=123",
-       "http://example.com/products/?product_id=123",
-       "http://example.com/foo/../products?product_id=123",
-       "invalid@url-net"]
-    end
-
-    before { task.metadata.staged_urls = SortedSet.new(urls) }
-
-    it "yields" do
-      expect { |spy| subject.call(task, &spy) }.to yield_control
-    end
-
-    it "normalizes and compacts URLs" do
-      subject.call(task)
-      expect(task.metadata.staged_urls).to eq(SortedSet[urls.first])
-    end
-  end
-end

data/spec/middleware/router_spec.rb

@@ -1,105 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Middleware::Router do
-  let(:task) { build(:task) }
-  subject { described_class.new }
-
-  describe "#call" do
-    let(:controller) do
-      Struct.new(:task).include(described_class::API).new(task)
-    end
-
-    before do
-      allow(controller.class.router).to receive(:invoke)
-        .with(Addressable::URI.parse(task.url), controller.steer)
-        .and_return(result)
-      task.metadata.controller = controller
-    end
-
-    context "with matching route" do
-      let(:action) { :action }
-      let(:params) { { "foo" => "bar" } }
-      let(:result) do
-        Wayfarer::Routing::Result::Match.new(action, params)
-      end
-
-      it "assigns the action" do
-        expect {
-          subject.call(task)
-        }.to change { task.metadata.action }.to(action)
-      end
-
-      it "merges params" do
-        task.metadata.params = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
-
-        expect {
-          subject.call(task)
-        }.to change { task.metadata.params }.to("foo" => "bar", "bar" => "qux")
-      end
-
-      it "yields" do
-        expect { |spy| subject.call(task, &spy) }.to yield_control
-      end
-    end
-
-    context "without matching route" do
-      let(:result) do
-        Wayfarer::Routing::Result::Mismatch.new
-      end
-
-      it "does not assign the action" do
-        expect {
-          subject.call(task)
-        }.not_to(change { task.metadata.action })
-      end
-
-      it "does not alter params" do
-        task.metadata.params = ActiveSupport::HashWithIndifferentAccess.new("bar" => "qux")
-
-        expect {
-          subject.call(task)
-        }.not_to(change { task.metadata.params })
-      end
-
-      it "does not yield" do
-        expect { |spy| subject.call(task, &spy) }.not_to yield_control
-      end
-    end
-  end
-
-  describe described_class::API do
-    subject(:controller) do
-      Struct.new(:task).include(described_class).new(task)
-    end
-
-    describe "::router" do
-      it "returns a router" do
-        expect(controller.class.router).to be_a(Wayfarer::Routing::Router)
-      end
-    end
-
-    describe "::route" do
-      it "adds a routing block" do
-        expect {
-          controller.class.route { to :index }
-        }.to change { controller.class.router.blocks.count }.by(1)
-      end
-    end
-
-    describe "::steer" do
-      it "overrides #steer" do
-        expect {
-          controller.class.steer { :foobar }
-        }.to change { controller.steer }.from([]).to(:foobar)
-      end
-    end
-
-    describe "#steer" do
-      it "returns [] by default" do
-        expect(controller.steer).to eq([])
-      end
-    end
-  end
-end

data/spec/middleware/stage_spec.rb

@@ -1,62 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Middleware::Stage do
-  let(:task) { build(:task) }
-  subject { described_class.new }
-
-  describe "#call" do
-    it "assigns an empty set" do
-      subject.call(task)
-      expect(task.metadata.staged_urls).to eq(SortedSet.new)
-    end
-
-    it "yields" do
-      expect { |spy| subject.call(task, &spy) }.to yield_control
-    end
-
-    it "enqueues tasks" do
-      urls = [test_app_path("/alpha"), test_app_path("/beta")]
-
-      spy.tap do |job|
-        expect(job).to receive(:crawl).with(urls.first, batch: task.batch).ordered
-        expect(job).to receive(:crawl).with(urls.second, batch: task.batch).ordered
-        task.metadata.job = double(class: job)
-      end
-
-      subject.call(task) do
-        task.metadata.staged_urls = SortedSet.new(urls)
-      end
-    end
-
-    it "resets staged URLs" do
-      task.metadata.staged_urls = SortedSet.new([test_app_path("/foo")])
-
-      expect {
-        subject.call(task)
-      }.to change { task.metadata.staged_urls.count }.to(0)
-    end
-  end
-
-  describe described_class::API do
-    subject(:controller) do
-      Struct.new(:task).include(described_class).new(task)
-    end
-
-    describe "#stage" do
-      before { task.metadata.staged_urls = SortedSet.new }
-
-      it "stages URLs" do
-        expect {
-          controller.stage(test_app_path("/foo"))
-        }.to change { task.metadata.staged_urls.count }.by(1)
-      end
-
-      it "converts to strings" do
-        controller.stage(Addressable::URI.parse(test_app_path("/foo")))
-        expect(task.metadata.staged_urls.to_a.first).to be_a(String)
-      end
-    end
-  end
-end

data/spec/networking/capybara_spec.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-require_relative "strategy"
-
-describe Wayfarer::Networking::Capybara, ferrum: true do
-  include_examples "Network strategy", strategy: described_class,
-                                       browser: true,
-                                       request_headers: false,
-                                       response_headers: true,
-                                       status_code: true
-end

data/spec/networking/ferrum_spec.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-require_relative "strategy"
-
-describe Wayfarer::Networking::Ferrum, ferrum: true do
-  include_examples "Network strategy", strategy: described_class,
-                                       browser: true,
-                                       request_headers: true,
-                                       response_headers: true,
-                                       status_code: true
-end

data/spec/networking/http_spec.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-require_relative "strategy"
-
-describe Wayfarer::Networking::HTTP do
-  include_examples "Network strategy", strategy: described_class,
-                                       browser: false,
-                                       request_headers: true,
-                                       response_headers: true,
-                                       status_code: true
-end

data/spec/networking/selenium_spec.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-require_relative "strategy"
-
-describe Wayfarer::Networking::Selenium, selenium: true do
-  include_examples "Network strategy", strategy: described_class,
-                                       browser: true,
-                                       request_headers: false,
-                                       response_headers: false,
-                                       status_code: false
-end

data/spec/page_spec.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Page do
-  describe "#doc" do
-    context "when Content-Type is absent" do
-      subject(:page) { build(:page, :html, headers: {}) }
-
-      it "returns a Nokogiri::HTML::Document" do
-        expect(page.doc).to be_a(Nokogiri::HTML::Document)
-      end
-    end
-
-    context "when Content-Type is HTML" do
-      subject(:page) { build(:page, :html) }
-
-      it "returns a Nokogiri::HTML::Document" do
-        expect(page.doc).to be_a(Nokogiri::HTML::Document)
-      end
-    end
-
-    context "when Content-Type is XML" do
-      subject(:page) { build(:page, :xml) }
-
-      it "returns a Nokogiri::XML::Document" do
-        expect(page.doc).to be_a(Nokogiri::XML::Document)
-      end
-    end
-
-    context "when Content-Type is JSON" do
-      subject(:page) { build(:page, :json) }
-
-      it "returns a Hash" do
-        expect(page.doc).to be_a(Hash)
-      end
-    end
-  end
-
-  describe "#meta" do
-    subject(:page) { build(:page, :html) }
-
-    it "returns a MetaInspector document" do
-      expect(page.meta).to be_a(MetaInspector::Document)
-    end
-  end
-end

data/spec/parsing/xml_spec.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Parsing::XML do
-  subject(:parser) { Wayfarer::Parsing::XML }
-
-  describe ".parse_html" do
-    it "returns a Nokogiri::HTML::Document" do
-      html_str = "<span>Foobar</span>"
-      doc = parser.parse_html(html_str)
-
-      expect(doc).to be_a Nokogiri::HTML::Document
-    end
-  end
-
-  describe ".parse_xml" do
-    it "returns a Nokogiri::XML::Document" do
-      xml_str = "<barqux>Foobar</barqux>"
-      doc = parser.parse_xml(xml_str)
-
-      expect(doc).to be_a Nokogiri::XML::Document
-    end
-  end
-end

data/spec/redis/barrier_spec.rb

@@ -1,78 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Redis::Barrier, redis: true do
-  let(:url) { "https://example.com" }
-  subject(:barrier) { Wayfarer::Redis::Barrier.new("batch") }
-
-  describe "#redis_key" do
-    it "returns the expected Redis key" do
-      expect(barrier.redis_key).to eq("wayfarer-barrier-batch")
-    end
-  end
-
-  describe "#reset!" do
-    it "resets seen URLs" do
-      barrier.seen?(url)
-      barrier.reset!
-      expect(barrier.seen?(url)).to be false
-    end
-  end
-
-  describe "#seen?" do
-    context "with seen URL" do
-      before { barrier.seen?(url) }
-
-      it "returns true" do
-        expect(barrier.seen?(url)).to be(true)
-      end
-    end
-
-    context "with unseen URL" do
-      it "returns false" do
-        expect(barrier.seen?(url)).to be(false)
-      end
-    end
-  end
-
-  describe "#peek" do
-    let(:seen_urls) { %w[https://yahoo.com https://google.com] }
-    let(:unseen_urls) { %w[https://w3c.org https://nasa.gov] }
-
-    before do
-      allow(Wayfarer::Redis::Version).to receive(:determine).and_return(redis_version)
-
-      seen_urls.each do |url|
-        barrier.seen?(url)
-      end
-    end
-
-    context "Redis >= 6.2.0" do
-      let(:redis_version) { [6, 2, 5] }
-
-      it "returns membership" do
-        urls = [*seen_urls, *unseen_urls]
-        expect(barrier.peek(urls)).to eq([true, true, false, false])
-      end
-    end
-
-    context "Redis < 6.2.0" do
-      let(:redis_version) { [2, 6, 10] }
-
-      it "returns membership" do
-        urls = [*seen_urls, *unseen_urls]
-        expect(barrier.peek(urls)).to eq([true, true, false, false])
-      end
-    end
-  end
-
-  describe "#unsee" do
-    it "removes a seen URL" do
-      barrier.seen?(url)
-      expect {
-        barrier.unsee(url)
-      }.to change { barrier.seen?(url) }.to(false)
-    end
-  end
-end

data/spec/redis/counter_spec.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Redis::Counter, redis: true do
-  subject(:counter) { Wayfarer::Redis::Counter.new("batch") }
-
-  describe "#redis_key" do
-    it "returns the expected Redis key" do
-      expect(counter.redis_key).to eq("wayfarer-counter-batch")
-    end
-  end
-
-  describe "#increment" do
-    it "increments" do
-      expect { counter.increment }.to change { counter.value }.by(1)
-    end
-  end
-
-  describe "#decrement" do
-    it "decrements" do
-      expect { counter.decrement }.to change { counter.value }.by(-1)
-    end
-  end
-
-  describe "#reset!" do
-    it "resets" do
-      3.times { counter.increment }
-      expect { counter.reset! }.to change { counter.value }.to(0)
-    end
-  end
-end

data/spec/redis/version_spec.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-require "spec_helpers"
-
-describe Wayfarer::Redis::Version, redis: true do
-  describe "::determine" do
-    it "returns Redis major, minor, patch version" do
-      Wayfarer::Redis::Version.instance_variable_set(:@determine, nil)
-      expect(Wayfarer::Redis::Version).to receive(:server_version).and_return("1.20.300")
-      expect(Wayfarer::Redis::Version.determine).to eq([1, 20, 300])
-    end
-  end
-end