wayfarer 0.0.3

data/spec/http_adapters/selenium_adapter_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::HTTPAdapters::SeleniumAdapter, selenium: true do
+  subject(:adapter) { Wayfarer::HTTPAdapters::SeleniumAdapter.new }
+  after { adapter.free }
+
+  describe "#fetch" do
+    it "returns a Page" do
+      uri = URI("http://0.0.0.0:9876/hello_world")
+      page = adapter.fetch(uri)
+      expect(page).to be_a Page
+    end
+
+    it "sets the correct URI" do
+      uri = URI("http://0.0.0.0:9876/status_code/404")
+      page = adapter.fetch(uri)
+      expect(page.uri.to_s).to eq "http://0.0.0.0:9876/status_code/404"
+    end
+
+    it "retrieves the correct HTTP status code" do
+      uri = URI("http://0.0.0.0:9876/status_code/404")
+      page = adapter.fetch(uri)
+      expect(page.status_code).to be 404
+    end
+
+    it "retrieves the correct response body" do
+      uri = URI("http://0.0.0.0:9876/hello_world")
+      page = adapter.fetch(uri)
+      expect(page.body).to match /Hello world!/
+    end
+
+    it "retrieves the correct response headers" do
+      uri = test_app("/hello_world")
+      page = adapter.fetch(uri)
+      expect(page.headers["hello"]).to eq ["world"]
+    end
+
+    context "when response is a redirect" do
+      it "follows the redirect" do
+        uri = test_app("/redirect?times=3")
+        page = adapter.fetch(uri)
+        expect(page.uri.to_s).to eq test_app("/redirect?times=0").to_s
+      end
+
+      context "when encountering a redirect loop" do
+        it "returns `nil` as HTTP status code" do
+          pending; fail
+        end
+      end
+    end
+  end
+end

data/spec/integration/callbacks_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer, integration: true do
+  describe "Callbacks" do
+    subject(:job) do
+      Class.new(Wayfarer::Job) do
+        config.reraise_exceptions = true
+      end
+    end
+
+    describe "ActiveJob-inherited callbacks" do
+      they "fire" do
+        job.class_eval do
+          before_perform { fail }
+        end
+
+        expect {
+          job.perform_now
+        }.to raise_error
+      end
+    end
+
+    describe "Own callbacks" do
+      describe "::before_crawl" do
+        it "fires" do
+          job.class_eval { before_crawl { fail } }
+
+          expect { job.perform_now }.to raise_error
+        end
+      end
+
+      describe "::after_crawl" do
+        it "fires" do
+          job.class_eval { after_crawl { fail } }
+
+          expect { job.perform_now }.to raise_error
+        end
+      end
+    end
+  end
+end

data/spec/integration/locals_spec.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer, integration: true do
+  describe "Locals" do
+    subject(:job) do
+      Class.new(Wayfarer::Job) do
+        config.reraise_exceptions = true
+      end
+    end
+
+    describe "Initial local values" do
+      let(:entry) { test_app("/hello_world") }
+
+      they "do not change" do
+        job.class_eval do
+          route.path "/hello_world", to: :foobar
+
+          let(:foo) { {} }
+
+          def foobar
+            foo[:bar] = :qux
+          end
+        end
+
+        expect {
+          job.perform_now(entry)
+        }.not_to change { job.locals[:foo] }
+      end
+
+      context "when declared with ::let" do
+        they "get replaced with thread-safe counterparts" do
+          job.class_eval do
+            route.path "/hello_world", to: :foobar
+
+            let(:hash) { {} }
+            let(:array) { [] }
+            let(:fixnum) { 123 }
+            let(:tru) { true }
+            let(:fuls) { false }
+
+            def foobar
+              fail unless hash.is_a? Concurrent::Hash
+              fail unless array.is_a? Concurrent::Array
+              fail unless fixnum.is_a? Concurrent::AtomicFixnum
+              fail unless tru.is_a? Concurrent::AtomicBoolean
+              fail unless fuls.is_a? Concurrent::AtomicBoolean
+            end
+          end
+
+          expect {
+            job.perform_now(entry)
+          }.not_to raise_error
+        end
+      end
+    end
+
+    describe "Locals" do
+      let(:entry) { test_app("/graph/index.html") }
+
+      they "can be mutated within actions" do
+        job.class_eval do
+          route.path "/graph/index.html", to: :index
+          route.path "/graph/details/a.html", to: :detail
+
+          let(:hash) { {} }
+
+          def index
+            hash[:foo] = 42
+            stage page.links
+          end
+
+          def detail
+            fail unless hash[:foo] == 42
+          end
+        end
+
+        expect {
+          job.perform_now(entry)
+        }.not_to raise_error
+      end
+
+      they "are accessible within callbacks" do
+        job.class_eval do
+          route.path "/graph/index.html", to: :index
+
+          let(:hash) { { foo: :bar} }
+
+          before_crawl do
+            fail unless hash[:foo] == :bar
+          end
+
+          after_crawl do
+            fail unless hash[:foo] == :qux
+          end
+
+          def index
+            hash[:foo] = :qux
+          end
+        end
+
+        expect { job.perform_now(entry) }.not_to raise_error
+      end
+    end
+  end
+end

data/spec/integration/peeking_spec.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer, integration: true do
+  subject(:job) do
+    Class.new(Wayfarer::Job) do
+      config.reraise_exceptions = true
+    end
+  end
+
+  let(:entry) { test_app("/graph/index.html") }
+
+  describe "Peeking" do
+    it "works" do
+      job.class_eval do
+        route.path "/graph/index.html", to: :index
+        route.path "/graph/details/a.html", to: :detail
+
+        def index
+          peek = yield "http://localhost:9876/graph/details/a.html"
+          fail unless peek == :ok
+        end
+
+        def detail
+          :ok
+        end
+      end
+
+      expect {
+        job.perform_now(entry)
+      }.not_to raise_error
+    end
+  end
+
+  describe "Recursive peeking" do
+    it "does not work" do
+      job.class_eval do
+        route.path "/graph/index.html", to: :index
+        route.path "/graph/details/a.html", to: :a
+        route.path "/graph/details/a.html", to: :b
+
+        def index
+          peek = yield "http://localhost:9876/graph/details/a.html"
+          fail unless peek == :ok
+        end
+
+        def a
+          yield "http://localhost:9876/graph/details/b.html" or :ok
+        end
+
+        def b
+          yield "http://localhost:9876/graph/details/a.html"
+        end
+      end
+
+      expect {
+        job.perform_now(entry)
+      }.not_to raise_error
+    end
+  end
+end

data/spec/job_spec.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Job do
+  subject(:job) { Class.new(Wayfarer::Job) }
+
+  describe "::prepare" do
+    it "copies the router" do
+      job.class_eval do
+        route.host "example.com", to: :foobar
+      end
+
+      prepared_job = job.prepare
+
+      expect(prepared_job.router).not_to be job.router
+
+      uri = URI("http://example.com")
+      expect(prepared_job.router.routes?(uri)).to be true
+    end
+
+    it "copies locals" do
+      job.class_eval do
+        let(:foo) { { bar: :qux } }
+        let(:baz) { 42 }
+      end
+
+      prepared_job = job.prepare
+
+      expect(prepared_job.locals).not_to be job.locals
+
+      expect(prepared_job.locals[:foo]).to be_a Concurrent::Hash
+      expect(prepared_job.locals[:baz]).to be_a Concurrent::AtomicFixnum
+    end
+
+    it "defines instance accessors for locals" do
+      job.class_eval do
+        let(:foo) { { bar: :qux } }
+        let(:baz) { 42 }
+      end
+
+      prepared_job = job.prepare
+
+      expect(prepared_job).to respond_to(:foo)
+      expect(prepared_job).to respond_to(:baz)
+    end
+  end
+
+  describe "::config" do
+    it "allows manipulating the configuration" do
+      expect {
+        job.config { |c| c.http_adapter = :selenium }
+      }.to change { job.config.http_adapter }.to(:selenium)
+    end
+
+    it "does not manipulate the global configuration" do
+      expect {
+        job.config { |c| c.http_adapter = :selenium }
+      }.not_to change { Wayfarer.config.http_adapter }
+    end
+  end
+
+  describe "#halt" do
+    it "sets the halting flag" do
+      job_instance = job.new
+
+      expect {
+        job_instance.send(:halt)
+      }.to change { job_instance.halts? }.from(false).to(true)
+    end
+  end
+
+  describe "#stage" do
+    it "stages URIs" do
+      job_instance = job.new
+      job_instance.page = Page.new(uri: URI("https://yahoo.com/qux"))
+
+      uris = %w(
+        http://google.com
+        http://example.com
+      )
+
+      expected = uris.map { |u| URI(u) }
+
+      expect {
+        job_instance.send(:stage, *uris)
+      }.to change { job_instance.staged_uris }.from([]).to(expected)
+    end
+
+    it "expands relative URIs" do
+      job_instance = job.new
+      job_instance.page = Page.new(uri: URI("https://yahoo.com/qux"))
+
+      uris = %w(
+        /foo/bar
+        bar/qux.html
+        barfoo
+      )
+
+      expected = %w(
+        https://yahoo.com/qux/foo/bar
+        https://yahoo.com/qux/bar/qux.html
+        https://yahoo.com/qux/barfoo
+      ).map { |u| URI(u) }
+
+      expect {
+        job_instance.send(:stage, *uris)
+      }.to change { job_instance.staged_uris }.from([]).to(expected)
+    end
+
+    context "with non-HTTP(s) URIs" do
+      it "ignores them and does not raise an exception" do
+        job_instance = job.new
+
+        expect {
+          job_instance.send(:stage, "ftp://example.com")
+        }.not_to raise_error
+
+        expect(job_instance.staged_uris).to be_empty
+      end
+    end
+  end
+end

data/spec/page_spec.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Page do
+  subject(:page) { fetch_page(test_app("/links/links.html")) }
+
+  describe "#doc" do
+    context "when Content-Type is HTML" do
+      it "returns a Nokogiri::HTML::Document" do
+        expect(page.doc).to be_a Nokogiri::HTML::Document
+      end
+    end
+
+    context "when Content-Type is XML" do
+      subject(:page) { fetch_page(test_app("/xml/dummy.xml")) }
+
+      it "returns a Nokogiri::XML::Document" do
+        expect(page.doc).to be_a Nokogiri::XML::Document
+      end
+    end
+
+    context "when Content-Type is JSON" do
+      subject(:page) { fetch_page(test_app("/json/dummy.json")) }
+
+      it "returns a Hash" do
+        expect(page.doc).to be_a Hash
+      end
+    end
+  end
+
+  describe "Forwarding" do
+    it "forwards calls to Pismo", mri_only: true do
+      Pismo::Document::ATTRIBUTE_METHODS.each do |method|
+        expect(page).to respond_to(method)
+      end
+    end
+  end
+end

data/spec/parsers/json_parser_spec.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Parsers::JSONParser do
+  subject(:parser) { Parsers::JSONParser }
+
+  describe ".parse" do
+    it "returns a Hash" do
+      json_str = <<-json
+        {
+          "id": 1,
+          "name": "Foo",
+          "price": 123,
+          "tags": [
+            "Bar",
+            "Eek"
+          ],
+          "stock": {
+            "warehouse": 300,
+            "retail": 20
+          }
+        }
+      json
+
+      doc = parser.parse(json_str)
+
+      expect(doc).to be_a Hash
+    end
+  end
+end

data/spec/parsers/xml_parser_spec.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Parsers::XMLParser do
+  subject(:parser) { Parsers::XMLParser }
+
+  describe ".parse_html" do
+    it "returns a Nokogiri::HTML::Document" do
+      html_str = "<span>Foobar</span>"
+      doc = parser.parse_html(html_str)
+
+      expect(doc).to be_a Nokogiri::HTML::Document
+    end
+  end
+
+  describe ".parse_xml" do
+    it "returns a Nokogiri::XML::Document" do
+      xml_str = "<barqux>Foobar</barqux>"
+      doc = parser.parse_xml(xml_str)
+
+      expect(doc).to be_a Nokogiri::XML::Document
+    end
+  end
+end

data/spec/processor_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Processor do
+  let(:job) { Class.new(Wayfarer::Job) }
+  let(:frontier) { Frontiers::MemoryFrontier.new(job.config) }
+  let(:dispatcher) { Dispatcher.new(job) }
+  subject(:processor) { Processor.new(job, frontier, dispatcher) }
+
+  describe "#halt!" do
+    it "sets a halting flag" do
+      processor.halt!
+      expect(processor).to be_halted
+    end
+  end
+
+  describe "#halted?" do
+    context "when not halted" do
+      it "returns false" do
+        expect(processor.halted?).to be false
+      end
+    end
+
+    context "when halted" do
+      it "returns true" do
+        processor.halt!
+        expect(processor.halted?).to be true
+      end
+    end
+  end
+end

data/spec/routing/custom_rule_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Routing::CustomRule do
+  let(:uri) { URI("http://example.com") }
+
+  describe "#matches?" do
+    context "with a block" do
+      context "when block is truthy" do
+        subject(:rule) { CustomRule.new -> (uri) { uri.is_a?(URI) } }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+
+      context "when block is fals-y" do
+        subject(:rule) { CustomRule.new -> (_) { false } }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+  end
+end

data/spec/routing/filetypes_rule_spec.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Routing::FiletypesRule do
+  subject(:rule) { FiletypesRule.new([:png, :js]) }
+
+  describe "#matches?" do
+    context "with matching URI" do
+      it "returns true" do
+        uris = %w(
+          http://example.com/foo.png
+          https://example.com/a/b.png/c.png
+          http://example.com/foobar.html.png
+          https://example.com/foo.png/bar.png
+        ).map { |u| URI(u) }
+
+        uris.each do |uri|
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+    end
+
+    context "with mismatching URI" do
+      it "returns false" do
+        uris = %w(
+          http://example.png
+          http://example.com.png
+          https://example.com/a/b.png/c
+          http://example.com.html
+          http://example.com/foobar.html
+          https://example.com/foo.png/bar.png.html
+        ).map { |u| URI(u) }
+
+        uris.each do |uri|
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+  end
+end

data/spec/routing/host_rule_spec.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Routing::HostRule do
+  subject(:rule) { HostRule.new(str_or_regexp) }
+
+  describe "#matches?" do
+    describe "String matching" do
+      let(:str_or_regexp) { "example.com" }
+
+      context "with matching URI" do
+        let(:uri) { URI("http://example.com/foo/bar") }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+
+      context "with mismatching URI" do
+        let(:uri) { URI("http://google.com/bar/qux") }
+
+        it "returns false" do
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+
+    describe "RegExp matching" do
+      let(:str_or_regexp) { /example.com/ }
+
+      context "with matching URI" do
+        let(:uri) { URI("http://sub.example.com") }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+
+      context "with mismatching URI" do
+        let(:uri) { URI("http://example.sub.com") }
+
+        it "returns false" do
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+  end
+end

data/spec/routing/path_rule_spec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Routing::PathRule, mri: true do
+  describe "#matches?" do
+    context "with a string" do
+      subject(:rule) { PathRule.new("/{alpha}/{beta}") }
+
+      context "with matching URI" do
+        let(:uri) { URI("http://example.com/foo/bar") }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+
+      context "with mismatching URI" do
+        let(:uri) { URI("http://example.com/foo") }
+
+        it "returns false" do
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+
+    context "with a RegExp" do
+      subject(:rule) { PathRule.new(/\/(\w+)\/(\w+)/) }
+
+      context "with matching URI" do
+        let(:uri) { URI("http://example.com/foo/bar") }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+
+      context "with mismatching URI" do
+        let(:uri) { URI("http://example.com/foo") }
+
+        it "returns false" do
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+  end
+
+  describe "#params" do
+    context "with a string" do
+      subject(:rule) { PathRule.new("/{alpha}/{beta}") }
+
+      it "returns the correct parameters" do
+        uri = URI("http://example.com/foo/bar")
+        expect(rule.params(uri)).to eq("alpha" => "foo", "beta" => "bar")
+      end
+    end
+
+    context "with a RegExp" do
+      subject(:rule) { PathRule.new(/\/(\w+)\/(\w+)/) }
+
+      it "returns the correct parameters" do
+        uri = URI("http://example.com/foo/bar")
+        expect(rule.params(uri)).to eq("0" => "foo", "1" => "bar")
+      end
+    end
+  end
+end