wayfarer-jruby 0.0.1

data/spec/http_adapters/selenium_adapter_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::HTTPAdapters::SeleniumAdapter, selenium: true do
+  subject(:adapter) { Wayfarer::HTTPAdapters::SeleniumAdapter.new }
+  after { adapter.free }
+
+  describe "#fetch" do
+    it "returns a Page" do
+      uri = URI("http://0.0.0.0:9876/hello_world")
+      page = adapter.fetch(uri)
+      expect(page).to be_a Page
+    end
+
+    it "sets the correct URI" do
+      uri = URI("http://0.0.0.0:9876/status_code/404")
+      page = adapter.fetch(uri)
+      expect(page.uri.to_s).to eq "http://0.0.0.0:9876/status_code/404"
+    end
+
+    it "retrieves the correct HTTP status code" do
+      uri = URI("http://0.0.0.0:9876/status_code/404")
+      page = adapter.fetch(uri)
+      expect(page.status_code).to be 404
+    end
+
+    it "retrieves the correct response body" do
+      uri = URI("http://0.0.0.0:9876/hello_world")
+      page = adapter.fetch(uri)
+      expect(page.body).to match /Hello world!/
+    end
+
+    it "retrieves the correct response headers" do
+      uri = test_app("/hello_world")
+      page = adapter.fetch(uri)
+      expect(page.headers["hello"]).to eq ["world"]
+    end
+
+    context "when response is a redirect" do
+      it "follows the redirect" do
+        uri = test_app("/redirect?times=3")
+        page = adapter.fetch(uri)
+        expect(page.uri.to_s).to eq test_app("/redirect?times=0").to_s
+      end
+
+      context "when encountering a redirect loop" do
+        it "returns `nil` as HTTP status code" do
+          pending; fail
+        end
+      end
+    end
+  end
+end

data/spec/integration/callbacks_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer, integration: true do
+  describe "Callbacks" do
+    subject(:job) do
+      Class.new(Wayfarer::Job) do
+        config.reraise_exceptions = true
+      end
+    end
+
+    describe "ActiveJob-inherited callbacks" do
+      they "fire" do
+        job.class_eval do
+          before_perform { fail }
+        end
+
+        expect {
+          job.perform_now
+        }.to raise_error
+      end
+    end
+
+    describe "Own callbacks" do
+      describe "::before_crawl" do
+        it "fires" do
+          job.class_eval { before_crawl { fail } }
+
+          expect { job.perform_now }.to raise_error
+        end
+      end
+
+      describe "::after_crawl" do
+        it "fires" do
+          job.class_eval { after_crawl { fail } }
+
+          expect { job.perform_now }.to raise_error
+        end
+      end
+    end
+  end
+end

data/spec/integration/locals_spec.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer, integration: true do
+  describe "Locals" do
+    subject(:job) do
+      Class.new(Wayfarer::Job) do
+        config.reraise_exceptions = true
+      end
+    end
+
+    describe "Initial local values" do
+      let(:entry) { test_app("/hello_world") }
+
+      they "do not change" do
+        job.class_eval do
+          route.path "/hello_world", to: :foobar
+
+          let(:foo) { {} }
+
+          def foobar
+            foo[:bar] = :qux
+          end
+        end
+
+        expect {
+          job.perform_now(entry)
+        }.not_to change { job.locals[:foo] }
+      end
+
+      context "when declared with ::let" do
+        they "get replaced with thread-safe counterparts" do
+          job.class_eval do
+            route.path "/hello_world", to: :foobar
+
+            let(:hash) { {} }
+            let(:array) { [] }
+            let(:fixnum) { 123 }
+            let(:tru) { true }
+            let(:fuls) { false }
+
+            def foobar
+              fail unless hash.is_a? Concurrent::Hash
+              fail unless array.is_a? Concurrent::Array
+              fail unless fixnum.is_a? Concurrent::AtomicFixnum
+              fail unless tru.is_a? Concurrent::AtomicBoolean
+              fail unless fuls.is_a? Concurrent::AtomicBoolean
+            end
+          end
+
+          expect {
+            job.perform_now(entry)
+          }.not_to raise_error
+        end
+      end
+    end
+
+    describe "Locals" do
+      let(:entry) { test_app("/graph/index.html") }
+
+      they "can be mutated within actions" do
+        job.class_eval do
+          route.path "/graph/index.html", to: :index
+          route.path "/graph/details/a.html", to: :detail
+
+          let(:hash) { {} }
+
+          def index
+            hash[:foo] = 42
+            stage page.links
+          end
+
+          def detail
+            fail unless hash[:foo] == 42
+          end
+        end
+
+        expect {
+          job.perform_now(entry)
+        }.not_to raise_error
+      end
+
+      they "are accessible within callbacks" do
+        job.class_eval do
+          route.path "/graph/index.html", to: :index
+
+          let(:hash) { { foo: :bar} }
+
+          before_crawl do
+            fail unless hash[:foo] == :bar
+          end
+
+          after_crawl do
+            fail unless hash[:foo] == :qux
+          end
+
+          def index
+            hash[:foo] = :qux
+          end
+        end
+
+        expect { job.perform_now(entry) }.not_to raise_error
+      end
+    end
+  end
+end

data/spec/job_spec.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Job do
+  subject(:job) { Class.new(Wayfarer::Job) }
+
+  describe "::prepare" do
+    it "copies the router" do
+      job.class_eval do
+        route.host "example.com", to: :foobar
+      end
+
+      prepared_job = job.prepare
+
+      expect(prepared_job.router).not_to be job.router
+
+      uri = URI("http://example.com")
+      expect(prepared_job.router.routes?(uri)).to be true
+    end
+
+    it "copies locals" do
+      job.class_eval do
+        let(:foo) { { bar: :qux } }
+        let(:baz) { 42 }
+      end
+
+      prepared_job = job.prepare
+
+      expect(prepared_job.locals).not_to be job.locals
+
+      expect(prepared_job.locals[:foo]).to be_a Concurrent::Hash
+      expect(prepared_job.locals[:baz]).to be_a Concurrent::AtomicFixnum
+    end
+
+    it "defines instance accessors for locals" do
+      job.class_eval do
+        let(:foo) { { bar: :qux } }
+        let(:baz) { 42 }
+      end
+
+      prepared_job = job.prepare
+
+      expect(prepared_job).to respond_to(:foo)
+      expect(prepared_job).to respond_to(:baz)
+    end
+  end
+
+  describe "::config" do
+    it "allows manipulating the configuration" do
+      expect {
+        job.config { |c| c.http_adapter = :selenium }
+      }.to change { job.config.http_adapter }.to(:selenium)
+    end
+
+    it "does not manipulate the global configuration" do
+      expect {
+        job.config { |c| c.http_adapter = :selenium }
+      }.not_to change { Wayfarer.config.http_adapter }
+    end
+  end
+
+  describe "#halt" do
+    it "sets the halting flag" do
+      job_instance = job.new
+
+      expect {
+        job_instance.send(:halt)
+      }.to change { job_instance.halts? }.from(false).to(true)
+    end
+  end
+
+  describe "#stage" do
+    it "stages URIs" do
+      job_instance = job.new
+
+      uris = %w(
+        http://google.com
+        http://example.com
+      )
+
+      expect {
+        job_instance.send(:stage, *uris)
+      }.to change { job_instance.staged_uris }.from([]).to(uris)
+    end
+  end
+end

data/spec/page_spec.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Page do
+  subject(:page) { fetch_page(test_app("/links/links.html")) }
+
+  describe "#doc" do
+    context "when Content-Type is HTML" do
+      it "returns a Nokogiri::HTML::Document" do
+        expect(page.doc).to be_a Nokogiri::HTML::Document
+      end
+    end
+
+    context "when Content-Type is XML" do
+      subject(:page) { fetch_page(test_app("/xml/dummy.xml")) }
+
+      it "returns a Nokogiri::XML::Document" do
+        expect(page.doc).to be_a Nokogiri::XML::Document
+      end
+    end
+
+    context "when Content-Type is JSON" do
+      subject(:page) { fetch_page(test_app("/json/dummy.json")) }
+
+      it "returns a Hash" do
+        expect(page.doc).to be_a Hash
+      end
+    end
+  end
+
+  describe "Forwarding" do
+    it "forwards calls to Pismo", mri_only: true do
+      Pismo::Document::ATTRIBUTE_METHODS.each do |method|
+        expect(page).to respond_to(method)
+      end
+    end
+  end
+end

data/spec/parsers/json_parser_spec.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Parsers::JSONParser do
+  subject(:parser) { Parsers::JSONParser }
+
+  describe ".parse" do
+    it "returns a Hash" do
+      json_str = <<-json
+        {
+          "id": 1,
+          "name": "Foo",
+          "price": 123,
+          "tags": [
+            "Bar",
+            "Eek"
+          ],
+          "stock": {
+            "warehouse": 300,
+            "retail": 20
+          }
+        }
+      json
+
+      doc = parser.parse(json_str)
+
+      expect(doc).to be_a Hash
+    end
+  end
+end

data/spec/parsers/xml_parser_spec.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Parsers::XMLParser do
+  subject(:parser) { Parsers::XMLParser }
+
+  describe ".parse_html" do
+    it "returns a Nokogiri::HTML::Document" do
+      html_str = "<span>Foobar</span>"
+      doc = parser.parse_html(html_str)
+
+      expect(doc).to be_a Nokogiri::HTML::Document
+    end
+  end
+
+  describe ".parse_xml" do
+    it "returns a Nokogiri::XML::Document" do
+      xml_str = "<barqux>Foobar</barqux>"
+      doc = parser.parse_xml(xml_str)
+
+      expect(doc).to be_a Nokogiri::XML::Document
+    end
+  end
+end

data/spec/processor_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Processor do
+  let(:job) { Class.new(Wayfarer::Job) }
+  let(:frontier) { Frontiers::MemoryFrontier.new(job.config) }
+  let(:dispatcher) { Dispatcher.new(job) }
+  subject(:processor) { Processor.new(job, frontier, dispatcher) }
+
+  describe "#halt!" do
+    it "sets a halting flag" do
+      processor.halt!
+      expect(processor).to be_halted
+    end
+  end
+
+  describe "#halted?" do
+    context "when not halted" do
+      it "returns false" do
+        expect(processor.halted?).to be false
+      end
+    end
+
+    context "when halted" do
+      it "returns true" do
+        processor.halt!
+        expect(processor.halted?).to be true
+      end
+    end
+  end
+end

data/spec/routing/host_rule_spec.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Routing::HostRule do
+  subject(:rule) { HostRule.new(str_or_regexp) }
+
+  describe "#matches?" do
+    describe "String matching" do
+      let(:str_or_regexp) { "example.com" }
+
+      context "with matching URI" do
+        let(:uri) { URI("http://example.com/foo/bar") }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+
+      context "with mismatching URI" do
+        let(:uri) { URI("http://google.com/bar/qux") }
+
+        it "returns false" do
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+
+    describe "RegExp matching" do
+      let(:str_or_regexp) { /example.com/ }
+
+      context "with matching URI" do
+        let(:uri) { URI("http://sub.example.com") }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+
+      context "with mismatching URI" do
+        let(:uri) { URI("http://example.sub.com") }
+
+        it "returns false" do
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+  end
+end