wayfarer-jruby 0.0.1

data/lib/wayfarer/routing/rule.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Wayfarer
+  module Routing
+    # Tree nodes
+    # @private
+    class Rule
+      include Enumerable
+
+      extend Forwardable
+
+      attr_reader :child_rules
+      attr_reader :target_action
+
+      delegate [:each] => :child_rules
+
+      def initialize(opts = {}, &proc)
+        @child_rules = []
+        @target_action = nil
+
+        build_child_rule_chain_from_options(opts)
+        instance_eval(&proc) if block_given?
+      end
+
+      def build_child_rule_chain_from_options(opts)
+        @target_action = opts.delete(:to)
+        opts.reduce(self) { |rule, (key, val)| rule.send(key, val) }
+      end
+
+      def matches?(uri)
+        return false unless match!(uri)
+        none? || any? { |child_rule| child_rule.matches?(uri) }
+      end
+
+      def invoke(uri)
+        rule_chain = matching_rule_chain(uri)
+
+        if rule_chain.any?
+          params = params_from_rule_chain(rule_chain, uri)
+          action = action_from_rule_chain(rule_chain)
+
+          [true, params, action]
+        else
+          false
+        end
+      end
+
+      # rubocop:disable Lint/AssignmentInCondition
+      def matching_rule_chain(uri, chain = [])
+        if match!(uri) && none?
+          chain << self
+        elsif matching_child = detect { |child_rule| child_rule.matches?(uri) }
+          matching_child.matching_rule_chain(uri, chain << self)
+        else
+          []
+        end
+      end
+      # rubocop:enable Lint/AssignmentInCondition
+
+      def params(*)
+        {}
+      end
+
+      def uri(*argv, &proc)
+        append_child_rule(URIRule.new(*argv, &proc))
+      end
+
+      def host(*argv, &proc)
+        append_child_rule(HostRule.new(*argv, &proc))
+      end
+
+      def path(*argv, &proc)
+        append_child_rule(PathRule.new(*argv, &proc))
+      end
+
+      def query(*argv, &proc)
+        append_child_rule(QueryRule.new(*argv, &proc))
+      end
+
+      private
+
+      def append_child_rule(other)
+        @child_rules << other
+        other
+      end
+
+      def match!(*)
+        any?
+      end
+
+      def params_from_rule_chain(rule_chain, uri)
+        rule_chain.reduce({}) { |hash, rule| hash.merge(rule.params(uri)) }
+      end
+
+      def action_from_rule_chain(rule_chain)
+        rule_chain.map(&:target_action).reverse.reject(&:nil?).first
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/uri_rule.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Wayfarer
+  module Routing
+    # @private
+    class URIRule < Rule
+      def initialize(uri_str, opts = {}, &proc)
+        @uri = URI(uri_str)
+        super(opts, &proc)
+      end
+
+      private
+
+      def match!(uri)
+        uri == @uri
+      end
+    end
+  end
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Configuration do
+  let(:config) { Configuration.new }
+
+  describe "::new" do
+    it "overrides defaults" do
+      config = Configuration.new(http_adapter: :selenium)
+      expect(config.http_adapter).to be :selenium
+    end
+  end
+
+  it "sets keys and values" do
+    config.foo = :foo
+    expect(config.foo).to be :foo
+  end
+
+  describe "#reset!" do
+    it "resets to defaults" do
+      config.max_http_redirects = 5
+      config.reset!
+      expect(config.max_http_redirects).to be 3
+    end
+  end
+end

data/spec/crawl_spec.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Crawl do
+  let(:uris) { %w(http://example.com) }
+  let(:job) { Class.new(Wayfarer::Job) }
+  subject(:crawl) { Crawl.new(job, *uris) }
+
+  describe "#frontier" do
+    context "by default" do
+      it "returns a MemoryFrontier" do
+        expect(crawl.frontier).to be_a MemoryFrontier
+      end
+    end
+
+    context "when Redis frontier is used", redis: true do
+      before { job.config.frontier = :redis }
+
+      it "returns a MemoryFrontier" do
+        expect(crawl.frontier).to be_a RedisFrontier
+      end
+    end
+
+    context "when memory bloomfilter is used", mri_only: true do
+      before { job.config.frontier = :memory_bloom }
+
+      it "returns a MemoryBloomfilter" do
+        expect(crawl.frontier).to be_a MemoryBloomfilter
+      end
+    end
+
+    context "when Redis bloomfilter is used", mri_only: true, redis: true do
+      before { job.config.frontier = :redis_bloom }
+
+      it "returns a RedisBloomfilter" do
+        expect(crawl.frontier).to be_a RedisBloomfilter
+      end
+    end
+
+    context "when memory trie is used", mri_only: true do
+      before { job.config.frontier = :memory_trie }
+
+      it "returns a RedisBloomfilter" do
+        expect(crawl.frontier).to be_a MemoryTrieFrontier
+      end
+    end
+  end
+end

data/spec/finders_spec.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Finders do
+  let(:page) { fetch_page(test_app("/finders.html")) }
+
+  describe "#links" do
+    context "without paths" do
+      it "returns all links" do
+        expect(page.links.map(&:to_s)).to eq %w(
+          http://localhost:9876/foo
+          http://localhost:9876/bar
+          http://localhost:9876/baz
+          http://google.com
+          http://yahoo.com
+          http://aol.com
+        )
+      end
+    end
+
+    context "with paths" do
+      it "returns targeted links" do
+        expect(page.links("ul li:nth-child(3) a").map(&:to_s)).to eq %w(
+          http://localhost:9876/baz
+        )
+      end
+    end
+  end
+
+  describe "#stylesheets" do
+    context "without paths" do
+      it "returns all stylesheets" do
+        expect(page.stylesheets.map(&:to_s)).to eq %w(
+          http://localhost:9876/a.css
+          http://localhost:9876/b.css
+          http://google.com/c.css
+        )
+      end
+    end
+
+    context "with paths" do
+      it "returns targeted stylesheets" do
+        expect(page.stylesheets("#stylesheet-c").map(&:to_s)).to eq %w(
+          http://google.com/c.css
+        )
+      end
+    end
+  end
+end

data/spec/frontiers/memory_bloomfilter_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::MemoryBloomfilter, mri_only: true do
+  it_behaves_like "Frontier", MemoryBloomfilter
+end

data/spec/frontiers/memory_frontier_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::MemoryFrontier do
+  it_behaves_like "Frontier", MemoryFrontier
+end

data/spec/frontiers/memory_trie_frontier_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::MemoryTrieFrontier, mri_only: true do
+  it_behaves_like "Frontier", MemoryTrieFrontier
+end

data/spec/frontiers/normalize_uris_spec.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::NormalizeURIs, mri_only: true do
+  let(:config) { Wayfarer.config }
+
+  let(:uris) do
+    %w(
+      http://example.com/foo?bar=1&zet=0
+      http://example.com//foo?zet=0&bar=1
+      http://example.com/foo?bar=1&zet=0#bar
+    )
+  end
+
+  subject(:frontier) do
+    MemoryFrontier.new(config).extend(NormalizeURIs)
+  end
+
+  describe "#stage" do
+    it "normalizes URIs" do
+      expect {
+        frontier.stage(*uris)
+      }.to change { frontier.staged_uris.count }.by(1)
+    end
+  end
+
+  describe "#staged?" do
+    it "normalizes URIs" do
+      frontier.stage(*uris)
+      uris.each { |u| expect(frontier.staged?(u)).to be true }
+    end
+  end
+
+  describe "#cache, #cached?" do
+    they "normalize URIs" do
+      frontier.cache(*uris)
+      uris.each { |u| expect(frontier.cached?(u)).to be true }
+    end
+  end
+
+  context "with normalization options" do
+    let(:config) do
+      Configuration.new(normalize_uri_options: { remove_hash: false })
+    end
+
+    let(:uris) do
+      %w(
+        http://example.com/foo
+        http://example.com/foo#bar
+      )
+    end
+
+    it "adheres to options" do
+      expect {
+        frontier.stage(*uris)
+      }.to change { frontier.staged_uris.count }.by(2)
+    end
+  end
+end

data/spec/frontiers/redis_bloomfilter_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::RedisBloomfilter, redis: true, mri_only: true do
+  it_behaves_like "Frontier", RedisBloomfilter
+end

data/spec/frontiers/redis_frontier_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::RedisFrontier, redis: true do
+  it_behaves_like "Frontier", RedisFrontier
+end

data/spec/http_adapters/adapter_pool_spec.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::HTTPAdapters::AdapterPool do
+  subject(:adapter_pool) { AdapterPool.new(Wayfarer::Job) }
+
+  describe "#with" do
+    after { adapter_pool.free }
+
+    context "when using Net::HTTP" do
+      it "yields a NetHTTPAdapter" do
+        adapter_pool.with do |adapter|
+          expect(adapter).to be_a NetHTTPAdapter
+        end
+      end
+    end
+
+    context "when using Selenium", selenium: true do
+      subject(:adapter_pool) do
+        job = Class.new(Wayfarer::Job)
+        job.config.http_adapter = :selenium
+
+        AdapterPool.new(job)
+      end
+
+      it "yields a SeleniumAdapter" do
+        adapter_pool.with do |adapter|
+          expect(adapter).to be_a SeleniumAdapter
+        end
+      end
+    end
+  end
+end

data/spec/http_adapters/net_http_adapter_spec.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::HTTPAdapters::NetHTTPAdapter do
+  subject(:adapter) { Wayfarer::HTTPAdapters::NetHTTPAdapter.instance }
+
+  describe "#fetch" do
+    it "returns a Page" do
+      uri = test_app("/hello_world")
+      page = adapter.fetch(uri)
+      expect(page).to be_a Page
+    end
+
+    it "sets the correct URI" do
+      uri = test_app("/status_code/404")
+      page = adapter.fetch(uri)
+      expect(page.uri).to eq test_app("/status_code/404")
+    end
+
+    it "retrieves the correct HTTP status code" do
+      uri = test_app("/status_code/404")
+      page = adapter.fetch(uri)
+      expect(page.status_code).to be 404
+    end
+
+    it "retrieves the correct response body" do
+      uri = test_app("/hello_world")
+      page = adapter.fetch(uri)
+      expect(page.body).to eq "Hello world!"
+    end
+
+    it "retrieves the correct response headers" do
+      uri = test_app("/hello_world")
+      page = adapter.fetch(uri)
+      expect(page.headers["hello"]).to eq ["world"]
+    end
+
+    context "with malformed URI" do
+      it "raises a MalformedURI" do
+        expect do
+          uri = URI("hptt://bro.ken")
+          page = adapter.fetch(uri)
+        end.to raise_error(
+          Wayfarer::HTTPAdapters::NetHTTPAdapter::MalformedURI
+        )
+      end
+    end
+
+    context "when response is a redirect" do
+      it "follows the redirect" do
+        uri = test_app("/redirect?times=3")
+        page = adapter.fetch(uri)
+
+        expect(page.uri.to_s).to eq "http://localhost:9876/redirect?times=0"
+      end
+
+      context "when maximum number of redirects reached" do
+        before { Wayfarer.config.max_http_redirects = 5 }
+        after  { Wayfarer.config.reset! }
+
+        it "raises a MaximumRedirectCountReached" do
+          expect do
+            uri = test_app("/redirect?times=6")
+            page = adapter.fetch(uri)
+          end.to raise_error(
+            Wayfarer::HTTPAdapters::NetHTTPAdapter::MaximumRedirectCountReached
+          )
+        end
+      end
+
+      context "when redirection URI is malformed" do
+        it "raises a MalformedRedirectURI" do
+          expect do
+            uri = test_app("/malformed_redirect")
+            page = adapter.fetch(uri)
+          end.to raise_error(
+            Wayfarer::HTTPAdapters::NetHTTPAdapter::MalformedRedirectURI
+          )
+        end
+      end
+    end
+  end
+end