wayfarer 0.0.3

data/lib/wayfarer/routing/protocol_rule.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Wayfarer
+  module Routing
+    # @private
+    class ProtocolRule < Rule
+      def initialize(protocol, opts = {}, &proc)
+        @protocol = protocol.to_s
+        super(opts, &proc)
+      end
+
+      private
+
+      def match!(uri)
+        uri.scheme == @protocol
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/query_rule.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "cgi"
+
+module Wayfarer
+  module Routing
+    # @private
+    class QueryRule < Rule
+      def initialize(field_constraints, opts = {}, &proc)
+        @field_constraints = field_constraints
+        super(opts, &proc)
+      end
+
+      private
+
+      def match!(uri)
+        CGI.parse(uri.query).none? do |field, vals| violates?(field, vals) end
+      rescue NoMethodError
+        # `CGI::parse` throws a `NoMethodError` if `uri.query` is an empty
+        # string
+        false
+      end
+
+      # rubocop:disable Lint/AssignmentInCondition
+      def violates?(field, vals)
+        return false unless constraint = @field_constraints[field.to_sym]
+        violates_constraint?(constraint, vals)
+      end
+      # rubocop:enable Lint/AssignmentInCondition
+
+      def violates_constraint?(constraint, vals)
+        case constraint
+        when String  then violates_string?(constraint, vals)
+        when Integer then violates_integer?(constraint, vals)
+        when Regexp  then violates_regexp?(constraint, vals)
+        when Range   then violates_range?(constraint, vals)
+        end
+      end
+
+      def violates_string?(str, vals)
+        vals.none? { |val| str == val }
+      end
+
+      def violates_integer?(int, vals)
+        vals.none? do |val| int == Integer(val) end
+      rescue ArgumentError
+        true
+      end
+
+      def violates_regexp?(regexp, vals)
+        vals.none? { |val| regexp.match(val) }
+      end
+
+      def violates_range?(range, vals)
+        vals.none? { |val| range.include?(val.to_i) }
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/router.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Wayfarer
+  module Routing
+    # A {Router} maps URIs onto a {Job}'s instance methods.
+    class Router
+      extend Forwardable
+
+      # @!attribute [r] rule
+      # @return [Rule]
+      attr_reader :rule
+
+      # @!attribute [r] blacklist
+      # @return [Rule]
+      attr_reader :blacklist
+
+      def initialize
+        @rule = Rule.new
+        @blacklist = Rule.new
+      end
+
+      delegate %i[
+        uri
+        host
+        path
+        query
+      ] => :rule
+
+      # Returns the associated instance method (action) of the first rule that
+      # matches a URI and the collected parameter hash from the rule chain.
+      # @return [[Boolean, Symbol, Hash]] if a matching rule exists.
+      # @return [false] if no matching rule exists or the URI is forbidden.
+      def route(uri)
+        return false if forbids?(uri)
+
+        # TODO: Use structs instead
+        is_matching, params, action = @rule.invoke(uri)
+        return action, params if is_matching && params
+
+        false
+      end
+
+      # Whether a route matches the URI.
+      # TODO: Test
+      def routes?(uri)
+        !!route(uri)
+      end
+
+      # Adds a {Rule} to the blacklist.
+      def forbid(opts = {}, &proc)
+        @blacklist.build_child_rule_chain_from_options(opts)
+        @blacklist.instance_eval(&proc) if block_given?
+        @blacklist
+      end
+
+      # Whether the URI is matched by the blacklist rule.
+      # @see #forbid
+      def forbids?(uri)
+        @blacklist.matches?(uri)
+      end
+
+      # Whether the URI is allowed.
+      # @see #forbid
+      def allows?(uri)
+        !forbids?(uri)
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/rule.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Wayfarer
+  module Routing
+    # Tree nodes
+    # @private
+    class Rule
+      include Enumerable
+
+      extend Forwardable
+
+      attr_reader :child_rules
+      attr_reader :target_action
+
+      delegate [:each] => :child_rules
+
+      def initialize(opts = {}, &proc)
+        @child_rules = []
+        @target_action = nil
+
+        build_child_rule_chain_from_options(opts)
+        instance_eval(&proc) if block_given?
+      end
+
+      def build_child_rule_chain_from_options(opts)
+        @target_action = opts.delete(:to)
+        opts.reduce(self) { |rule, (key, val)| rule.send(key, val) }
+      end
+
+      def matches?(uri)
+        return false unless match!(uri)
+        none? || any? { |child_rule| child_rule.matches?(uri) }
+      end
+
+      def invoke(uri)
+        rule_chain = matching_rule_chain(uri)
+
+        if rule_chain.any?
+          params = params_from_rule_chain(rule_chain, uri)
+          action = action_from_rule_chain(rule_chain)
+
+          [true, params, action]
+        else
+          false
+        end
+      end
+
+      # rubocop:disable Lint/AssignmentInCondition
+      def matching_rule_chain(uri, chain = [])
+        if match!(uri) && none?
+          chain << self
+        elsif matching_child = detect { |child_rule| child_rule.matches?(uri) }
+          matching_child.matching_rule_chain(uri, chain << self)
+        else
+          []
+        end
+      end
+      # rubocop:enable Lint/AssignmentInCondition
+
+      def params(*)
+        {}
+      end
+
+      def uri(*argv, &proc)
+        append_child_rule(URIRule.new(*argv, &proc))
+      end
+
+      def host(*argv, &proc)
+        append_child_rule(HostRule.new(*argv, &proc))
+      end
+
+      def path(*argv, &proc)
+        append_child_rule(PathRule.new(*argv, &proc))
+      end
+
+      def query(*argv, &proc)
+        append_child_rule(QueryRule.new(*argv, &proc))
+      end
+
+      def protocol(*argv, &proc)
+        append_child_rule(ProtocolRule.new(*argv, &proc))
+      end
+
+      def if(*argv, &proc)
+        append_child_rule(CustomRule.new(*argv, &proc))
+      end
+
+      def filetypes(*argv, &proc)
+        append_child_rule(FiletypesRule.new(*argv, &proc))
+      end
+
+      private
+
+      def append_child_rule(other)
+        @child_rules << other
+        other
+      end
+
+      def match!(*)
+        any?
+      end
+
+      def params_from_rule_chain(rule_chain, uri)
+        rule_chain.reduce({}) { |hash, rule| hash.merge(rule.params(uri)) }
+      end
+
+      def action_from_rule_chain(rule_chain)
+        rule_chain.map(&:target_action).reverse.reject(&:nil?).first
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/uri_rule.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Wayfarer
+  module Routing
+    # @private
+    class URIRule < Rule
+      def initialize(uri_str, opts = {}, &proc)
+        @uri = URI(uri_str)
+        super(opts, &proc)
+      end
+
+      private
+
+      def match!(uri)
+        uri == @uri
+      end
+    end
+  end
+end

data/lib/wayfarer.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# rubocop:disable Style/Documentation
+require "logger"
+require "uri"
+
+# Plumbing
+require_relative "wayfarer/configuration"
+
+# Routing
+require_relative "wayfarer/routing/rule"
+require_relative "wayfarer/routing/uri_rule"
+require_relative "wayfarer/routing/host_rule"
+require_relative "wayfarer/routing/path_rule"
+require_relative "wayfarer/routing/query_rule"
+require_relative "wayfarer/routing/protocol_rule"
+require_relative "wayfarer/routing/filetypes_rule"
+require_relative "wayfarer/routing/custom_rule"
+require_relative "wayfarer/routing/router"
+
+# Networking
+require_relative "wayfarer/http_adapters/net_http_adapter"
+require_relative "wayfarer/http_adapters/selenium_adapter"
+require_relative "wayfarer/http_adapters/adapter_pool"
+
+# Parsers
+require_relative "wayfarer/parsers/xml_parser"
+require_relative "wayfarer/parsers/json_parser"
+
+# Frontiers
+require_relative "wayfarer/frontiers/frontier"
+require_relative "wayfarer/frontiers/memory_frontier"
+require_relative "wayfarer/frontiers/redis_frontier"
+require_relative "wayfarer/frontiers/normalize_uris"
+
+unless RUBY_PLATFORM == "java"
+  require_relative "wayfarer/frontiers/memory_trie_frontier"
+  require_relative "wayfarer/frontiers/memory_bloomfilter"
+  require_relative "wayfarer/frontiers/redis_bloomfilter"
+end
+
+# Processing
+require_relative "wayfarer/crawl"
+require_relative "wayfarer/crawl_observer"
+require_relative "wayfarer/locals"
+require_relative "wayfarer/job"
+require_relative "wayfarer/finders"
+require_relative "wayfarer/page"
+require_relative "wayfarer/dispatcher"
+require_relative "wayfarer/processor"
+
+module Wayfarer
+  VERSION = "0.0.3"
+
+  def self.logger
+    return @logger if @logger
+
+    @logger = Logger.new(STDOUT)
+    @logger.level = Logger::WARN
+    @logger
+  end
+
+  def self.config
+    @config ||= Configuration.new
+    yield(@config) if block_given?
+    @config
+  end
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Configuration do
+  let(:config) { Configuration.new }
+
+  describe "::new" do
+    it "overrides defaults" do
+      config = Configuration.new(http_adapter: :selenium)
+      expect(config.http_adapter).to be :selenium
+    end
+  end
+
+  it "sets keys and values" do
+    config.foo = :foo
+    expect(config.foo).to be :foo
+  end
+
+  describe "#reset!" do
+    it "resets to defaults" do
+      config.max_http_redirects = 5
+      config.reset!
+      expect(config.max_http_redirects).to be 3
+    end
+  end
+end

data/spec/crawl_spec.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Crawl do
+  let(:uris) { %w(http://example.com) }
+  let(:job) { Class.new(Wayfarer::Job) }
+  subject(:crawl) { Crawl.new(job, *uris) }
+
+  describe "#frontier" do
+    context "by default" do
+      it "returns a MemoryFrontier" do
+        expect(crawl.frontier).to be_a MemoryFrontier
+      end
+    end
+
+    context "when Redis frontier is used", redis: true do
+      before { job.config.frontier = :redis }
+
+      it "returns a MemoryFrontier" do
+        expect(crawl.frontier).to be_a RedisFrontier
+      end
+    end
+
+    context "when memory bloomfilter is used", mri_only: true do
+      before { job.config.frontier = :memory_bloom }
+
+      it "returns a MemoryBloomfilter" do
+        expect(crawl.frontier).to be_a MemoryBloomfilter
+      end
+    end
+
+    context "when Redis bloomfilter is used", mri_only: true, redis: true do
+      before { job.config.frontier = :redis_bloom }
+
+      it "returns a RedisBloomfilter" do
+        expect(crawl.frontier).to be_a RedisBloomfilter
+      end
+    end
+
+    context "when memory trie is used", mri_only: true do
+      before { job.config.frontier = :memory_trie }
+
+      it "returns a RedisBloomfilter" do
+        expect(crawl.frontier).to be_a MemoryTrieFrontier
+      end
+    end
+  end
+end

data/spec/finders_spec.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Finders do
+  let(:page) { fetch_page(test_app("/finders.html")) }
+
+  describe "#links" do
+    context "without paths" do
+      it "returns all links" do
+        expect(page.links.map(&:to_s)).to eq %w(
+          http://localhost:9876/foo
+          http://localhost:9876/bar
+          http://localhost:9876/baz
+          http://google.com
+          http://yahoo.com
+          http://aol.com
+        )
+      end
+    end
+
+    context "with paths" do
+      it "returns targeted links" do
+        expect(page.links("ul li:nth-child(3) a").map(&:to_s)).to eq %w(
+          http://localhost:9876/baz
+        )
+      end
+    end
+  end
+
+  describe "#stylesheets" do
+    context "without paths" do
+      it "returns all stylesheets" do
+        expect(page.stylesheets.map(&:to_s)).to eq %w(
+          http://localhost:9876/a.css
+          http://localhost:9876/b.css
+          http://google.com/c.css
+        )
+      end
+    end
+
+    context "with paths" do
+      it "returns targeted stylesheets" do
+        expect(page.stylesheets("#stylesheet-c").map(&:to_s)).to eq %w(
+          http://google.com/c.css
+        )
+      end
+    end
+  end
+end

data/spec/frontiers/memory_bloomfilter_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::MemoryBloomfilter, mri_only: true do
+  it_behaves_like "Frontier", MemoryBloomfilter
+end

data/spec/frontiers/memory_frontier_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::MemoryFrontier do
+  it_behaves_like "Frontier", MemoryFrontier
+end

data/spec/frontiers/memory_trie_frontier_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::MemoryTrieFrontier, mri_only: true do
+  it_behaves_like "Frontier", MemoryTrieFrontier
+end

data/spec/frontiers/normalize_uris_spec.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::NormalizeURIs, mri_only: true do
+  let(:config) { Wayfarer.config }
+
+  let(:uris) do
+    %w(
+      http://example.com/foo?bar=1&zet=0
+      http://example.com//foo?zet=0&bar=1
+      http://example.com/foo?bar=1&zet=0#bar
+    )
+  end
+
+  subject(:frontier) do
+    MemoryFrontier.new(config).extend(NormalizeURIs)
+  end
+
+  describe "#stage" do
+    it "normalizes URIs" do
+      expect {
+        frontier.stage(*uris)
+      }.to change { frontier.staged_uris.count }.by(1)
+    end
+  end
+
+  describe "#staged?" do
+    it "normalizes URIs" do
+      frontier.stage(*uris)
+      uris.each { |u| expect(frontier.staged?(u)).to be true }
+    end
+  end
+
+  describe "#cache, #cached?" do
+    they "normalize URIs" do
+      frontier.cache(*uris)
+      uris.each { |u| expect(frontier.cached?(u)).to be true }
+    end
+  end
+
+  context "with normalization options" do
+    let(:config) do
+      Configuration.new(normalize_uri_options: { remove_hash: false })
+    end
+
+    let(:uris) do
+      %w(
+        http://example.com/foo
+        http://example.com/foo#bar
+      )
+    end
+
+    it "adheres to options" do
+      expect {
+        frontier.stage(*uris)
+      }.to change { frontier.staged_uris.count }.by(2)
+    end
+  end
+end

data/spec/frontiers/redis_bloomfilter_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::RedisBloomfilter, redis: true, mri_only: true do
+  it_behaves_like "Frontier", RedisBloomfilter
+end

data/spec/frontiers/redis_frontier_spec.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Frontiers::RedisFrontier, redis: true do
+  it_behaves_like "Frontier", RedisFrontier
+end

data/spec/http_adapters/adapter_pool_spec.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::HTTPAdapters::AdapterPool do
+  subject(:adapter_pool) { AdapterPool.new(Wayfarer::Job) }
+
+  describe "#with" do
+    after { adapter_pool.free }
+
+    context "when using Net::HTTP" do
+      it "yields a NetHTTPAdapter" do
+        adapter_pool.with do |adapter|
+          expect(adapter).to be_a NetHTTPAdapter
+        end
+      end
+    end
+
+    context "when using Selenium", selenium: true do
+      subject(:adapter_pool) do
+        job = Class.new(Wayfarer::Job)
+        job.config.http_adapter = :selenium
+
+        AdapterPool.new(job)
+      end
+
+      it "yields a SeleniumAdapter" do
+        adapter_pool.with do |adapter|
+          expect(adapter).to be_a SeleniumAdapter
+        end
+      end
+    end
+  end
+end

data/spec/http_adapters/net_http_adapter_spec.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::HTTPAdapters::NetHTTPAdapter do
+  subject(:adapter) { Wayfarer::HTTPAdapters::NetHTTPAdapter.instance }
+
+  describe "#fetch" do
+    it "returns a Page" do
+      uri = test_app("/hello_world")
+      page = adapter.fetch(uri)
+      expect(page).to be_a Page
+    end
+
+    it "sets the correct URI" do
+      uri = test_app("/status_code/404")
+      page = adapter.fetch(uri)
+      expect(page.uri).to eq test_app("/status_code/404")
+    end
+
+    it "retrieves the correct HTTP status code" do
+      uri = test_app("/status_code/404")
+      page = adapter.fetch(uri)
+      expect(page.status_code).to be 404
+    end
+
+    it "retrieves the correct response body" do
+      uri = test_app("/hello_world")
+      page = adapter.fetch(uri)
+      expect(page.body).to eq "Hello world!"
+    end
+
+    it "retrieves the correct response headers" do
+      uri = test_app("/hello_world")
+      page = adapter.fetch(uri)
+      expect(page.headers["hello"]).to eq ["world"]
+    end
+
+    context "with malformed URI" do
+      it "raises a MalformedURI" do
+        expect do
+          uri = URI("hptt://bro.ken")
+          page = adapter.fetch(uri)
+        end.to raise_error(
+          Wayfarer::HTTPAdapters::NetHTTPAdapter::MalformedURI
+        )
+      end
+    end
+
+    context "when response is a redirect" do
+      it "follows the redirect" do
+        uri = test_app("/redirect?times=3")
+        page = adapter.fetch(uri)
+
+        expect(page.uri.to_s).to eq "http://localhost:9876/redirect?times=0"
+      end
+
+      context "when maximum number of redirects reached" do
+        before { Wayfarer.config.max_http_redirects = 5 }
+        after  { Wayfarer.config.reset! }
+
+        it "raises a MaximumRedirectCountReached" do
+          expect do
+            uri = test_app("/redirect?times=6")
+            page = adapter.fetch(uri)
+          end.to raise_error(
+            Wayfarer::HTTPAdapters::NetHTTPAdapter::MaximumRedirectCountReached
+          )
+        end
+      end
+
+      context "when redirection URI is malformed" do
+        it "raises a MalformedRedirectURI" do
+          expect do
+            uri = test_app("/malformed_redirect")
+            page = adapter.fetch(uri)
+          end.to raise_error(
+            Wayfarer::HTTPAdapters::NetHTTPAdapter::MalformedRedirectURI
+          )
+        end
+      end
+    end
+  end
+end