wayfarer 0.0.3 → 0.4.0

data/lib/wayfarer/routing/path_rule.rb

@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-require "mustermann"
-
-module Wayfarer
-  module Routing
-    # @private
-    class PathRule < Rule
-      attr_reader :matcher
-
-      def initialize(arg, opts = {}, &proc)
-        @matcher = if arg.is_a? String
-                     Mustermann.new(arg, type: Wayfarer.config.mustermann_type)
-                   else
-                     arg
-                   end
-
-        super(opts, &proc)
-      end
-
-      def params(uri)
-        return {} unless match!(uri)
-
-        path = uri.path
-
-        if @matcher.is_a? Mustermann
-          @matcher.params(path)
-        else
-          captures = @matcher.match(full_path(uri)).captures
-
-          captures.each.with_index.reduce({}) do |hash, (capture, i)|
-            hash.merge(i.to_s => capture)
-          end
-        end
-      end
-
-      private
-
-      # rubocop:disable Style/CaseEquality
-      def match!(uri)
-        if @matcher.is_a? Mustermann
-          @matcher === uri.path
-        else
-          @matcher =~ full_path(uri)
-        end
-      end
-      # rubocop:enable Style/CaseEquality
-
-      def full_path(uri)
-        "#{uri.path}?#{uri.query}##{uri.fragment}"
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/protocol_rule.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-require "uri"
-
-module Wayfarer
-  module Routing
-    # @private
-    class ProtocolRule < Rule
-      def initialize(protocol, opts = {}, &proc)
-        @protocol = protocol.to_s
-        super(opts, &proc)
-      end
-
-      private
-
-      def match!(uri)
-        uri.scheme == @protocol
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/router.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-require "forwardable"
-
-module Wayfarer
-  module Routing
-    # A {Router} maps URIs onto a {Job}'s instance methods.
-    class Router
-      extend Forwardable
-
-      # @!attribute [r] rule
-      # @return [Rule]
-      attr_reader :rule
-
-      # @!attribute [r] blacklist
-      # @return [Rule]
-      attr_reader :blacklist
-
-      def initialize
-        @rule = Rule.new
-        @blacklist = Rule.new
-      end
-
-      delegate %i[
-        uri
-        host
-        path
-        query
-      ] => :rule
-
-      # Returns the associated instance method (action) of the first rule that
-      # matches a URI and the collected parameter hash from the rule chain.
-      # @return [[Boolean, Symbol, Hash]] if a matching rule exists.
-      # @return [false] if no matching rule exists or the URI is forbidden.
-      def route(uri)
-        return false if forbids?(uri)
-
-        # TODO: Use structs instead
-        is_matching, params, action = @rule.invoke(uri)
-        return action, params if is_matching && params
-
-        false
-      end
-
-      # Whether a route matches the URI.
-      # TODO: Test
-      def routes?(uri)
-        !!route(uri)
-      end
-
-      # Adds a {Rule} to the blacklist.
-      def forbid(opts = {}, &proc)
-        @blacklist.build_child_rule_chain_from_options(opts)
-        @blacklist.instance_eval(&proc) if block_given?
-        @blacklist
-      end
-
-      # Whether the URI is matched by the blacklist rule.
-      # @see #forbid
-      def forbids?(uri)
-        @blacklist.matches?(uri)
-      end
-
-      # Whether the URI is allowed.
-      # @see #forbid
-      def allows?(uri)
-        !forbids?(uri)
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/rule.rb

@@ -1,114 +0,0 @@
-# frozen_string_literal: true
-
-require "forwardable"
-
-module Wayfarer
-  module Routing
-    # Tree nodes
-    # @private
-    class Rule
-      include Enumerable
-
-      extend Forwardable
-
-      attr_reader :child_rules
-      attr_reader :target_action
-
-      delegate [:each] => :child_rules
-
-      def initialize(opts = {}, &proc)
-        @child_rules = []
-        @target_action = nil
-
-        build_child_rule_chain_from_options(opts)
-        instance_eval(&proc) if block_given?
-      end
-
-      def build_child_rule_chain_from_options(opts)
-        @target_action = opts.delete(:to)
-        opts.reduce(self) { |rule, (key, val)| rule.send(key, val) }
-      end
-
-      def matches?(uri)
-        return false unless match!(uri)
-        none? || any? { |child_rule| child_rule.matches?(uri) }
-      end
-
-      def invoke(uri)
-        rule_chain = matching_rule_chain(uri)
-
-        if rule_chain.any?
-          params = params_from_rule_chain(rule_chain, uri)
-          action = action_from_rule_chain(rule_chain)
-
-          [true, params, action]
-        else
-          false
-        end
-      end
-
-      # rubocop:disable Lint/AssignmentInCondition
-      def matching_rule_chain(uri, chain = [])
-        if match!(uri) && none?
-          chain << self
-        elsif matching_child = detect { |child_rule| child_rule.matches?(uri) }
-          matching_child.matching_rule_chain(uri, chain << self)
-        else
-          []
-        end
-      end
-      # rubocop:enable Lint/AssignmentInCondition
-
-      def params(*)
-        {}
-      end
-
-      def uri(*argv, &proc)
-        append_child_rule(URIRule.new(*argv, &proc))
-      end
-
-      def host(*argv, &proc)
-        append_child_rule(HostRule.new(*argv, &proc))
-      end
-
-      def path(*argv, &proc)
-        append_child_rule(PathRule.new(*argv, &proc))
-      end
-
-      def query(*argv, &proc)
-        append_child_rule(QueryRule.new(*argv, &proc))
-      end
-
-      def protocol(*argv, &proc)
-        append_child_rule(ProtocolRule.new(*argv, &proc))
-      end
-
-      def if(*argv, &proc)
-        append_child_rule(CustomRule.new(*argv, &proc))
-      end
-
-      def filetypes(*argv, &proc)
-        append_child_rule(FiletypesRule.new(*argv, &proc))
-      end
-
-      private
-
-      def append_child_rule(other)
-        @child_rules << other
-        other
-      end
-
-      def match!(*)
-        any?
-      end
-
-      def params_from_rule_chain(rule_chain, uri)
-        rule_chain.reduce({}) { |hash, rule| hash.merge(rule.params(uri)) }
-      end
-
-      def action_from_rule_chain(rule_chain)
-        rule_chain.map(&:target_action).reverse.reject(&:nil?).first
-      end
-    end
-  end
-end

data/lib/wayfarer/routing/uri_rule.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-require "uri"
-
-module Wayfarer
-  module Routing
-    # @private
-    class URIRule < Rule
-      def initialize(uri_str, opts = {}, &proc)
-        @uri = URI(uri_str)
-        super(opts, &proc)
-      end
-
-      private
-
-      def match!(uri)
-        uri == @uri
-      end
-    end
-  end
-end

data/spec/configuration_spec.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Configuration do
-  let(:config) { Configuration.new }
-
-  describe "::new" do
-    it "overrides defaults" do
-      config = Configuration.new(http_adapter: :selenium)
-      expect(config.http_adapter).to be :selenium
-    end
-  end
-
-  it "sets keys and values" do
-    config.foo = :foo
-    expect(config.foo).to be :foo
-  end
-
-  describe "#reset!" do
-    it "resets to defaults" do
-      config.max_http_redirects = 5
-      config.reset!
-      expect(config.max_http_redirects).to be 3
-    end
-  end
-end

data/spec/crawl_spec.rb

@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Crawl do
-  let(:uris) { %w(http://example.com) }
-  let(:job) { Class.new(Wayfarer::Job) }
-  subject(:crawl) { Crawl.new(job, *uris) }
-
-  describe "#frontier" do
-    context "by default" do
-      it "returns a MemoryFrontier" do
-        expect(crawl.frontier).to be_a MemoryFrontier
-      end
-    end
-
-    context "when Redis frontier is used", redis: true do
-      before { job.config.frontier = :redis }
-
-      it "returns a MemoryFrontier" do
-        expect(crawl.frontier).to be_a RedisFrontier
-      end
-    end
-
-    context "when memory bloomfilter is used", mri_only: true do
-      before { job.config.frontier = :memory_bloom }
-
-      it "returns a MemoryBloomfilter" do
-        expect(crawl.frontier).to be_a MemoryBloomfilter
-      end
-    end
-
-    context "when Redis bloomfilter is used", mri_only: true, redis: true do
-      before { job.config.frontier = :redis_bloom }
-
-      it "returns a RedisBloomfilter" do
-        expect(crawl.frontier).to be_a RedisBloomfilter
-      end
-    end
-
-    context "when memory trie is used", mri_only: true do
-      before { job.config.frontier = :memory_trie }
-
-      it "returns a RedisBloomfilter" do
-        expect(crawl.frontier).to be_a MemoryTrieFrontier
-      end
-    end
-  end
-end

data/spec/finders_spec.rb

@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Finders do
-  let(:page) { fetch_page(test_app("/finders.html")) }
-
-  describe "#links" do
-    context "without paths" do
-      it "returns all links" do
-        expect(page.links.map(&:to_s)).to eq %w(
-          http://localhost:9876/foo
-          http://localhost:9876/bar
-          http://localhost:9876/baz
-          http://google.com
-          http://yahoo.com
-          http://aol.com
-        )
-      end
-    end
-
-    context "with paths" do
-      it "returns targeted links" do
-        expect(page.links("ul li:nth-child(3) a").map(&:to_s)).to eq %w(
-          http://localhost:9876/baz
-        )
-      end
-    end
-  end
-
-  describe "#stylesheets" do
-    context "without paths" do
-      it "returns all stylesheets" do
-        expect(page.stylesheets.map(&:to_s)).to eq %w(
-          http://localhost:9876/a.css
-          http://localhost:9876/b.css
-          http://google.com/c.css
-        )
-      end
-    end
-
-    context "with paths" do
-      it "returns targeted stylesheets" do
-        expect(page.stylesheets("#stylesheet-c").map(&:to_s)).to eq %w(
-          http://google.com/c.css
-        )
-      end
-    end
-  end
-end

data/spec/frontiers/memory_bloomfilter_spec.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Frontiers::MemoryBloomfilter, mri_only: true do
-  it_behaves_like "Frontier", MemoryBloomfilter
-end

data/spec/frontiers/memory_frontier_spec.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Frontiers::MemoryFrontier do
-  it_behaves_like "Frontier", MemoryFrontier
-end

data/spec/frontiers/memory_trie_frontier_spec.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Frontiers::MemoryTrieFrontier, mri_only: true do
-  it_behaves_like "Frontier", MemoryTrieFrontier
-end

data/spec/frontiers/normalize_uris_spec.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Frontiers::NormalizeURIs, mri_only: true do
-  let(:config) { Wayfarer.config }
-
-  let(:uris) do
-    %w(
-      http://example.com/foo?bar=1&zet=0
-      http://example.com//foo?zet=0&bar=1
-      http://example.com/foo?bar=1&zet=0#bar
-    )
-  end
-
-  subject(:frontier) do
-    MemoryFrontier.new(config).extend(NormalizeURIs)
-  end
-
-  describe "#stage" do
-    it "normalizes URIs" do
-      expect {
-        frontier.stage(*uris)
-      }.to change { frontier.staged_uris.count }.by(1)
-    end
-  end
-
-  describe "#staged?" do
-    it "normalizes URIs" do
-      frontier.stage(*uris)
-      uris.each { |u| expect(frontier.staged?(u)).to be true }
-    end
-  end
-
-  describe "#cache, #cached?" do
-    they "normalize URIs" do
-      frontier.cache(*uris)
-      uris.each { |u| expect(frontier.cached?(u)).to be true }
-    end
-  end
-
-  context "with normalization options" do
-    let(:config) do
-      Configuration.new(normalize_uri_options: { remove_hash: false })
-    end
-
-    let(:uris) do
-      %w(
-        http://example.com/foo
-        http://example.com/foo#bar
-      )
-    end
-
-    it "adheres to options" do
-      expect {
-        frontier.stage(*uris)
-      }.to change { frontier.staged_uris.count }.by(2)
-    end
-  end
-end

data/spec/frontiers/redis_bloomfilter_spec.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Frontiers::RedisBloomfilter, redis: true, mri_only: true do
-  it_behaves_like "Frontier", RedisBloomfilter
-end

data/spec/frontiers/redis_frontier_spec.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Frontiers::RedisFrontier, redis: true do
-  it_behaves_like "Frontier", RedisFrontier
-end

data/spec/http_adapters/adapter_pool_spec.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::HTTPAdapters::AdapterPool do
-  subject(:adapter_pool) { AdapterPool.new(Wayfarer::Job) }
-
-  describe "#with" do
-    after { adapter_pool.free }
-
-    context "when using Net::HTTP" do
-      it "yields a NetHTTPAdapter" do
-        adapter_pool.with do |adapter|
-          expect(adapter).to be_a NetHTTPAdapter
-        end
-      end
-    end
-
-    context "when using Selenium", selenium: true do
-      subject(:adapter_pool) do
-        job = Class.new(Wayfarer::Job)
-        job.config.http_adapter = :selenium
-
-        AdapterPool.new(job)
-      end
-
-      it "yields a SeleniumAdapter" do
-        adapter_pool.with do |adapter|
-          expect(adapter).to be_a SeleniumAdapter
-        end
-      end
-    end
-  end
-end

data/spec/http_adapters/net_http_adapter_spec.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::HTTPAdapters::NetHTTPAdapter do
-  subject(:adapter) { Wayfarer::HTTPAdapters::NetHTTPAdapter.instance }
-
-  describe "#fetch" do
-    it "returns a Page" do
-      uri = test_app("/hello_world")
-      page = adapter.fetch(uri)
-      expect(page).to be_a Page
-    end
-
-    it "sets the correct URI" do
-      uri = test_app("/status_code/404")
-      page = adapter.fetch(uri)
-      expect(page.uri).to eq test_app("/status_code/404")
-    end
-
-    it "retrieves the correct HTTP status code" do
-      uri = test_app("/status_code/404")
-      page = adapter.fetch(uri)
-      expect(page.status_code).to be 404
-    end
-
-    it "retrieves the correct response body" do
-      uri = test_app("/hello_world")
-      page = adapter.fetch(uri)
-      expect(page.body).to eq "Hello world!"
-    end
-
-    it "retrieves the correct response headers" do
-      uri = test_app("/hello_world")
-      page = adapter.fetch(uri)
-      expect(page.headers["hello"]).to eq ["world"]
-    end
-
-    context "with malformed URI" do
-      it "raises a MalformedURI" do
-        expect do
-          uri = URI("hptt://bro.ken")
-          page = adapter.fetch(uri)
-        end.to raise_error(
-          Wayfarer::HTTPAdapters::NetHTTPAdapter::MalformedURI
-        )
-      end
-    end
-
-    context "when response is a redirect" do
-      it "follows the redirect" do
-        uri = test_app("/redirect?times=3")
-        page = adapter.fetch(uri)
-
-        expect(page.uri.to_s).to eq "http://localhost:9876/redirect?times=0"
-      end
-
-      context "when maximum number of redirects reached" do
-        before { Wayfarer.config.max_http_redirects = 5 }
-        after  { Wayfarer.config.reset! }
-
-        it "raises a MaximumRedirectCountReached" do
-          expect do
-            uri = test_app("/redirect?times=6")
-            page = adapter.fetch(uri)
-          end.to raise_error(
-            Wayfarer::HTTPAdapters::NetHTTPAdapter::MaximumRedirectCountReached
-          )
-        end
-      end
-
-      context "when redirection URI is malformed" do
-        it "raises a MalformedRedirectURI" do
-          expect do
-            uri = test_app("/malformed_redirect")
-            page = adapter.fetch(uri)
-          end.to raise_error(
-            Wayfarer::HTTPAdapters::NetHTTPAdapter::MalformedRedirectURI
-          )
-        end
-      end
-    end
-  end
-end

data/spec/http_adapters/selenium_adapter_spec.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::HTTPAdapters::SeleniumAdapter, selenium: true do
-  subject(:adapter) { Wayfarer::HTTPAdapters::SeleniumAdapter.new }
-  after { adapter.free }
-
-  describe "#fetch" do
-    it "returns a Page" do
-      uri = URI("http://0.0.0.0:9876/hello_world")
-      page = adapter.fetch(uri)
-      expect(page).to be_a Page
-    end
-
-    it "sets the correct URI" do
-      uri = URI("http://0.0.0.0:9876/status_code/404")
-      page = adapter.fetch(uri)
-      expect(page.uri.to_s).to eq "http://0.0.0.0:9876/status_code/404"
-    end
-
-    it "retrieves the correct HTTP status code" do
-      uri = URI("http://0.0.0.0:9876/status_code/404")
-      page = adapter.fetch(uri)
-      expect(page.status_code).to be 404
-    end
-
-    it "retrieves the correct response body" do
-      uri = URI("http://0.0.0.0:9876/hello_world")
-      page = adapter.fetch(uri)
-      expect(page.body).to match /Hello world!/
-    end
-
-    it "retrieves the correct response headers" do
-      uri = test_app("/hello_world")
-      page = adapter.fetch(uri)
-      expect(page.headers["hello"]).to eq ["world"]
-    end
-
-    context "when response is a redirect" do
-      it "follows the redirect" do
-        uri = test_app("/redirect?times=3")
-        page = adapter.fetch(uri)
-        expect(page.uri.to_s).to eq test_app("/redirect?times=0").to_s
-      end
-
-      context "when encountering a redirect loop" do
-        it "returns `nil` as HTTP status code" do
-          pending; fail
-        end
-      end
-    end
-  end
-end