wayfarer 0.0.3 → 0.4.0

data/spec/routing/rule_spec.rb

@@ -1,251 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Routing::Rule do
-  subject(:rule) { Rule.new }
-
-  describe "#initialize" do
-    it "evaluates the Proc in its instance context" do
-      this = nil
-      rule = Rule.new { this = self }
-      expect(this).to be rule
-    end
-
-    it "builds the expected Rule chain from options" do
-      rule = Rule.new(host: "example.com", query: { bar: 42 })
-
-      first  = rule.child_rules.first
-      second = first.child_rules.first
-      third  = second.child_rules.first
-
-      expect(first).to be_a HostRule
-      expect(second).to be_a QueryRule
-    end
-  end
-
-  describe "#params" do
-    it "returns an empty Hash" do
-      expect(rule.params(URI("http://example.com"))).to eq({})
-    end
-  end
-
-  describe "#matching_rule_chain" do
-    let(:uri) { URI("http://example.com/foo/bar") }
-
-    context "with matching Rule" do
-      let(:host_rule_a) { HostRule.new("example.com") }
-      let(:host_rule_b) { HostRule.new("google.com") }
-      let(:path_rule_a) { PathRule.new("/{alpha}/{beta}") }
-      let(:path_rule_b) { PathRule.new("/{gamma}/{delta}") }
-
-      before do
-        host_rule_a.send(:append_child_rule, path_rule_a)
-        host_rule_a.send(:append_child_rule, path_rule_b)
-
-        host_rule_b.send(:append_child_rule, path_rule_a)
-        host_rule_b.send(:append_child_rule, path_rule_b)
-      end
-
-      subject(:rule) do
-        rule = Rule.new
-        rule.send(:append_child_rule, host_rule_a)
-        rule.send(:append_child_rule, host_rule_b)
-        rule
-      end
-
-      it "returns the expected Array" do
-        expect(rule.matching_rule_chain(uri)).to eq [
-          rule, host_rule_a, path_rule_a
-        ]
-      end
-    end
-
-    context "without matching Rule" do
-      subject(:rule) do
-        rule = Rule.new
-        rule.host("yahoo.com")
-        rule.host("google.com")
-        rule
-      end
-
-      it "returns an empty Array" do
-        expect(rule.matching_rule_chain(uri)).to eq []
-      end
-    end
-  end
-
-  describe "#params_from_rule_chain" do
-    let(:uri) { URI("http://example.com/foo/bar") }
-
-    context "with matching Rule" do
-      subject(:rule_chain) do
-        rule = Rule.new
-        rule.host("example.com").path("/{alpha}/{beta}")
-        rule.host("example.com").path("/{foo}/{bar}")
-        rule.matching_rule_chain(uri)
-      end
-
-      it "returns the expected URI parameters" do
-        expect(rule.send(:params_from_rule_chain, rule_chain, uri)).to eq("alpha" => "foo", "beta" => "bar")
-      end
-    end
-  end
-
-  describe "#invoke" do
-    let(:uri) { URI("http://example.com/foo/bar") }
-
-    context "with matching Rule" do
-      subject(:rule) do
-        rule = Rule.new
-        rule.host("example.com").path("/{alpha}/{beta}")
-        rule
-      end
-
-      it "returns a truthy value" do
-        expect(rule.invoke(uri)).to be_truthy
-      end
-
-      it "returns the URI parameters" do
-        _, params = rule.invoke(uri)
-        expect(params).to eq("alpha" => "foo", "beta" => "bar")
-      end
-    end
-
-    context "with mismatching Rule" do
-      subject(:rule) do
-        rule = Rule.new
-        rule.host("google.com").path("/{alpha}/{beta}")
-        rule
-      end
-
-      it "returns false" do
-        expect(rule.invoke(uri)).to be false
-      end
-    end
-  end
-
-  describe "#uri" do
-    it "adds a URIRule as a sub-rule" do
-      rule.uri("http://example.com/foo/bar")
-      expect(rule.child_rules.first).to be_a URIRule
-    end
-
-    it "returns the added URIRule" do
-      expect(rule.uri("http://example.com/foo/bar")).to be_a URIRule
-    end
-  end
-
-  describe "#host" do
-    it "adds a HostRule as a sub-rule" do
-      rule.host("example.com")
-      expect(rule.child_rules.first).to be_a HostRule
-    end
-
-    it "returns the added HostRule" do
-      expect(rule.host("example.com")).to be_a HostRule
-    end
-  end
-
-  describe "#path" do
-    it "adds a PathRule as a sub-rule" do
-      rule.path("/foo/bar")
-      expect(rule.child_rules.first).to be_a PathRule
-    end
-
-    it "returns the added PathRule" do
-      expect(rule.path("/foo/bar")).to be_a PathRule
-    end
-  end
-
-  describe "#query" do
-    it "adds a QueryRule as a sub-rule" do
-      rule.query(foo: "bar")
-      expect(rule.child_rules.first).to be_a QueryRule
-    end
-
-    it "returns the added QueryRule" do
-      expect(rule.query(foo: "bar")).to be_a QueryRule
-    end
-  end
-
-  describe "#protocol" do
-    it "adds a ProtocolRule as a sub-rule" do
-      rule.protocol(:https)
-      expect(rule.child_rules.first).to be_a ProtocolRule
-    end
-
-    it "returns the added QueryRule" do
-      expect(rule.protocol(:https)).to be_a ProtocolRule
-    end
-  end
-
-  describe "#if" do
-    it "adds a CustomRule as a sub-rule" do
-      rule.if -> () { true }
-      expect(rule.child_rules.first).to be_a CustomRule
-    end
-
-    it "returns the added QueryRule" do
-      expect(rule.if -> () { true }).to be_a CustomRule
-    end
-  end
-
-    describe "#filetypes" do
-      it "adds a FiletypesRule as a sub-rule" do
-        rule.filetypes [:png, :jpg]
-        expect(rule.child_rules.first).to be_a FiletypesRule
-      end
-
-      it "returns the added FiletypeRule" do
-        expect(rule.filetypes [:png]).to be_a FiletypesRule
-      end
-    end
-
-  describe "#build_child_rule_chain_from_options" do
-    subject(:rule) do
-      rule = Rule.new
-      rule.build_child_rule_chain_from_options(opts)
-      rule
-    end
-
-    describe ":path option" do
-      let(:opts) { { path: "/foo" } }
-
-      it "adds a PathRule" do
-        expect(rule.child_rules.first).to be_a PathRule
-      end
-    end
-
-    describe ":query option" do
-      let(:opts) { { query: { bar: "qux" } } }
-
-      it "adds a QueryRule" do
-        expect(rule.child_rules.first).to be_a QueryRule
-      end
-    end
-
-    describe ":host option" do
-      let(:opts) { { host: "example.com" } }
-
-      it "adds a HostRule" do
-        expect(rule.child_rules.first).to be_a HostRule
-      end
-    end
-
-    context "with multiple options present" do
-      let(:opts) do
-        { host: "example.com", path: "/foo", query: { bar: "qux" } }
-      end
-
-      it "adds chained sub-rules" do
-        first  = rule.child_rules.first
-        second = first.child_rules.first
-        third  = second.child_rules.first
-
-        expect(first).to be_a HostRule
-        expect(second).to be_a PathRule
-        expect(third).to be_a QueryRule
-      end
-    end
-  end
-end

data/spec/routing/uri_rule_spec.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-require "spec_helpers"
-
-describe Wayfarer::Routing::URIRule do
-  subject(:rule) { URIRule.new("http://example.com/foo/bar") }
-
-  describe "#matches?" do
-    context "with matching URI" do
-      let(:uri) { URI("http://example.com/foo/bar") }
-
-      it "returns true" do
-        expect(rule.matches?(uri)).to be true
-      end
-    end
-
-    context "with mismatching URI" do
-      let(:uri) { URI("http://example.com/foo/qux") }
-
-      it "returns true" do
-        expect(rule.matches?(uri)).to be false
-      end
-    end
-  end
-end

data/spec/shared/frontier.rb

@@ -1,96 +0,0 @@
-# frozen_string_literal: true
-require "securerandom"
-
-RSpec.shared_examples "Frontier" do |klass|
-  subject(:frontier) { klass.new(Configuration.new) }
-
-  after { frontier.free }
-
-  describe "#stage" do
-    let(:uri) { "http://example.com/foo#bar" }
-
-    it "stages a URI" do
-      frontier.stage("http://example.com")
-      expect(frontier.staged?("http://example.com")).to be true
-    end
-
-    it "stages multiple URIs" do
-      frontier.stage("http://example.com", "http://google.com")
-      expect(frontier.staged?("http://example.com")).to be true
-      expect(frontier.staged?("http://google.com")).to be true
-    end
-  end
-
-  describe "#cache" do
-    it "caches a URI" do
-      frontier.cache("http://example.com")
-      expect(frontier.cached?("http://example.com")).to be true
-    end
-
-    it "caches multiple URIs" do
-      frontier.cache("http://example.com", "http://google.com")
-      expect(frontier.cached?("http://example.com")).to be true
-      expect(frontier.cached?("http://google.com")).to be true
-    end
-  end
-
-  describe "#cycle" do
-    let(:uri) { "http://example.com" }
-
-    it "does not stage the same URI twice" do
-      2.times { frontier.stage(uri) }
-
-      expect { frontier.cycle }.to change { frontier.current_uris.count }.by(1)
-    end
-
-    it "caches URIs" do
-      frontier.stage(uri)
-
-      2.times { frontier.cycle }
-
-      expect(frontier.cached?(uri)).to be true
-    end
-
-    context "with staged URIs" do
-      before { frontier.stage(uri) }
-
-      it "swaps current and staged URIs" do
-        frontier.cycle
-        expect(frontier.current_uris).to eq [URI(uri)]
-        expect(frontier.staged_uris).to eq []
-      end
-
-      it "returns true" do
-        expect(frontier.cycle).to be true
-      end
-    end
-
-    context "without staged URIs" do
-      it "returns false" do
-        expect(frontier.cycle).to be false
-      end
-    end
-
-    it "does not set cached URIs as current" do
-      frontier.cache(uri)
-      frontier.stage(uri)
-      frontier.cycle
-
-      expect(frontier.current_uris).to be_empty
-    end
-
-    context "when circulation is allowed" do
-      subject(:frontier) do
-        klass.new(Configuration.new(allow_circulation: true))
-      end
-
-      it "sets cached URIs as current" do
-        3.times do
-          frontier.stage(uri)
-          frontier.cycle
-          expect(frontier.current_uris).to eq [URI(uri)]
-        end
-      end
-    end
-  end
-end

data/wayfarer-jruby.gemspec

@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-Gem::Specification.new do |s|
-  s.name          = "wayfarer-jruby"
-  s.version       = "0.0.3"
-  s.license       = "MIT"
-
-  s.homepage      = "http://github.com/bauerd/wayfarer"
-  s.description   = "Versatile web crawling with JRuby"
-  s.summary       = s.description
-
-  s.date          = "2017-05-31"
-  s.authors       = ["Dominic Bauer"]
-  s.email         = "bauerdominic@gmail.com"
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {spec}/*`.split("\n")
-
-  s.require_paths = ["lib"]
-
-  s.executables << "wayfarer"
-
-  s.add_runtime_dependency "connection_pool",            "~> 2.2"
-  s.add_runtime_dependency "nokogiri",                   "~> 1.6"
-  s.add_runtime_dependency "selenium-webdriver",         "~> 3.4"
-  s.add_runtime_dependency "selenium-emulated_features", "2.0.1"
-  s.add_runtime_dependency "net-http-persistent",        "~> 2.9"
-  s.add_runtime_dependency "mime-types",                 "~> 2.6"
-  s.add_runtime_dependency "mustermann",                 "~> 0.4"
-  s.add_runtime_dependency "activejob",                  "~> 4.2"
-  s.add_runtime_dependency "thor",                       "~> 0.19"
-  s.add_runtime_dependency "activesupport",              "~> 4.2"
-  s.add_runtime_dependency "capybara",                   "~> 2.5"
-  s.add_runtime_dependency "chronic",                    "~> 0.10"
-  s.add_runtime_dependency "redis",                      "~> 3.2"
-  s.add_runtime_dependency "concurrent-ruby",            "~> 1.0"
-  s.add_runtime_dependency "hooks",                      "~> 0.4"
-  s.add_runtime_dependency "normalize_url",              "~> 0.0.6"
-
-  s.add_development_dependency "pry",      "~> 0.10"
-  s.add_development_dependency "rake",     "~> 10.4"
-  s.add_development_dependency "rspec",    "~> 3.3"
-  s.add_development_dependency "sinatra",  "~> 1.4"
-  s.add_development_dependency "rubocop",  "~> 0.34"
-  s.add_development_dependency "yard",     "~> 0.9"
-  s.add_development_dependency "bourbon",  "~> 4.3"
-  s.add_development_dependency "neat",     "~> 1.8"
-  s.add_development_dependency "faker",    "~> 1.7"
-end