wayfarer-jruby 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72b70dc0921adae626373012636e5646abae4586
-  data.tar.gz: 480b0c60cd58062d700de8a8f480386ba57f6dc7
+  metadata.gz: 295c1dd844209a188bfac6929bfc3085b8e757c2
+  data.tar.gz: 94ed8f5f78858c334c727bf5de4b2f870129e436
 SHA512:
-  metadata.gz: 523c57e24ff35bf95f3ef49b4c2d0e169eef5d237ee3614593217e5f9d51721e67fb326a87fb86fd9b075b3aa938e78ff96a04d6e5589f92a8b6de969addab43
-  data.tar.gz: 6b686f439ea1ca38304150ee0f1a3d927bb7a36d6a87de580e273f608127ebde01b152cca3fc69cb8b78ef2082e7d3efb9edddd5b1f889ee07547ed4649a1e61
+  metadata.gz: 31d3e3fc05ccec48e995eb05bc8207faf96efc907217ac75575d61f0a4f40eb8217b206df76d97d6897b6f2244b4cc3672e2687ead3b9bb8e74b098e8a926bcf
+  data.tar.gz: c1510f79a039cfbe6faf585689a5726f386249244dd8d932614bb4459f1a753aeed645b624d7e5ff4fc3d0e244f8cde8e17912c5d9feb6e2bf733d2b5c00f0de

data/Changelog.md

@@ -0,0 +1,10 @@
+## 0.0.2
+
+Features:
+
+* `Job#stage` now expands relative paths/URIs
+* Added `ProtocolRule` and `CustomRule`
+
+Bugfixes:
+
+* Fixed the CLI which relied on a removed method alias

data/README.md

@@ -6,7 +6,9 @@ Versatile web crawling with (J)Ruby
 
 * [__Usage and more__ on the website](https://bauerd.github.io/wayfarer/)
 * [__API documentation__ on Ruby-Doc.org](http://www.rubydoc.info/github/bauerd/wayfarer) (`master` branch)
-* [__Releases__ on RubyGems.org](https://rubygems.org/gems/wayfarer)
+* __Releases__ on RubyGems.org:
+  * [wayfarer](https://rubygems.org/gems/wayfarer)
+  * [wayfarer-jruby](https://rubygems.org/gems/wayfarer-jruby)
 
 MRI:
 ```

data/bin/wayfarer

@@ -61,7 +61,7 @@ module Wayfarer
 
     def set_log_level(options)
       if level = options[:log_level]
-        Wayfarer.log.level = log_level_from_string(level)
+        Wayfarer.logger.level = log_level_from_string(level)
       end
     end
 

data/docs/_includes/navigation.html

@@ -105,6 +105,16 @@
               Query rules
             </a>
           </li>
+          <li class="navigation__page">
+            <a class="navigation__link" href="{{base}}/routing/protocol_rules.html">
+              Protocol rules
+            </a>
+          </li>
+          <li class="navigation__page">
+            <a class="navigation__link" href="{{base}}/routing/custom_rules.html">
+              Custom rules
+            </a>
+          </li>
         </ul>
       </li>
 

data/docs/_layouts/default.html

@@ -11,7 +11,7 @@
             <a href="{{base}}" class="site-header__link">{{ site.title }}</a>
           </h1>
           <div class="site-header__version">
-            0.0.1
+            0.0.2
           </div>
         </header>
 

data/docs/guides/tutorial.md

@@ -157,15 +157,15 @@ Note that we still have a hard-coded URI in `#repository`. Usually, there are tw
 1. Constructing the successor URI from the current URI.
 2. Reading the URI from the HTTP response, e.g. extracting an `<a>` tag's `href` property.
 
-For the first case, say we're on `https://github.com/:user/:repo` and want to go to `https://github.com/:user/:repo/issues`. All that separates both URIs is the last path segment, and we can simply append it at runtime:
+For the first case, say we're on `https://github.com/:user/:repo` and want to go to `https://github.com/:user/:repo/issues`. `#stage` takes relative paths and URIs too, and constructs absolute URIs by appending to the current page's URI:
 
 {% highlight ruby %}
 class CollectGithubIssues < Wayfarer::Job
   # ...
 
   def index
-    # page#uri returns a URI object
-    stage page.uri.to_s << "/issues"
+    # Stages "#{page.uri}/issues"
+    stage "issues"
   end
 
   # ...

data/docs/routing/custom_rules.md

@@ -0,0 +1,16 @@
+---
+layout: default
+title: Custom rules
+---
+
+# Custom rules
+
+Custom rules take a block that gets yielded the URI or an object that responds to `#call(uri)`. If the block or the delegate return a truthy value, the rule matches.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.if -> (uri) { uri.host == uri.host.reverse }
+end
+{% endhighlight %}
+
+* Matches only URIs with palindrome hosts

data/docs/routing/protocol_rules.md

@@ -0,0 +1,17 @@
+---
+layout: default
+title: Protocol rules
+---
+
+# Protocol rules
+
+Protocol rules match against symbols/strings.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.protocol :https
+end
+{% endhighlight %}
+
+* Matches `https://example.com`.
+* Does not match `http://example.com`.

data/lib/wayfarer.rb

@@ -13,6 +13,8 @@ require_relative "wayfarer/routing/uri_rule"
 require_relative "wayfarer/routing/host_rule"
 require_relative "wayfarer/routing/path_rule"
 require_relative "wayfarer/routing/query_rule"
+require_relative "wayfarer/routing/protocol_rule"
+require_relative "wayfarer/routing/custom_rule"
 require_relative "wayfarer/routing/router"
 
 # Networking
@@ -47,7 +49,7 @@ require_relative "wayfarer/dispatcher"
 require_relative "wayfarer/processor"
 
 module Wayfarer
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 
   def self.logger
     return @logger if @logger

data/lib/wayfarer/job.rb

@@ -144,11 +144,22 @@ module Wayfarer
     end
 
     # Adds URIs to process in the next cycle.
-    # If a relative URI is given, the page's protocol and hostname get
-    # prepended.
+    # If a relative path is given, an absolute URI is constructed from the
+    # current {#page}'s URI.
     # @param [String, URI, Array<String>, Array<URI>]
     def stage(*uris)
-      @staged_uris.push(*uris.flatten)
+      expanded = uris.flatten.map do |u|
+        if (uri = URI(u)).absolute?
+          uri
+        else
+          # URI#join would discard the path of page.uri.path
+          current = page.uri.dup
+          current.path = File.join(page.uri.path, uri.path)
+          current
+        end
+      end
+
+      @staged_uris.push(*expanded)
     end
 
     # The {Page} representing the URI currently processed by an action.

data/lib/wayfarer/routing/custom_rule.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Wayfarer
+  module Routing
+    # @private
+    class CustomRule < Rule
+      def initialize(delegate_or_block = proc, opts = {}, &proc)
+        @delegate_or_block = delegate_or_block
+        super(opts, &proc)
+      end
+
+      private
+
+      def match!(uri)
+        !!@delegate_or_block.call(uri)
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/protocol_rule.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Wayfarer
+  module Routing
+    # @private
+    class ProtocolRule < Rule
+      def initialize(protocol, opts = {}, &proc)
+        @protocol = protocol.to_s
+        super(opts, &proc)
+      end
+
+      private
+
+      def match!(uri)
+        uri.scheme == @protocol
+      end
+    end
+  end
+end

data/lib/wayfarer/routing/rule.rb

@@ -79,6 +79,14 @@ module Wayfarer
         append_child_rule(QueryRule.new(*argv, &proc))
       end
 
+      def protocol(*argv, &proc)
+        append_child_rule(ProtocolRule.new(*argv, &proc))
+      end
+
+      def if(*argv, &proc)
+        append_child_rule(CustomRule.new(*argv, &proc))
+      end
+
       private
 
       def append_child_rule(other)

data/spec/integration/peeking_spec.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer, integration: true do
+  subject(:job) do
+    Class.new(Wayfarer::Job) do
+      config.reraise_exceptions = true
+    end
+  end
+
+  let(:entry) { test_app("/graph/index.html") }
+
+  describe "Peeking" do
+    it "works" do
+      job.class_eval do
+        route.path "/graph/index.html", to: :index
+        route.path "/graph/details/a.html", to: :detail
+
+        def index
+          peek = yield "http://localhost:9876/graph/details/a.html"
+          fail unless peek == :ok
+        end
+
+        def detail
+          :ok
+        end
+      end
+
+      expect {
+        job.perform_now(entry)
+      }.not_to raise_error
+    end
+  end
+
+  describe "Recursive peeking" do
+    it "does not work" do
+      job.class_eval do
+        route.path "/graph/index.html", to: :index
+        route.path "/graph/details/a.html", to: :a
+        route.path "/graph/details/a.html", to: :b
+
+        def index
+          peek = yield "http://localhost:9876/graph/details/a.html"
+          fail unless peek == :ok
+        end
+
+        def a
+          yield "http://localhost:9876/graph/details/b.html" or :ok
+        end
+
+        def b
+          yield "http://localhost:9876/graph/details/a.html"
+        end
+      end
+
+      expect {
+        job.perform_now(entry)
+      }.not_to raise_error
+    end
+  end
+end

data/spec/job_spec.rb

@@ -72,15 +72,39 @@ describe Wayfarer::Job do
   describe "#stage" do
     it "stages URIs" do
       job_instance = job.new
+      job_instance.page = Page.new(uri: URI("https://yahoo.com/qux"))
 
       uris = %w(
         http://google.com
         http://example.com
       )
 
+      expected = uris.map { |u| URI(u) }
+
+      expect {
+        job_instance.send(:stage, *uris)
+      }.to change { job_instance.staged_uris }.from([]).to(expected)
+    end
+
+    it "expands relative URIs" do
+      job_instance = job.new
+      job_instance.page = Page.new(uri: URI("https://yahoo.com/qux"))
+
+      uris = %w(
+        /foo/bar
+        bar/qux.html
+        barfoo
+      )
+
+      expected = %w(
+        https://yahoo.com/qux/foo/bar
+        https://yahoo.com/qux/bar/qux.html
+        https://yahoo.com/qux/barfoo
+      ).map { |u| URI(u) }
+
       expect {
         job_instance.send(:stage, *uris)
-      }.to change { job_instance.staged_uris }.from([]).to(uris)
+      }.to change { job_instance.staged_uris }.from([]).to(expected)
     end
   end
 end

data/spec/routing/custom_rule_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Routing::CustomRule do
+  let(:uri) { URI("http://example.com") }
+
+  describe "#matches?" do
+    context "with a block" do
+      context "when block is truthy" do
+        subject(:rule) { CustomRule.new -> (uri) { uri.is_a?(URI) } }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+
+      context "when block is fals-y" do
+        subject(:rule) { CustomRule.new -> (_) { false } }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+  end
+end

data/spec/routing/protocol_rule_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+require "spec_helpers"
+
+describe Wayfarer::Routing::ProtocolRule do
+  describe "#matches?" do
+    context "with a string" do
+      subject(:rule) { ProtocolRule.new(:http) }
+
+      context "with matching URI" do
+        let(:uri) { URI("http://example.com") }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be true
+        end
+      end
+
+      context "with mismatching URI" do
+        let(:uri) { URI("https://example.com") }
+
+        it "returns true" do
+          expect(rule.matches?(uri)).to be false
+        end
+      end
+    end
+  end
+end

data/spec/routing/rule_spec.rb

@@ -168,6 +168,28 @@ describe Wayfarer::Routing::Rule do
     end
   end
 
+  describe "#protocol" do
+    it "adds a ProtocolRule as a sub-rule" do
+      rule.protocol(:https)
+      expect(rule.child_rules.first).to be_a ProtocolRule
+    end
+
+    it "returns the added QueryRule" do
+      expect(rule.protocol(:https)).to be_a ProtocolRule
+    end
+  end
+
+  describe "#if" do
+    it "adds a CustomRule as a sub-rule" do
+      rule.if -> () { true }
+      expect(rule.child_rules.first).to be_a CustomRule
+    end
+
+    it "returns the added QueryRule" do
+      expect(rule.if -> () { true }).to be_a CustomRule
+    end
+  end
+
   describe "#build_child_rule_chain_from_options" do
     subject(:rule) do
       rule = Rule.new

data/wayfarer-jruby.gemspec

@@ -2,14 +2,14 @@
 
 Gem::Specification.new do |s|
   s.name          = "wayfarer-jruby"
-  s.version       = "0.0.1"
+  s.version       = "0.0.2"
   s.license       = "MIT"
 
   s.homepage      = "http://github.com/bauerd/wayfarer"
   s.description   = "Versatile web crawling with JRuby"
   s.summary       = s.description
 
-  s.date          = "2014-11-12"
+  s.date          = "2017-05-31"
   s.authors       = ["Dominic Bauer"]
   s.email         = "bauerdominic@gmail.com"
 

data/wayfarer.gemspec

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 Gem::Specification.new do |s|
   s.name          = "wayfarer"
-  s.version       = "0.0.1"
+  s.version       = "0.0.2"
   s.license       = "MIT"
 
   s.homepage      = "http://github.com/bauerd/wayfarer"
   s.description   = "Versatile web crawling with Ruby"
   s.summary       = s.description
 
-  s.date          = "2014-11-12"
+  s.date          = "2017-05-31"
   s.authors       = ["Dominic Bauer"]
   s.email         = "bauerdominic@gmail.com"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayfarer-jruby
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Dominic Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-12 00:00:00.000000000 Z
+date: 2017-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: connection_pool
@@ -374,6 +374,7 @@ files:
 - ".ruby-version"
 - ".travis.yml"
 - ".yardopts"
+- Changelog.md
 - Gemfile
 - LICENSE
 - README.md
@@ -516,8 +517,10 @@ files:
 - docs/recipes/javascript.md
 - docs/recipes/multiple_uris.md
 - docs/recipes/screenshots.md
+- docs/routing/custom_rules.md
 - docs/routing/host_rules.md
 - docs/routing/path_rules.md
+- docs/routing/protocol_rules.md
 - docs/routing/query_rules.md
 - docs/routing/routes.md
 - docs/routing/uri_rules.md
@@ -545,8 +548,10 @@ files:
 - lib/wayfarer/parsers/json_parser.rb
 - lib/wayfarer/parsers/xml_parser.rb
 - lib/wayfarer/processor.rb
+- lib/wayfarer/routing/custom_rule.rb
 - lib/wayfarer/routing/host_rule.rb
 - lib/wayfarer/routing/path_rule.rb
+- lib/wayfarer/routing/protocol_rule.rb
 - lib/wayfarer/routing/query_rule.rb
 - lib/wayfarer/routing/router.rb
 - lib/wayfarer/routing/rule.rb
@@ -565,13 +570,16 @@ files:
 - spec/http_adapters/selenium_adapter_spec.rb
 - spec/integration/callbacks_spec.rb
 - spec/integration/locals_spec.rb
+- spec/integration/peeking_spec.rb
 - spec/job_spec.rb
 - spec/page_spec.rb
 - spec/parsers/json_parser_spec.rb
 - spec/parsers/xml_parser_spec.rb
 - spec/processor_spec.rb
+- spec/routing/custom_rule_spec.rb
 - spec/routing/host_rule_spec.rb
 - spec/routing/path_rule_spec.rb
+- spec/routing/protocol_rule_spec.rb
 - spec/routing/query_rule_spec.rb
 - spec/routing/router_spec.rb
 - spec/routing/rule_spec.rb