wayfarer 0.0.3

data/docs/routing/custom_rules.md

@@ -0,0 +1,16 @@
+---
+layout: default
+title: Custom rules
+---
+
+# Custom rules
+
+Custom rules take a block that gets yielded the URI or an object that responds to `#call(uri)`. If the block or the delegate return a truthy value, the rule matches.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.if -> (uri) { uri.host == uri.host.reverse }
+end
+{% endhighlight %}
+
+* Matches only URIs with palindrome hosts

data/docs/routing/filetypes_rules.md

@@ -0,0 +1,21 @@
+---
+layout: default
+title: Filetypes rules
+---
+
+# Filetypes rules
+
+Filetypes rules match against the URI path's file extension.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.filetypes [:png, :jpg], to: :image
+  route.forbid.filetypes [:php, :js]
+end
+{% endhighlight %}
+
+Matches:
+
+* `http://example.com/foo.png`
+* `http://example.com/foo.jpg`
+* `https://example.com/qux/bar.jpg`

data/docs/routing/host_rules.md

@@ -0,0 +1,24 @@
+---
+layout: default
+title: Host rules
+---
+
+# Host rules
+
+Host rules match against a host string or RegExp.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.host "example.com"
+  route.host /example/
+end
+{% endhighlight %}
+
+Matches:
+
+* All URIs hosted on `"example.com"`.
+* All URIs that contain `"example"`.
+
+<aside class="note">
+<code>"www.host.net"</code> and <code>"host.net"</code> are not considered equal. You have to specify the exact host when using strings. Consider using <code>/host.net/</code> instead.
+</aside>

data/docs/routing/path_rules.md

@@ -0,0 +1,33 @@
+---
+layout: default
+title: Path rules
+---
+
+# Path rules
+
+Path rules match against the path of a URI. Both strings and RegExps are accepted, and path segment pattern matching and RegExp captures are supported.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.path "/:alpha/:beta", to: :foo
+  route.path /^foobar\/(.+)/, to: :bar
+
+  def foo
+    params[:alpha]
+    params[:beta]
+  end
+
+  def foo
+    params["0"]
+  end
+end
+{% endhighlight %}
+
+Matches:
+
+* All URIs with path segments matching `/:alpha/:beta`, e.g. `https://example.com/foo/bar`
+* All URIs starting with `"/foobar/"`.
+
+<aside class="note">
+<code>/:alpha/:beta</code> and <code>:alpha/:beta</code> are not considered equal. Note the opening slash.
+</aside>

data/docs/routing/protocol_rules.md

@@ -0,0 +1,17 @@
+---
+layout: default
+title: Protocol rules
+---
+
+# Protocol rules
+
+Protocol rules match against symbols/strings.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.protocol :https
+end
+{% endhighlight %}
+
+* Matches `https://example.com`.
+* Does not match `http://example.com`.

data/docs/routing/query_rules.md

@@ -0,0 +1,69 @@
+---
+layout: default
+title: Query rules
+---
+
+# Query rules
+
+Query rules impose constraints on key-value query parameters. Strings, integers, RegExps and ranges are supported.
+
+## String constraints
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.query arg: "foo"
+end
+{% endhighlight %}
+
+* Matches `https://example.com?arg=foo`.
+
+## Integer constraints
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.query arg: 42
+end
+{% endhighlight %}
+
+* Matches `https://example.com?arg=42`.
+
+---
+
+## RegExp constraints
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.query arg: /foo/
+end
+{% endhighlight %}
+
+* Matches `https://example.com?arg=foo`.
+* Matches `https://example.com?arg=foobar`.
+
+---
+
+## Range constraints
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.query arg: 1..10
+end
+{% endhighlight %}
+
+* Matches `https://example.com?arg=1`.
+* Matches […]
+* Matches `https://example.com?arg=10`.
+
+---
+
+## Compound constraints
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.query foo: 1..5, bar: /baz/, qux: "zot", toto: 2
+end
+{% endhighlight %}
+
+* Matches `https://example.com?foo=4&bar=bazqux&qux=zot&toto=2`.
+
+---

data/docs/routing/routes.md

@@ -0,0 +1,96 @@
+---
+layout: default
+title: Routes
+categories: [Routing]
+---
+
+# Routes
+
+* Routes are filters for interesting URIs.
+* Routes put constraints on URIs that should get processed.
+* Routes map URIs to instance methods (actions).
+* Routes are tree nodes and thus nestable.
+
+Currently, the following rules are available:
+
+* [URI rules](uri_rules.html) match URIs against a string.
+* [Host rules](/routing/host_rules.html) match hosts against strings and RegExps.
+* [Path rules](/routing/path_rules.html) match paths against pattern strings and RegExps. They support  path segment capturing.
+* [Query rules](/routing/query_rules.html) match key-value pairs of query parameters against strings, integers, RegExps and ranges.
+
+Routes can be fordidden. URIs that match forbidden rules are never processed.
+
+## Route declaration
+
+### Declaration order matching
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.host "example.com", to: :foo
+  route.path "/foo", to: :bar
+
+  # Is equivalent to:
+  #
+  # routes do
+  #   host "example.com", to: :foo
+  #   path "/foo", to: :bar
+  # end
+end
+{% endhighlight %}
+
+* Dispatches `https://example.com/foo` to `:foo`.
+* Dispatches `https://example.com` to `:foo`.
+* Dispatches `https://yahoo.com/foo` to `:bar`.
+
+---
+
+### Nesting routes (child rules)
+
+A route matches if it has a child rule that matches. This applies recursively.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.host "example.com", to: :foo do
+    path "/foo"
+  end
+
+  # Is equivalent to:
+  #
+  # route.host "example.com", path: "/foo", to: :foo
+  # route.path "/foo", host: "example.com", to: :foo
+end
+{% endhighlight %}
+
+* Dispatches `https://example.com/foo` to `:foo`.
+* Does not dispatch `https://example.com`.
+* Does not dispatch `https://yahoo.com/foo`.
+
+---
+
+### Deepest routes override actions
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.host "example.com", to: :foo do
+    path "/foo", to: :bar
+  end
+end
+{% endhighlight %}
+
+* Dispatches `https://example.com/foo` to `:bar`.
+* Does not dispatch `https://example.com`.
+* Does not dispatch `https://yahoo.com/foo`.
+
+---
+
+### Forbidding rules
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.forbid.path "/foo"
+  route.host "example.com", to: :foo
+end
+{% endhighlight %}
+
+* Dispatches `https://example.com` to `:bar`.
+* Does not dispatch `https://example.com/foo`.

data/docs/routing/uri_rules.md

@@ -0,0 +1,18 @@
+---
+layout: default
+title: URI Rules
+---
+
+# URI rules
+
+URI rules match against a string.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  route.uri "https://example.com"
+end
+{% endhighlight %}
+
+Matches:
+
+* Only `https://example.com`

data/examples/collect_github_issues.rb

@@ -0,0 +1,65 @@
+require_relative "../lib/wayfarer"
+
+class CollectGithubIssues < Wayfarer::Job
+  config.connection_count = 4
+  config.logger.level = :fatal
+
+  let(:records) { [] }
+
+  routes do
+    host "github.com" do
+      path "/:user/:repo",            to: :repository
+      path "/:user/:repo/issues",     to: :index
+      path "/:user/:repo/issues/:id", to: :show
+    end
+  end
+
+  after_crawl do
+    records.each do |issue|
+      # Save them somewhere?
+      puts issue
+    end
+  end
+
+  def repository
+    stage navigation_links
+  end
+
+  def index
+    stage issue_listing_links, next_page
+  end
+
+  def show
+    return halt if records.count > 30
+
+    records << {
+      id: params[:id],
+      title: issue_title,
+      author: issue_author
+    }
+  end
+
+  private
+
+  def issue_title
+    doc.css(".js-issue-title").text.strip
+  end
+
+  def issue_author
+    doc.css(".TableObject-item .author").text.strip
+  end
+
+  def navigation_links
+    page.links ".reponav-item"
+  end
+
+  def issue_listing_links
+    page.links ".issues-listing"
+  end
+
+  def next_page
+    page.links ".next_page"
+  end
+end
+
+CollectGithubIssues.perform_now("https://github.com/rails/rails")

data/examples/find_foobar_on_wikipedia.rb

@@ -0,0 +1,23 @@
+require_relative "../lib/wayfarer"
+
+class FindFoobarOnWikipedia < Wayfarer::Job
+  config.http_adapter = :selenium
+  config.selenium_argv = [:chrome]
+  config.connection_count = 4
+
+  let(:keywords) { [] }
+
+  route.host "en.wikipedia.org", to: :article
+
+  def article
+    if page.body =~ /Foobar/
+      driver.save_screenshot("/tmp/foobar.png")
+      return halt
+    end
+
+    keywords << page.keywords
+    stage page.links
+  end
+end
+
+FindFoobarOnWikipedia.perform_now("https://en.wikipedia.org/wiki/Special:Random")

data/lib/wayfarer/configuration.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "ostruct"
+require "securerandom"
+require "forwardable"
+
+module Wayfarer
+  class Configuration < OpenStruct
+    extend Forwardable
+
+    DEFAULTS = {
+      # Print full stacktraces?
+      print_stacktraces: true,
+
+      # Crash when encountering unhandled exceptions?
+      reraise_exceptions: false,
+
+      # Allow processing URIs multiple times?
+      allow_circulation: false,
+
+      # How many HTTP connections/Selenium drivers to use
+      # 1:1 correspondence with spawned threads
+      connection_count: 1,
+
+      # Which HTTP adapter to use. Supported are :net_http and :selenium
+      http_adapter: :net_http,
+
+      # Which frontier to use.
+      frontier: :memory,
+
+      # How long a thread may hold an HTTP adapter.
+      # Threads that exceed this limit fail with an exception.
+      connection_timeout: Float::INFINITY,
+
+      # How many 3xx redirects to follow. Has no effect when using Selenium
+      max_http_redirects: 3,
+
+      # Argument vector for instantiating Selenium drivers
+      selenium_argv: [:firefox],
+
+      # Argument vector for instantiating a Redis connection
+      redis_opts: {
+        host: "localhost",
+        port: 6379
+      }.freeze,
+
+      # Size of browser windows
+      window_size: [1024, 768],
+
+      # Which Mustermann pattern type to use when matching URI paths
+      # TODO: Mention in docs
+      mustermann_type: :sinatra,
+
+      # Options for instantiating Bloomfilters
+      bloomfilter_opts: {
+        size: 100,
+        hashes: 2,
+        seed: 1,
+        bucket: 3,
+        raise: false
+      },
+
+      # Whether to normalize URIs
+      normalize_uris: true,
+
+      # URI normalization options
+      # See: https://github.com/rwz/normalize_url
+      normalize_uri_options: {}
+    }.freeze
+
+    attr_reader :uuid
+
+    def initialize(overrides = {})
+      super(DEFAULTS.merge(overrides))
+      @uuid = SecureRandom.uuid
+    end
+
+    def logger
+      @logger ||= Wayfarer.logger.dup
+    end
+
+    def reset!
+      DEFAULTS.each { |key, val| self[key] = val }
+    end
+  end
+end

data/lib/wayfarer/crawl.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "observer"
+require "normalize_url"
+
+module Wayfarer
+  class Crawl
+    extend Forwardable
+    include Observable
+
+    # The prepared job.
+    # @!attribute [r] job
+    attr_reader :job
+
+    # @!attribute [r] dispatcher
+    attr_reader :dispatcher
+
+    delegate config: :job
+    delegate logger: :config
+
+    def initialize(job, *uris)
+      @job = job.prepare
+      @uris = uris
+      @dispatcher = Dispatcher.new(@job)
+      @processor = Processor.new(@job, frontier, @dispatcher)
+    end
+
+    def execute
+      trap_signals
+
+      CrawlObserver.new(@processor, @dispatcher, config.logger)
+
+      @job.run_hook(:before_crawl)
+      @processor.run(*@uris)
+      @job.run_hook(:after_crawl)
+    ensure
+      untrap_signals
+    end
+
+    # A frontier with initially pre-staged URIs.
+    # @return [Frontier]
+    def frontier
+      return @frontier if @frontier
+
+      @frontier = case config.frontier
+                  when :memory_trie
+                    Frontiers::MemoryTrieFrontier.new(config)
+                  when :redis
+                    Frontiers::RedisFrontier.new(config)
+                  when :memory_bloom
+                    Frontiers::MemoryBloomfilter.new(config)
+                  when :redis_bloom
+                    Frontiers::RedisBloomfilter.new(config)
+                  else
+                    Frontiers::MemoryFrontier.new(config)
+                  end
+
+      @frontier.extend(Frontiers::NormalizeURIs) if config.normalize_uris
+
+      @frontier.stage(*@uris) # TODO: Test
+
+      @frontier
+    end
+
+    private
+
+    def trap_signals
+      @cached_sigint_handler = trap(:INT) {
+        halt!
+        @cached_sigint_handler.try(:call)
+        exit(-1)
+      }
+    end
+
+    def untrap_signals
+      trap(:INT) { @cached_sigint_handler.try(:call) }
+    end
+  end
+end

data/lib/wayfarer/crawl_observer.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Wayfarer
+  class CrawlObserver
+    module Events
+      FirstCycle = Struct.new(:frontier)
+      NewCycle = Struct.new(:current_uris_count)
+      DispatchedURI = Struct.new(:action, :uri)
+      CycleFinished = Class.new
+      Peeking = Struct.new(:uri)
+      AboutToCycle = Struct.new(:staged_uris_count)
+      MismatchedURI = Struct.new(:uri)
+      HaltInitiated = Struct.new(:action, :uri)
+      StagingURIs = Struct.new(:staged_uris_count)
+      UnhandledError = Struct.new(:exception)
+    end
+
+    module ObservableShortcuts
+      def notify_observers!(*argv)
+        changed
+        notify_observers(*argv)
+      end
+    end
+
+    extend Forwardable
+
+    attr_reader :logger
+
+    def initialize(*observables, logger)
+      @logger = logger
+      observables.each { |obsv| obsv.add_observer(self) }
+    end
+
+    def update(event)
+      case event
+      when Events::FirstCycle     then first_cycle(event)
+      when Events::NewCycle       then new_cycle(event)
+      when Events::DispatchedURI  then dispatched_uri(event)
+      when Events::CycleFinished  then cycle_finished
+      when Events::Peeking        then peeking(event)
+      when Events::AboutToCycle   then about_to_cycle(event)
+      when Events::MismatchedURI  then mismatched_uri(event)
+      when Events::HaltInitiated  then halt_initiated(event)
+      when Events::StagingURIs    then staging_uris(event)
+      when Events::UnhandledError then unhandled_error(event)
+      end
+    end
+
+    private
+
+    def first_cycle(event)
+      logger.info("First cycle")
+      logger.info("Frontier: #{event.frontier}")
+    end
+
+    def new_cycle(event)
+      logger.info("Current cycle contains #{event.current_uris_count} URI(s)")
+    end
+
+    def dispatched_uri(event)
+      logger.info("Dispatched to \##{event.action}: #{event.uri}")
+    end
+
+    def cycle_finished
+      logger.info("No URIs left in current cycle")
+    end
+
+    def peeking(event)
+      logger.info("Peeking into: #{event.uri}")
+    end
+
+    def about_to_cycle(event)
+      logger.info("About to cycle. #{event.staged_uris_count} staged URI(s)")
+    end
+
+    def mismatched_uri(event)
+      logger.debug("No matching route for: #{event.uri}")
+    end
+
+    def halt_initiated(event)
+      logger.info("Halt initiated from \##{event.action} at: #{event.uri}")
+    end
+
+    def staging_uris(event)
+      logger.info("Staging #{event.staged_uris_count} URI(s)")
+    end
+
+    def unhandled_error(event)
+      level = config.reraise_exceptions ? :fatal : :error
+
+      if config.print_stacktraces
+        logger.public_send level, <<~LOGGER
+          Unhandled exception in an action: #{event.exception.class.inspect}
+          #{event.exception.backtrace.map(&:to_s).join("\n* ")}
+        LOGGER
+      else
+        logger.public_send level, <<~LOGGER
+          Unhandled exception in an action: #{event.exception.class.inspect}
+        LOGGER
+      end
+    end
+  end
+end