wayfarer 0.0.3 → 0.4.0

data/docs/_sass/vendor/pygments.scss

@@ -1,356 +0,0 @@
-/* Generated by Pygments CSS Theme Builder - https://jwarby.github.io/jekyll-pygments-themes/builder.html */
-/* Base Style */
-.highlight pre {
-  color: #333333;
-  background-color: transparent;
-}
-/* Punctuation */
-.highlight .p {
-  color: #333333;
-  background-color: transparent;
-}
-/* Error */
-.highlight .err {
-  color: #333333;
-  background-color: transparent;
-}
-/* Base Style */
-.highlight .n {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Attribute */
-.highlight .na {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Builtin */
-.highlight .nb {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Class */
-.highlight .nc {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Constant */
-.highlight .no {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Decorator */
-.highlight .nd {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Entity */
-.highlight .ni {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Name Exception */
-.highlight .ne {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Function */
-.highlight .nf {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Label */
-.highlight .nl {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Namespace */
-.highlight .nn {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Other */
-.highlight .nx {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Property */
-.highlight .py {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Tag */
-.highlight .nt {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Variable */
-.highlight .nv {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Variable Class */
-.highlight .vc {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Variable Global */
-.highlight .vg {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Variable Instance */
-.highlight .vi {
-  color: #333333;
-  background-color: transparent;
-}
-/* Name Builtin Pseudo */
-.highlight .bp {
-  color: #333333;
-  background-color: transparent;
-}
-/* Base Style */
-.highlight .g {
-  color: #333333;
-  background-color: transparent;
-}
-/*  */
-.highlight .gd {
-  color: #333333;
-  background-color: transparent;
-}
-/* Base Style */
-.highlight .o {
-  color: #333333;
-  background-color: transparent;
-}
-/* Operator Word */
-.highlight .ow {
-  color: #333333;
-  background-color: transparent;
-}
-/* Base Style */
-.highlight .c {
-  color: #727273;
-  background-color: transparent;
-}
-/* Comment Multiline */
-.highlight .cm {
-  color: #727273;
-  background-color: transparent;
-}
-/* Comment Preproc */
-.highlight .cp {
-  color: #727273;
-  background-color: transparent;
-}
-/* Comment Single */
-.highlight .c1 {
-  color: #727273;
-  background-color: transparent;
-}
-/* Comment Special */
-.highlight .cs {
-  color: #727273;
-  background-color: transparent;
-}
-/* Base Style */
-.highlight .k {
-  color: #333333;
-  background-color: transparent;
-}
-/* Keyword Constant */
-.highlight .kc {
-  color: #333333;
-  background-color: transparent;
-}
-/* Keyword Declaration */
-.highlight .kd {
-  color: #2f3661;
-  background-color: transparent;
-}
-/* Keyword Namespace */
-.highlight .kn {
-  color: #333333;
-  background-color: transparent;
-}
-/* Keyword Pseudo */
-.highlight .kp {
-  color: #333333;
-  background-color: transparent;
-}
-/* Keyword Reserved */
-.highlight .kr {
-  color: #333333;
-  background-color: transparent;
-}
-/* Keyword Type */
-.highlight .kt {
-  color: #333333;
-  background-color: transparent;
-}
-/* Base Style */
-.highlight .l {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal Date */
-.highlight .ld {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal Number */
-.highlight .m {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal Number Float */
-.highlight .mf {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal Number Hex */
-.highlight .mh {
-  color: #333333;
-  background-color: transparent;
-}
-/* Literal Number Integer */
-.highlight .mi {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal Number Oct */
-.highlight .mo {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal Number Integer Long */
-.highlight .il {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String */
-.highlight .s {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Backtick */
-.highlight .sb {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Char */
-.highlight .sc {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Doc */
-.highlight .sd {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Double */
-.highlight .s2 {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Escape */
-.highlight .se {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Heredoc */
-.highlight .sh {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Interpol */
-.highlight .si {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Other */
-.highlight .sx {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Regex */
-.highlight .sr {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Single */
-.highlight .s1 {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Literal String Symbol */
-.highlight .ss {
-  color: #a20e30;
-  background-color: transparent;
-}
-/* Base Style */
-.highlight .g {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Deleted */
-.highlight .gd {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Emph */
-.highlight .ge {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Error */
-.highlight .gr {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Heading */
-.highlight .gh {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Inserted */
-.highlight .gi {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Output */
-.highlight .go {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Prompt */
-.highlight .gp {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Strong */
-.highlight .gs {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Subheading */
-.highlight .gu {
-  color: #333333;
-  background-color: transparent;
-}
-/* Generic Traceback */
-.highlight .gt {
-  color: #333333;
-  background-color: transparent;
-}
-/* Other */
-.highlight .x {
-  color: #333333;
-  background-color: transparent;
-}
-/* Text Whitespace */
-.highlight .w {
-  color: #333333;
-  background-color: transparent;
-}

data/docs/automating_browsers/capybara.md

@@ -1,70 +0,0 @@
----
-layout: default
-title: Using Capybara
----
-
-# Using Capybara
-When using Selenium, Wayfarer supports Selenium drivers. You can execute JavaScript, take screenshots, interact with the page, and so on. For an exhaustive list, see [the official API documentation](http://www.rubydoc.info/gems/selenium-webdriver/0.0.28/Selenium/WebDriver/Driver).
-
-See [examples/selenium.rb](../examples/selenium.rb).
-
-## Setup
-Inside your instance methods, you have access to `#driver`, which returns a Selenium driver:
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  config do |c|
-    c.http_adapter = :selenium
-    c.selenium_argv = [:firefox]
-    c.connection_count = 4 # Number of instantiated WebDrivers
-  end
-
-  draw uri: "https://example.com"
-  def foo
-    driver # => #<Selenium::WebDriver::Driver:...>
-  end
-end
-{% endhighlight %}
-
-### Selenium Grid
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  config do |c|
-    c.http_adapter = :selenium
-    c.selenium_argv = [
-      :remote, url: "http://localhost:4444/wd/hub", desired_capabilities: :firefox
-    ]
-  end
-end
-{% endhighlight %}
-
-## Executing JavaScript
-```ruby
-class DummyJob < Wayfarer::Job
-  config do |c|
-    c.http_adapter = :selenium
-    c.selenium_argv = [:firefox]
-  end
-
-  draw uri: "https://example.com"
-  def example
-    driver.execute_script("console.log('Hello from wayfarer!')")
-  end
-end
-```
-
-## Taking screenshots
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  config do |c|
-    c.http_adapter = :selenium
-    c.selenium_argv = [:firefox]
-    c.window_size: [1024, 768]
-  end
-
-  draw uri: "https://example.com"
-  def example
-    driver.save_screenshot("/tmp/screenshot.png")
-  end
-end
-{% endhighlight %}

data/docs/css/screen.scss

@@ -1,7 +0,0 @@
----
----
-
-@charset "utf-8";
-
-@import "variables";
-@import "base";

data/docs/guides/cli.md

@@ -1,52 +0,0 @@
----
-layout: default
-title: CLI
----
-
-# Command-line interface
-Wayfarer ships with a small executable, `wayfarer`.
-
-Job classes are loaded by naming convention, e.g. if you pass `./directory/foo_bar.rb` as the `FILE` parameter, that file is expected to define the class `FooBar`. You can leave off the `.rb` extension.
-
-## `% wayfarer route FILE URI`
-Loads the job defined in `FILE`, and prints the first matching route for `URI`.
-
-## `% wayfarer enqueue FILE URI`
-Loads and enqueues the job in `FILE`, starting from `URI`.
-
-* `--log_level LEVEL`
-  Option. Which log messages to print.
-
-  * Default: `info`
-  * Recognized values: `unknown`, `debug`, `error`, `fatal`, `info`, `warn`
-
-* `--queue_adapter ADAPTER`
-  Option. Which ActiveJob queue adapter to use (e.g. `sidekiq`, `resque`).
-  * Recognized values: strings, see [documentation](http://api.rubyonrails.org/)
-
-* `--wait VALUE`
-  Option. Point of time when the enqueued job should be run.
-
-    1. If the value can be converted to an integer, it represents the seconds from now.
-    2. If the value can be parsed by `Time::parse`, the job gets scheduled at that point in time.
-    3. If the value is a human-readable time string that [Chronic](https://github.com/mojombo/chronic) can make sense of, the job is scheduled at that point in time.
-
-__Examples:__
-
-60 seconds from now:
-
-```
-% wayfarer enqueue ./foo_bar http://google.com --wait 60
-```
-
-6pm, today:
-
-```
-% wayfarer enqueue ./foo_bar http://google.com --wait 18:00
-```
-
-Tomorrow:
-
-```
-% wayfarer enqueue ./foo_bar http://google.com --wait tomorrow
-```

data/docs/guides/frontiers.md

@@ -1,93 +0,0 @@
----
-layout: default
-title: Frontiers
----
-
-# Frontiers
-
-Frontiers keep track of three sets of URIs:
-
-* Current URIs that are being processed
-* Staged URIs that might be processed in the next cycle
-* Cached URIs that have been processed
-
-All frontiers expose the same behaviour.
-
-<pre class="illustration">
-┌──────────────────────────────────────────────────────────┐
-│                          STAGED                          │
-│          {https://alpha.com, https://beta.com}           │
-└──────────────────────────────────────────────────────────┘
-┌──────────────────────────────────────────────────────────┐
-│                         CURRENT                          │
-│                   {https://gamma.com}                    │
-└──────────────────────────────────────────────────────────┘
-┌──────────────────────────────────────────────────────────┐
-│                          CACHED                          │
-│                    {https://beta.com}                    │
-└──────────────────────────────────────────────────────────┘
-                             │
-                           Cycle
-                             │
-                             ▼
-┌──────────────────────────────────────────────────────────┐
-│                         STAGED'                          │
-│                          {...}                           │
-└──────────────────────────────────────────────────────────┘
-┌──────────────────────────────────────────────────────────┐
-│                CURRENT' = STAGED \ CACHED                │
-│                   {https://alpha.com}                    │
-└──────────────────────────────────────────────────────────┘
-┌──────────────────────────────────────────────────────────┐
-│                CACHED' = CACHED ∪ CURRENT                │
-│          {https://beta.com, https://gamma.com}           │
-└──────────────────────────────────────────────────────────┘
-</pre>
-
-## Available frontiers
-Currently, there are 5 frontiers available:
-
-2. `:memory` (default): Uses sets from the standard lib.
-4. `:redis`: Uses Redis sets.
-3. `:memory_bloom`: Uses a [Bloom filter](https://github.com/igrigorik/bloomfilter-rb).
-5. `:redis_bloom`: Uses a Redis-backed Bloom filter.
-1. `:memory_trie`: Uses a [trie](https://github.com/tyler/trie) and sets.
-
-| Frontier | MRI support | JRuby support |
-| --- | --- |
-| `:memory` | Yes | Yes
-| `:redis` | Yes | Yes
-| `:memory_bloom` | Yes | No
-| `:redis_bloom` | Yes | No
-| `:memory_trie` | Yes | No
-
-## Setting the frontier
-
-Set the `:frontier` configuration key:
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  config.frontier = :foobar
-end
-{% endhighlight %}
-
-### Using a Redis frontier
-
-Set the `:redis_opts` and `:frontier` configuration keys:
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  config.redis_opts = { port: 4242 }
-  config.frontier = :redis
-end
-{% endhighlight %}
-
-### Setting bloomfilter parameters
-
-Set the `:bloomfilter_opts` configuration key:
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  config.bloomfilter_opts = { ... }
-end
-{% endhighlight %}

data/docs/guides/halting.md

@@ -1,23 +0,0 @@
----
-layout: default
-title: Halting
----
-
-# Halting
-Processing can be stopped by calling `#halt` within actions.
-
-`#halt` does not return immediately. Instead, it sets a halting flag internally, and once the action returns, all threads will stop instead of processing further URIs.
-
-Job instances run in separate threads. When a job signals that it wants to halt, all other threads will finish their current work, but will not process any further URIs. All instances have the chance to get their current work done.
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  def example
-    halt
-    puts "This will be printed!"
-
-    return halt
-    puts "This will not be printed!"
-  end
-end
-{% endhighlight %}

data/docs/guides/job_queues.md

@@ -1,26 +0,0 @@
----
-layout: default
-title: Locals
----
-
-# Job queues
-
-Thanks to [ActiveJob](http://edgeguides.rubyonrails.org/active_job_basics.html), jobs can be enqueued with various backends, e.g. Sidekiq or Resque:
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  # Overrides ActiveJob's global setting
-  self.queue_adapter = :resque
-
-  # Identifier for enqueued jobs
-  queue_as :dummy_job
-
-  # Alternatively, pass a block
-  queue_as do
-    [:first, :second].sample
-  end
-end
-
-# Alternatively, set the queue explicitly on call:
-DummyJob.set(queue: :something_else).perform_later(*uris)
-{% endhighlight %}

data/docs/guides/locals.md

@@ -1,36 +0,0 @@
----
-layout: default
-title: Locals
----
-
-# Locals
-
-Locals are Wayfarer's replacement for job instance variables. Both `let` and `let!` declare variables that are accessible within [callbacks]({{base}}/callbacks.html) and actions.
-
-Even though you might recognise them from RSpec, they have differing semantics: Values in `let` blocks will be replaced with thread-safe counterparts once the job is run. `let!` skips this. Both evaluate their block immediately.
-
-| Standard lib | Counterpart |
-| --- | --- |
-| Booleans | [`Concurrent::AtomicBoolean`](http://ruby-concurrency.github.io/concurrent-ruby/Concurrent/AtomicBoolean.html) |
-| `Fixnum` | [`Concurrent::AtomicFixnum`](http://ruby-concurrency.github.io/concurrent-ruby/Concurrent/AtomicFixnum.html) |
-| `Hash` | [`Concurrent::Hash`](http://ruby-concurrency.github.io/concurrent-ruby/Concurrent/Hash.html) |
-| `Array` | [`Concurrent::Array`](http://ruby-concurrency.github.io/concurrent-ruby/Concurrent/Array.html) |
-| Everything else | Untouched |
-
-{% highlight ruby %}
-class DummyJob < Wayfarer::Job
-  let(:values) { [1, 2, 3] }
-
-  before_crawl do
-    values.reverse!
-  end
-
-  after_crawl do
-    values # => [3, 2, 1, 0]
-  end
-
-  def some_action
-    values << 0
-  end
-end
-{% endhighlight %}

data/docs/guides/logging.md

@@ -1,22 +0,0 @@
----
-layout: default
-title: Logging
----
-
-# Logging
-
-{% highlight ruby %}
-# Global configuration serves as the template
-Wayfarer.logger.level = :fatal
-
-class DummyJob < Wayfarer::Job
-  # Jobs can tweak their logger
-  config.logger.level = :warn
-  config.logger.progname = "dummy-job"
-
-  def example
-    logger.info "No"
-    logger.warn "Yes"
-  end
-end
-{% endhighlight %}