wayfarer 0.0.3

data/docs/_sass/vendor/pygments.scss

@@ -0,0 +1,356 @@
+/* Generated by Pygments CSS Theme Builder - https://jwarby.github.io/jekyll-pygments-themes/builder.html */
+/* Base Style */
+.highlight pre {
+  color: #333333;
+  background-color: transparent;
+}
+/* Punctuation */
+.highlight .p {
+  color: #333333;
+  background-color: transparent;
+}
+/* Error */
+.highlight .err {
+  color: #333333;
+  background-color: transparent;
+}
+/* Base Style */
+.highlight .n {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Attribute */
+.highlight .na {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Builtin */
+.highlight .nb {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Class */
+.highlight .nc {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Constant */
+.highlight .no {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Decorator */
+.highlight .nd {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Entity */
+.highlight .ni {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Name Exception */
+.highlight .ne {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Function */
+.highlight .nf {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Label */
+.highlight .nl {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Namespace */
+.highlight .nn {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Other */
+.highlight .nx {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Property */
+.highlight .py {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Tag */
+.highlight .nt {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Variable */
+.highlight .nv {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Variable Class */
+.highlight .vc {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Variable Global */
+.highlight .vg {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Variable Instance */
+.highlight .vi {
+  color: #333333;
+  background-color: transparent;
+}
+/* Name Builtin Pseudo */
+.highlight .bp {
+  color: #333333;
+  background-color: transparent;
+}
+/* Base Style */
+.highlight .g {
+  color: #333333;
+  background-color: transparent;
+}
+/*  */
+.highlight .gd {
+  color: #333333;
+  background-color: transparent;
+}
+/* Base Style */
+.highlight .o {
+  color: #333333;
+  background-color: transparent;
+}
+/* Operator Word */
+.highlight .ow {
+  color: #333333;
+  background-color: transparent;
+}
+/* Base Style */
+.highlight .c {
+  color: #727273;
+  background-color: transparent;
+}
+/* Comment Multiline */
+.highlight .cm {
+  color: #727273;
+  background-color: transparent;
+}
+/* Comment Preproc */
+.highlight .cp {
+  color: #727273;
+  background-color: transparent;
+}
+/* Comment Single */
+.highlight .c1 {
+  color: #727273;
+  background-color: transparent;
+}
+/* Comment Special */
+.highlight .cs {
+  color: #727273;
+  background-color: transparent;
+}
+/* Base Style */
+.highlight .k {
+  color: #333333;
+  background-color: transparent;
+}
+/* Keyword Constant */
+.highlight .kc {
+  color: #333333;
+  background-color: transparent;
+}
+/* Keyword Declaration */
+.highlight .kd {
+  color: #2f3661;
+  background-color: transparent;
+}
+/* Keyword Namespace */
+.highlight .kn {
+  color: #333333;
+  background-color: transparent;
+}
+/* Keyword Pseudo */
+.highlight .kp {
+  color: #333333;
+  background-color: transparent;
+}
+/* Keyword Reserved */
+.highlight .kr {
+  color: #333333;
+  background-color: transparent;
+}
+/* Keyword Type */
+.highlight .kt {
+  color: #333333;
+  background-color: transparent;
+}
+/* Base Style */
+.highlight .l {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal Date */
+.highlight .ld {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal Number */
+.highlight .m {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal Number Float */
+.highlight .mf {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal Number Hex */
+.highlight .mh {
+  color: #333333;
+  background-color: transparent;
+}
+/* Literal Number Integer */
+.highlight .mi {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal Number Oct */
+.highlight .mo {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal Number Integer Long */
+.highlight .il {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String */
+.highlight .s {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Backtick */
+.highlight .sb {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Char */
+.highlight .sc {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Doc */
+.highlight .sd {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Double */
+.highlight .s2 {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Escape */
+.highlight .se {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Heredoc */
+.highlight .sh {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Interpol */
+.highlight .si {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Other */
+.highlight .sx {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Regex */
+.highlight .sr {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Single */
+.highlight .s1 {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Literal String Symbol */
+.highlight .ss {
+  color: #a20e30;
+  background-color: transparent;
+}
+/* Base Style */
+.highlight .g {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Deleted */
+.highlight .gd {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Emph */
+.highlight .ge {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Error */
+.highlight .gr {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Heading */
+.highlight .gh {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Inserted */
+.highlight .gi {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Output */
+.highlight .go {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Prompt */
+.highlight .gp {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Strong */
+.highlight .gs {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Subheading */
+.highlight .gu {
+  color: #333333;
+  background-color: transparent;
+}
+/* Generic Traceback */
+.highlight .gt {
+  color: #333333;
+  background-color: transparent;
+}
+/* Other */
+.highlight .x {
+  color: #333333;
+  background-color: transparent;
+}
+/* Text Whitespace */
+.highlight .w {
+  color: #333333;
+  background-color: transparent;
+}

data/docs/automating_browsers/capybara.md

@@ -0,0 +1,70 @@
+---
+layout: default
+title: Using Capybara
+---
+
+# Using Capybara
+When using Selenium, Wayfarer supports Selenium drivers. You can execute JavaScript, take screenshots, interact with the page, and so on. For an exhaustive list, see [the official API documentation](http://www.rubydoc.info/gems/selenium-webdriver/0.0.28/Selenium/WebDriver/Driver).
+
+See [examples/selenium.rb](../examples/selenium.rb).
+
+## Setup
+Inside your instance methods, you have access to `#driver`, which returns a Selenium driver:
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  config do |c|
+    c.http_adapter = :selenium
+    c.selenium_argv = [:firefox]
+    c.connection_count = 4 # Number of instantiated WebDrivers
+  end
+
+  draw uri: "https://example.com"
+  def foo
+    driver # => #<Selenium::WebDriver::Driver:...>
+  end
+end
+{% endhighlight %}
+
+### Selenium Grid
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  config do |c|
+    c.http_adapter = :selenium
+    c.selenium_argv = [
+      :remote, url: "http://localhost:4444/wd/hub", desired_capabilities: :firefox
+    ]
+  end
+end
+{% endhighlight %}
+
+## Executing JavaScript
+```ruby
+class DummyJob < Wayfarer::Job
+  config do |c|
+    c.http_adapter = :selenium
+    c.selenium_argv = [:firefox]
+  end
+
+  draw uri: "https://example.com"
+  def example
+    driver.execute_script("console.log('Hello from wayfarer!')")
+  end
+end
+```
+
+## Taking screenshots
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  config do |c|
+    c.http_adapter = :selenium
+    c.selenium_argv = [:firefox]
+    c.window_size: [1024, 768]
+  end
+
+  draw uri: "https://example.com"
+  def example
+    driver.save_screenshot("/tmp/screenshot.png")
+  end
+end
+{% endhighlight %}

data/docs/css/screen.scss

@@ -0,0 +1,7 @@
+---
+---
+
+@charset "utf-8";
+
+@import "variables";
+@import "base";

data/docs/guides/callbacks.md

@@ -0,0 +1,45 @@
+---
+layout: default
+title: Callbacks
+---
+
+# Callbacks
+
+Besides all [ActiveJob callbacks](http://api.rubyonrails.org/classes/ActiveJob/Callbacks/ClassMethods.html), three other callbacks are available. You have access to [locals](/guides/locals) in all callbacks.
+
+## `before_crawl`
+Fires __once__ before any pages have been retrieved.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  before_crawl { puts "Work is about to happen"  }
+end
+{% endhighlight %}
+
+## `after_crawl`
+Fires __once__ after all pages have been retrieved and processing is done.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  after_crawl { puts "Work did happen"  }
+end
+{% endhighlight %}
+
+## `setup_adapter`
+Fires for every adapter immediately after its creation.
+
+The block gets yielded an adapter. When using the Selenium HTTP adapter, both a WebDriver and a wrapping Capybara driver get yielded.
+
+{% highlight ruby %}
+class DummyJob < Wayfarer::Job
+  config.http_adapter = :selenium
+  config.connection_count = 4
+
+  setup_adapter do |adapter, driver, browser|
+    # This block gets called 4 times with different adapters
+    adapter # => The HTTP adapter
+    driver  # => #<Selenium::WebDriver::Driver:...> or nil
+    browser # => #<Capybara::Selenium::Driver:...> or nil
+  end
+end
+{% endhighlight %}

data/docs/guides/cli.md

@@ -0,0 +1,52 @@
+---
+layout: default
+title: CLI
+---
+
+# Command-line interface
+Wayfarer ships with a small executable, `wayfarer`.
+
+Job classes are loaded by naming convention, e.g. if you pass `./directory/foo_bar.rb` as the `FILE` parameter, that file is expected to define the class `FooBar`. You can leave off the `.rb` extension.
+
+## `% wayfarer route FILE URI`
+Loads the job defined in `FILE`, and prints the first matching route for `URI`.
+
+## `% wayfarer enqueue FILE URI`
+Loads and enqueues the job in `FILE`, starting from `URI`.
+
+* `--log_level LEVEL`
+  Option. Which log messages to print.
+
+  * Default: `info`
+  * Recognized values: `unknown`, `debug`, `error`, `fatal`, `info`, `warn`
+
+* `--queue_adapter ADAPTER`
+  Option. Which ActiveJob queue adapter to use (e.g. `sidekiq`, `resque`).
+  * Recognized values: strings, see [documentation](http://api.rubyonrails.org/)
+
+* `--wait VALUE`
+  Option. Point of time when the enqueued job should be run.
+
+    1. If the value can be converted to an integer, it represents the seconds from now.
+    2. If the value can be parsed by `Time::parse`, the job gets scheduled at that point in time.
+    3. If the value is a human-readable time string that [Chronic](https://github.com/mojombo/chronic) can make sense of, the job is scheduled at that point in time.
+
+__Examples:__
+
+60 seconds from now:
+
+```
+% wayfarer enqueue ./foo_bar http://google.com --wait 60
+```
+
+6pm, today:
+
+```
+% wayfarer enqueue ./foo_bar http://google.com --wait 18:00
+```
+
+Tomorrow:
+
+```
+% wayfarer enqueue ./foo_bar http://google.com --wait tomorrow
+```

data/docs/guides/configuration.md

@@ -0,0 +1,184 @@
+---
+layout: default
+title: Configuration
+---
+
+# Configuration
+
+All job classes base their configuration off the global one.
+
+{% highlight ruby %}
+# Setting a key globally applies to all jobs ...
+Wayfarer.config.key = :value
+
+class DummyJob < Wayfarer::Job
+  # ... unless a job overrides it
+  config.key = :other_value
+end
+
+class DummyJob < Wayfarer::Job
+  # Have it yielded
+  config { |c| c.key = :other_value }
+end
+{% endhighlight %}
+
+## Recognized keys and values
+
+### `print_stacktraces`
+  * Default: `true`
+  * Recognized values: Booleans
+
+Whether to print stacktraces when encounterting unhandled exceptions in job actions. See [Error handling]({{base}}/guides/error_handling.html).
+
+---
+
+### `reraise_exceptions`
+
+* Default: `false`
+* Recognized values: Booleans
+
+Whether to crash when encountering unhandled exceptions in job actions. See [Error handling]({{base}}/guides/error_handling.html).
+
+---
+
+### `allow_circulation`
+
+* Default: `false`
+* Recognized values: Booleans
+
+Whether URIs may be visited twice.
+
+<aside class="note">
+Allowing circulation might cause your jobs to not terminate.
+</aside>
+
+---
+
+### `normalize_uris`
+
+* Default: `true`
+* Recognized values: Booleans
+
+Whether to strip fragments, reorder query keys, etc. when staging and caching URIs. Customizable with the `:normalize_uri_options` key. See [normalize_url](https://github.com/rwz/normalize_url).
+
+---
+
+### `normalize_uri_options`
+
+* Default: `{}`
+* Recognized values: See [normalize_url](https://github.com/rwz/normalize_url).
+
+---
+
+### `frontier`
+* Default: `:memory`
+* Recognized values: See [(Redis) frontiers](frontiers.html).
+
+Which frontier to use.
+
+<aside class="note">
+Bloom filters may yield false positives. See the <a href="https://en.wikipedia.org/wiki/Bloom_filter">Wikipedia article</a>.
+</aside>
+
+---
+
+### `connection_count`
+
+* Default: `4`
+* Recognized values: Integers
+
+How many threads and HTTP adapters to use (1:1 correspondence).
+
+---
+
+### `http_adapter`
+
+* Default: `:net_http`
+* Recognized values: `:net_http`, `:selenium`
+
+Which HTTP adapter to use. See [Selenium & Capybara](selenium_capybara.html).
+
+---
+
+### `connection_timeout`
+
+* Default: `Float::INFINITY`
+* Recognized values: Floats
+
+Time in seconds that a job instance may hold an HTTP adapter. Instances that exceed this time limit raise an exception.
+
+---
+
+### `max_http_redirects`
+
+* Default: `3`
+* Recognized values: Integers
+
+How many 3xx redirects to follow.
+
+<aside class="note">
+Has no effect when using the <code>:selenium</code> HTTP adapter.
+</aside>
+
+---
+
+### `selenium_argv`
+
+* Default: `[:firefox]`
+* Recognized values: See [Selenium & Capybara](selenium_capybara.html)
+
+Argument vector passed to [`Selenium::WebDriver::Driver::for`](http://www.rubydoc.info/gems/selenium-webdriver/Selenium/WebDriver/Driver#for-class_method).
+
+---
+
+### `redis_opts`
+
+* Default: `{ host: "localhost", port: 6379 }`
+* Recognized values: [See documentation](http://www.rubydoc.info/github/redis/redis-rb/Redis%3Ainitialize)
+
+Options passed to [`Redis#initialize`](http://www.rubydoc.info/github/redis/redis-rb/Redis%3Ainitialize).
+
+---
+
+### `bloomfilter_opts`
+
+* Default:
+```
+{
+  size: 100,
+  hashes: 2,
+  seed: 1,
+  bucket: 3,
+  raise: false
+}
+```
+* Recognized values:
+  * `size`: Integers; number of buckets in a bloom filter
+  * `hashes`: Integers; number of hash functions
+  * `seed`: Integers; seed of hash functions
+  * `bucket`: Integers; number of bits in a bloom filter bucket
+  * `raise`: Booleans; whether to raise on bucket overflow
+
+Options for [bloomfilter-rb](https://github.com/igrigorik/bloomfilter-rb).
+
+---
+
+### `window_size`
+
+* Default: `[1024, 768]`
+* Recognized values: `[Integer, Integer]`
+
+Dimensions of browser windows.
+
+<aside class="note">
+Only has an effect when using the <code>:selenium</code> HTTP adapter.
+</aside>
+
+---
+
+### `mustermann_type`
+
+* Default: `:sinatra`
+* Recognized values: [See documentation](https://github.com/sinatra/mustermann)
+
+Which [Mustermann](https://github.com/sinatra/mustermann) pattern type to use.