wayfarer-jruby 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/.rbenv-gemsets +1 -0
- data/.rspec +3 -0
- data/.rubocop.yml +21 -0
- data/.ruby-version +1 -0
- data/.travis.yml +5 -0
- data/.yardopts +3 -0
- data/Gemfile +11 -0
- data/LICENSE +19 -0
- data/README.md +19 -0
- data/Rakefile +114 -0
- data/benchmark/frontiers.rb +143 -0
- data/bin/wayfarer +116 -0
- data/docs/.gitignore +2 -0
- data/docs/_config.yml +15 -0
- data/docs/_includes/base.html +7 -0
- data/docs/_includes/head.html +10 -0
- data/docs/_includes/navigation.html +172 -0
- data/docs/_layouts/default.html +42 -0
- data/docs/_sass/base.scss +439 -0
- data/docs/_sass/variables.scss +24 -0
- data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +19 -0
- data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +425 -0
- data/docs/_sass/vendor/bourbon/_bourbon.scss +90 -0
- data/docs/_sass/vendor/bourbon/addons/_border-color.scss +29 -0
- data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +48 -0
- data/docs/_sass/vendor/bourbon/addons/_border-style.scss +28 -0
- data/docs/_sass/vendor/bourbon/addons/_border-width.scss +28 -0
- data/docs/_sass/vendor/bourbon/addons/_buttons.scss +69 -0
- data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +25 -0
- data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +30 -0
- data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +31 -0
- data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +27 -0
- data/docs/_sass/vendor/bourbon/addons/_margin.scss +29 -0
- data/docs/_sass/vendor/bourbon/addons/_padding.scss +29 -0
- data/docs/_sass/vendor/bourbon/addons/_position.scss +51 -0
- data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +66 -0
- data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +27 -0
- data/docs/_sass/vendor/bourbon/addons/_size.scss +56 -0
- data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +118 -0
- data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +34 -0
- data/docs/_sass/vendor/bourbon/addons/_triangle.scss +63 -0
- data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +29 -0
- data/docs/_sass/vendor/bourbon/css3/_animation.scss +61 -0
- data/docs/_sass/vendor/bourbon/css3/_appearance.scss +5 -0
- data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +5 -0
- data/docs/_sass/vendor/bourbon/css3/_background-image.scss +44 -0
- data/docs/_sass/vendor/bourbon/css3/_background.scss +57 -0
- data/docs/_sass/vendor/bourbon/css3/_border-image.scss +61 -0
- data/docs/_sass/vendor/bourbon/css3/_calc.scss +6 -0
- data/docs/_sass/vendor/bourbon/css3/_columns.scss +67 -0
- data/docs/_sass/vendor/bourbon/css3/_filter.scss +6 -0
- data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +327 -0
- data/docs/_sass/vendor/bourbon/css3/_font-face.scss +29 -0
- data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +6 -0
- data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +12 -0
- data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +6 -0
- data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +15 -0
- data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +38 -0
- data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +40 -0
- data/docs/_sass/vendor/bourbon/css3/_perspective.scss +12 -0
- data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +10 -0
- data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +40 -0
- data/docs/_sass/vendor/bourbon/css3/_selection.scss +44 -0
- data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +27 -0
- data/docs/_sass/vendor/bourbon/css3/_transform.scss +21 -0
- data/docs/_sass/vendor/bourbon/css3/_transition.scss +81 -0
- data/docs/_sass/vendor/bourbon/css3/_user-select.scss +5 -0
- data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +16 -0
- data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +25 -0
- data/docs/_sass/vendor/bourbon/functions/_contains.scss +31 -0
- data/docs/_sass/vendor/bourbon/functions/_is-length.scss +16 -0
- data/docs/_sass/vendor/bourbon/functions/_is-light.scss +26 -0
- data/docs/_sass/vendor/bourbon/functions/_is-number.scss +16 -0
- data/docs/_sass/vendor/bourbon/functions/_is-size.scss +23 -0
- data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +74 -0
- data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +24 -0
- data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +26 -0
- data/docs/_sass/vendor/bourbon/functions/_shade.scss +24 -0
- data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +22 -0
- data/docs/_sass/vendor/bourbon/functions/_tint.scss +24 -0
- data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +37 -0
- data/docs/_sass/vendor/bourbon/functions/_unpack.scss +32 -0
- data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +26 -0
- data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +108 -0
- data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +53 -0
- data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +24 -0
- data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +35 -0
- data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +51 -0
- data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +77 -0
- data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +41 -0
- data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +74 -0
- data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +55 -0
- data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +28 -0
- data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +31 -0
- data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +15 -0
- data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +55 -0
- data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +7 -0
- data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +8 -0
- data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +9 -0
- data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +1 -0
- data/docs/_sass/vendor/neat/_neat-helpers.scss +11 -0
- data/docs/_sass/vendor/neat/_neat.scss +23 -0
- data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +49 -0
- data/docs/_sass/vendor/neat/functions/_private.scss +114 -0
- data/docs/_sass/vendor/neat/grid/_box-sizing.scss +15 -0
- data/docs/_sass/vendor/neat/grid/_direction-context.scss +33 -0
- data/docs/_sass/vendor/neat/grid/_display-context.scss +28 -0
- data/docs/_sass/vendor/neat/grid/_fill-parent.scss +22 -0
- data/docs/_sass/vendor/neat/grid/_media.scss +92 -0
- data/docs/_sass/vendor/neat/grid/_omega.scss +87 -0
- data/docs/_sass/vendor/neat/grid/_outer-container.scss +34 -0
- data/docs/_sass/vendor/neat/grid/_pad.scss +25 -0
- data/docs/_sass/vendor/neat/grid/_private.scss +35 -0
- data/docs/_sass/vendor/neat/grid/_row.scss +52 -0
- data/docs/_sass/vendor/neat/grid/_shift.scss +50 -0
- data/docs/_sass/vendor/neat/grid/_span-columns.scss +94 -0
- data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +97 -0
- data/docs/_sass/vendor/neat/grid/_visual-grid.scss +42 -0
- data/docs/_sass/vendor/neat/mixins/_clearfix.scss +25 -0
- data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +13 -0
- data/docs/_sass/vendor/neat/settings/_grid.scss +51 -0
- data/docs/_sass/vendor/neat/settings/_visual-grid.scss +27 -0
- data/docs/_sass/vendor/normalize-3.0.2.scss +427 -0
- data/docs/_sass/vendor/pygments.scss +356 -0
- data/docs/automating_browsers/capybara.md +70 -0
- data/docs/css/screen.scss +7 -0
- data/docs/guides/callbacks.md +45 -0
- data/docs/guides/cli.md +52 -0
- data/docs/guides/configuration.md +184 -0
- data/docs/guides/error_handling.md +46 -0
- data/docs/guides/frontiers.md +93 -0
- data/docs/guides/halting.md +23 -0
- data/docs/guides/job_queues.md +26 -0
- data/docs/guides/locals.md +36 -0
- data/docs/guides/logging.md +22 -0
- data/docs/guides/page_objects.md +67 -0
- data/docs/guides/peeking.md +46 -0
- data/docs/guides/selenium_capybara.md +100 -0
- data/docs/guides/tutorial.md +452 -0
- data/docs/index.md +82 -0
- data/docs/js/navigation.js +11 -0
- data/docs/misc/contributing.md +20 -0
- data/docs/misc/testing.md +11 -0
- data/docs/recipes/authentication.md +23 -0
- data/docs/recipes/csv.md +29 -0
- data/docs/recipes/javascript.md +20 -0
- data/docs/recipes/multiple_uris.md +18 -0
- data/docs/recipes/screenshots.md +20 -0
- data/docs/routing/host_rules.md +24 -0
- data/docs/routing/path_rules.md +33 -0
- data/docs/routing/query_rules.md +69 -0
- data/docs/routing/routes.md +96 -0
- data/docs/routing/uri_rules.md +18 -0
- data/examples/collect_github_issues.rb +65 -0
- data/examples/find_foobar_on_wikipedia.rb +23 -0
- data/lib/wayfarer.rb +65 -0
- data/lib/wayfarer/configuration.rb +86 -0
- data/lib/wayfarer/crawl.rb +79 -0
- data/lib/wayfarer/crawl_observer.rb +103 -0
- data/lib/wayfarer/dispatcher.rb +104 -0
- data/lib/wayfarer/finders.rb +61 -0
- data/lib/wayfarer/frontiers/frontier.rb +79 -0
- data/lib/wayfarer/frontiers/memory_bloomfilter.rb +32 -0
- data/lib/wayfarer/frontiers/memory_frontier.rb +76 -0
- data/lib/wayfarer/frontiers/memory_trie_frontier.rb +39 -0
- data/lib/wayfarer/frontiers/normalize_uris.rb +48 -0
- data/lib/wayfarer/frontiers/redis_bloomfilter.rb +34 -0
- data/lib/wayfarer/frontiers/redis_frontier.rb +83 -0
- data/lib/wayfarer/http_adapters/adapter_pool.rb +62 -0
- data/lib/wayfarer/http_adapters/net_http_adapter.rb +77 -0
- data/lib/wayfarer/http_adapters/selenium_adapter.rb +80 -0
- data/lib/wayfarer/job.rb +192 -0
- data/lib/wayfarer/locals.rb +40 -0
- data/lib/wayfarer/page.rb +94 -0
- data/lib/wayfarer/parsers/json_parser.rb +20 -0
- data/lib/wayfarer/parsers/xml_parser.rb +27 -0
- data/lib/wayfarer/processor.rb +103 -0
- data/lib/wayfarer/routing/host_rule.rb +19 -0
- data/lib/wayfarer/routing/path_rule.rb +54 -0
- data/lib/wayfarer/routing/query_rule.rb +59 -0
- data/lib/wayfarer/routing/router.rb +71 -0
- data/lib/wayfarer/routing/rule.rb +102 -0
- data/lib/wayfarer/routing/uri_rule.rb +21 -0
- data/spec/configuration_spec.rb +26 -0
- data/spec/crawl_spec.rb +48 -0
- data/spec/finders_spec.rb +49 -0
- data/spec/frontiers/memory_bloomfilter_spec.rb +6 -0
- data/spec/frontiers/memory_frontier_spec.rb +6 -0
- data/spec/frontiers/memory_trie_frontier_spec.rb +6 -0
- data/spec/frontiers/normalize_uris_spec.rb +59 -0
- data/spec/frontiers/redis_bloomfilter_spec.rb +6 -0
- data/spec/frontiers/redis_frontier_spec.rb +6 -0
- data/spec/http_adapters/adapter_pool_spec.rb +33 -0
- data/spec/http_adapters/net_http_adapter_spec.rb +83 -0
- data/spec/http_adapters/selenium_adapter_spec.rb +53 -0
- data/spec/integration/callbacks_spec.rb +42 -0
- data/spec/integration/locals_spec.rb +106 -0
- data/spec/job_spec.rb +86 -0
- data/spec/page_spec.rb +38 -0
- data/spec/parsers/json_parser_spec.rb +30 -0
- data/spec/parsers/xml_parser_spec.rb +24 -0
- data/spec/processor_spec.rb +31 -0
- data/spec/routing/host_rule_spec.rb +48 -0
- data/spec/routing/path_rule_spec.rb +66 -0
- data/spec/routing/query_rule_spec.rb +124 -0
- data/spec/routing/router_spec.rb +67 -0
- data/spec/routing/rule_spec.rb +218 -0
- data/spec/routing/uri_rule_spec.rb +24 -0
- data/spec/shared/frontier.rb +96 -0
- data/spec/spec_helpers.rb +62 -0
- data/spec/wayfarer_spec.rb +24 -0
- data/support/static/finders.html +38 -0
- data/support/static/graph/details/a.html +10 -0
- data/support/static/graph/details/b.html +10 -0
- data/support/static/graph/index.html +20 -0
- data/support/static/json/dummy.json +13 -0
- data/support/static/links/links.html +28 -0
- data/support/static/xml/dummy.xml +120 -0
- data/support/test_app.rb +45 -0
- data/wayfarer-jruby.gemspec +49 -0
- data/wayfarer.gemspec +53 -0
- metadata +616 -0
data/docs/index.md
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Versatile web crawling with (J)Ruby
|
4
|
+
---
|
5
|
+
|
6
|
+
# Versatile web crawling with (J)Ruby
|
7
|
+
|
8
|
+
Wayfarer is the swiss army knife for web crawling.
|
9
|
+
|
10
|
+
MRI:
|
11
|
+
```
|
12
|
+
% [sudo] gem install wayfarer
|
13
|
+
```
|
14
|
+
|
15
|
+
JRuby:
|
16
|
+
```
|
17
|
+
% [sudo] gem install wayfarer-jruby
|
18
|
+
```
|
19
|
+
|
20
|
+
If you …
|
21
|
+
|
22
|
+
* __Need to crawl page graphs breadth-first__
|
23
|
+
* __Need to extract whatever data__
|
24
|
+
* Do it multi-threaded
|
25
|
+
* Integrate with Rails seamlessly
|
26
|
+
* Want to automate a web browser
|
27
|
+
* Need to execute arbitrary JavaScript
|
28
|
+
* Need URI normalization
|
29
|
+
* Need to take screenshots
|
30
|
+
* Want to use a job queue and make work happen later
|
31
|
+
* Want in-memory and Redis-backed frontiers, tries and Bloom filters
|
32
|
+
|
33
|
+
… then you might like Wayfarer!
|
34
|
+
|
35
|
+
## What it looks like
|
36
|
+
|
37
|
+
Say you want to …
|
38
|
+
|
39
|
+
* Automate Google Chrome
|
40
|
+
* Start off with a random Wikipedia article
|
41
|
+
* Follow all links until you find a page with the word "Foobar"
|
42
|
+
* Take a screenshot of the page containing "Foobar"
|
43
|
+
* Extract keywords from every page you encounter
|
44
|
+
* Use 4 threads and Chrome processes to do so
|
45
|
+
|
46
|
+
This amounts to 16 lines of code with Wayfarer:
|
47
|
+
|
48
|
+
{% highlight ruby %}
|
49
|
+
require "wayfarer"
|
50
|
+
|
51
|
+
class FindFoobarOnWikipedia < Wayfarer::Job
|
52
|
+
config.http_adapter = :selenium
|
53
|
+
config.selenium_argv = [:chrome]
|
54
|
+
config.connection_count = 4
|
55
|
+
|
56
|
+
let(:keywords) { [] }
|
57
|
+
|
58
|
+
route.host "en.wikipedia.org", to: :article
|
59
|
+
|
60
|
+
def article
|
61
|
+
if page.body =~ /Foobar/
|
62
|
+
driver.save_screenshot("/tmp/foobar.png")
|
63
|
+
return halt
|
64
|
+
end
|
65
|
+
|
66
|
+
keywords << page.keywords
|
67
|
+
stage page.links
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
FindFoobarOnWikipedia.perform_now("https://en.wikipedia.org/wiki/Special:Random")
|
72
|
+
{% endhighlight %}
|
73
|
+
|
74
|
+
Wayfarer integrates with [ActiveJob]() and supports your favorite job queue out of the box. Your job is ready to be enqueued:
|
75
|
+
{% highlight ruby %}
|
76
|
+
FindFoobarOnWikipedia.perform_later("https://en.wikipedia.org/wiki/Special:Random")
|
77
|
+
{% endhighlight %}
|
78
|
+
|
79
|
+
### Where to go from here
|
80
|
+
|
81
|
+
* [The tutorial]() shows how to collect all open issues from a GitHub repository
|
82
|
+
* Read the [API documentation]()
|
@@ -0,0 +1,11 @@
|
|
1
|
+
document.addEventListener("DOMContentLoaded", function() {
|
2
|
+
var links = document.querySelectorAll(".navigation__link");
|
3
|
+
|
4
|
+
for (i = 0; i < links.length; i++) {
|
5
|
+
var link = links[i];
|
6
|
+
|
7
|
+
if (link.pathname === window.location.pathname) {
|
8
|
+
link.classList.add("navigation__link--active");
|
9
|
+
}
|
10
|
+
}
|
11
|
+
});
|
@@ -0,0 +1,20 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Contributing
|
4
|
+
---
|
5
|
+
|
6
|
+
# Contributing
|
7
|
+
|
8
|
+
1. Fork the repository
|
9
|
+
2. Ensure the development dependencies are installed:
|
10
|
+
`% bundle install --with development`
|
11
|
+
2. Make changes
|
12
|
+
3. Ensure your (new?) tests pass:
|
13
|
+
`% bundle exec rake test`
|
14
|
+
4. Autocorrect RubuCop offenses:
|
15
|
+
`% bundle exec rake rubocop:auto_correct`
|
16
|
+
5. Fix remaining offenses or have a good excuse not to:
|
17
|
+
`% bundle exec rake rubocop`
|
18
|
+
6. Write commit messages at least not worse than mine
|
19
|
+
7. Open a pull request on GitHub
|
20
|
+
8. Thank you
|
@@ -0,0 +1,23 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Authentication
|
4
|
+
---
|
5
|
+
|
6
|
+
# Authentication
|
7
|
+
|
8
|
+
Authentication is best handled with the `setup_adapter` callback See [Callbacks]({{base}}/guides/callbacks.html).
|
9
|
+
|
10
|
+
{% highlight ruby %}
|
11
|
+
class DummyJob < Wayfarer::Job
|
12
|
+
config.http_adapter = :selenium
|
13
|
+
|
14
|
+
setup_adapter do |_, _, browser|
|
15
|
+
browser.visit("https://foo.com/login")
|
16
|
+
|
17
|
+
browser.fill_in("E-mail", with: "foo@bar.com")
|
18
|
+
browser.fill_in("Password", with: "password")
|
19
|
+
|
20
|
+
browser.click_button("Log in")
|
21
|
+
end
|
22
|
+
end
|
23
|
+
{% endhighlight %}
|
data/docs/recipes/csv.md
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: CSV output
|
4
|
+
---
|
5
|
+
|
6
|
+
# CSV output
|
7
|
+
|
8
|
+
See:
|
9
|
+
|
10
|
+
* [Locals]({{base}}/guides/locals.html)
|
11
|
+
* [Callbacks]({{base}}/guides/callbacks.html)
|
12
|
+
|
13
|
+
{% highlight ruby %}
|
14
|
+
require "csv" # from Ruby's standard lib
|
15
|
+
|
16
|
+
class DummyJob < Wayfarer::Job
|
17
|
+
let(:records) { [] }
|
18
|
+
|
19
|
+
after_crawl do
|
20
|
+
CSV.open("output.csv", "w") do |csv|
|
21
|
+
records.each { |r| csv << [r[:id], r[:name]] }
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def detail
|
26
|
+
records << { id: ..., name: ... }
|
27
|
+
end
|
28
|
+
end
|
29
|
+
{% endhighlight %}
|
@@ -0,0 +1,20 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Executing JavaScript
|
4
|
+
---
|
5
|
+
|
6
|
+
# Executing JavaScript
|
7
|
+
|
8
|
+
In order to execute JavaScript in a page's DOM context, use the Selenium HTTP adapter and call `#execute_script` on the WebDriver object:
|
9
|
+
|
10
|
+
{% highlight ruby %}
|
11
|
+
class DummyJob < Wayfarer::Job
|
12
|
+
config.http_adapter = :selenium
|
13
|
+
|
14
|
+
# ...
|
15
|
+
|
16
|
+
def foo
|
17
|
+
pathname = driver.execute_script("return window.location.pathname")
|
18
|
+
end
|
19
|
+
end
|
20
|
+
{% endhighlight %}
|
@@ -0,0 +1,18 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Starting from multiple URIs
|
4
|
+
---
|
5
|
+
|
6
|
+
# Starting from multiple URIs
|
7
|
+
|
8
|
+
You can pass in as many URIs as desired when performing jobs:
|
9
|
+
|
10
|
+
{% highlight ruby %}
|
11
|
+
class DummyJob < Wayfarer::Job
|
12
|
+
# ...
|
13
|
+
end
|
14
|
+
|
15
|
+
uris = [...]
|
16
|
+
|
17
|
+
DummyJob.perform_now(*uris)
|
18
|
+
{% endhighlight %}
|
@@ -0,0 +1,20 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Taking screenshots
|
4
|
+
---
|
5
|
+
|
6
|
+
# Taking screenshots
|
7
|
+
|
8
|
+
In order to take screenshots, use the Selenium HTTP adapter and call `#save_screenshot` on the WebDriver object:
|
9
|
+
|
10
|
+
{% highlight ruby %}
|
11
|
+
class DummyJob < Wayfarer::Job
|
12
|
+
config.http_adapter = :selenium
|
13
|
+
|
14
|
+
# ...
|
15
|
+
|
16
|
+
def foo
|
17
|
+
driver.save_screenshot("my_screenshot.png")
|
18
|
+
end
|
19
|
+
end
|
20
|
+
{% endhighlight %}
|
@@ -0,0 +1,24 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Host rules
|
4
|
+
---
|
5
|
+
|
6
|
+
# Host rules
|
7
|
+
|
8
|
+
Host rules match against a host string or RegExp.
|
9
|
+
|
10
|
+
{% highlight ruby %}
|
11
|
+
class DummyJob < Wayfarer::Job
|
12
|
+
route.host "example.com"
|
13
|
+
route.host /example/
|
14
|
+
end
|
15
|
+
{% endhighlight %}
|
16
|
+
|
17
|
+
Matches:
|
18
|
+
|
19
|
+
* All URIs hosted on `"example.com"`.
|
20
|
+
* All URIs that contain `"example"`.
|
21
|
+
|
22
|
+
<aside class="note">
|
23
|
+
<code>"www.host.net"</code> and <code>"host.net"</code> are not considered equal. You have to specify the exact host when using strings. Consider using <code>/host.net/</code> instead.
|
24
|
+
</aside>
|
@@ -0,0 +1,33 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Path rules
|
4
|
+
---
|
5
|
+
|
6
|
+
# Path rules
|
7
|
+
|
8
|
+
Path rules match against the path of a URI. Both strings and RegExps are accepted, and path segment pattern matching and RegExp captures are supported.
|
9
|
+
|
10
|
+
{% highlight ruby %}
|
11
|
+
class DummyJob < Wayfarer::Job
|
12
|
+
route.path "/:alpha/:beta", to: :foo
|
13
|
+
route.path /^foobar\/(.+)/, to: :bar
|
14
|
+
|
15
|
+
def foo
|
16
|
+
params[:alpha]
|
17
|
+
params[:beta]
|
18
|
+
end
|
19
|
+
|
20
|
+
def foo
|
21
|
+
params["0"]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
{% endhighlight %}
|
25
|
+
|
26
|
+
Matches:
|
27
|
+
|
28
|
+
* All URIs with path segments matching `/:alpha/:beta`, e.g. `https://example.com/foo/bar`
|
29
|
+
* All URIs starting with `"/foobar/"`.
|
30
|
+
|
31
|
+
<aside class="note">
|
32
|
+
<code>/:alpha/:beta</code> and <code>:alpha/:beta</code> are not considered equal. Note the opening slash.
|
33
|
+
</aside>
|
@@ -0,0 +1,69 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Query rules
|
4
|
+
---
|
5
|
+
|
6
|
+
# Query rules
|
7
|
+
|
8
|
+
Query rules impose constraints on key-value query parameters. Strings, integers, RegExps and ranges are supported.
|
9
|
+
|
10
|
+
## String constraints
|
11
|
+
|
12
|
+
{% highlight ruby %}
|
13
|
+
class DummyJob < Wayfarer::Job
|
14
|
+
route.query arg: "foo"
|
15
|
+
end
|
16
|
+
{% endhighlight %}
|
17
|
+
|
18
|
+
* Matches `https://example.com?arg=foo`.
|
19
|
+
|
20
|
+
## Integer constraints
|
21
|
+
|
22
|
+
{% highlight ruby %}
|
23
|
+
class DummyJob < Wayfarer::Job
|
24
|
+
route.query arg: 42
|
25
|
+
end
|
26
|
+
{% endhighlight %}
|
27
|
+
|
28
|
+
* Matches `https://example.com?arg=42`.
|
29
|
+
|
30
|
+
---
|
31
|
+
|
32
|
+
## RegExp constraints
|
33
|
+
|
34
|
+
{% highlight ruby %}
|
35
|
+
class DummyJob < Wayfarer::Job
|
36
|
+
route.query arg: /foo/
|
37
|
+
end
|
38
|
+
{% endhighlight %}
|
39
|
+
|
40
|
+
* Matches `https://example.com?arg=foo`.
|
41
|
+
* Matches `https://example.com?arg=foobar`.
|
42
|
+
|
43
|
+
---
|
44
|
+
|
45
|
+
## Range constraints
|
46
|
+
|
47
|
+
{% highlight ruby %}
|
48
|
+
class DummyJob < Wayfarer::Job
|
49
|
+
route.query arg: 1..10
|
50
|
+
end
|
51
|
+
{% endhighlight %}
|
52
|
+
|
53
|
+
* Matches `https://example.com?arg=1`.
|
54
|
+
* Matches […]
|
55
|
+
* Matches `https://example.com?arg=10`.
|
56
|
+
|
57
|
+
---
|
58
|
+
|
59
|
+
## Compound constraints
|
60
|
+
|
61
|
+
{% highlight ruby %}
|
62
|
+
class DummyJob < Wayfarer::Job
|
63
|
+
route.query foo: 1..5, bar: /baz/, qux: "zot", toto: 2
|
64
|
+
end
|
65
|
+
{% endhighlight %}
|
66
|
+
|
67
|
+
* Matches `https://example.com?foo=4&bar=bazqux&qux=zot&toto=2`.
|
68
|
+
|
69
|
+
---
|
@@ -0,0 +1,96 @@
|
|
1
|
+
---
|
2
|
+
layout: default
|
3
|
+
title: Routes
|
4
|
+
categories: [Routing]
|
5
|
+
---
|
6
|
+
|
7
|
+
# Routes
|
8
|
+
|
9
|
+
* Routes are filters for interesting URIs.
|
10
|
+
* Routes put constraints on URIs that should get processed.
|
11
|
+
* Routes map URIs to instance methods (actions).
|
12
|
+
* Routes are tree nodes and thus nestable.
|
13
|
+
|
14
|
+
Currently, the following rules are available:
|
15
|
+
|
16
|
+
* [URI rules](uri_rules.html) match URIs against a string.
|
17
|
+
* [Host rules](/routing/host_rules.html) match hosts against strings and RegExps.
|
18
|
+
* [Path rules](/routing/path_rules.html) match paths against pattern strings and RegExps. They support path segment capturing.
|
19
|
+
* [Query rules](/routing/query_rules.html) match key-value pairs of query parameters against strings, integers, RegExps and ranges.
|
20
|
+
|
21
|
+
Routes can be fordidden. URIs that match forbidden rules are never processed.
|
22
|
+
|
23
|
+
## Route declaration
|
24
|
+
|
25
|
+
### Declaration order matching
|
26
|
+
|
27
|
+
{% highlight ruby %}
|
28
|
+
class DummyJob < Wayfarer::Job
|
29
|
+
route.host "example.com", to: :foo
|
30
|
+
route.path "/foo", to: :bar
|
31
|
+
|
32
|
+
# Is equivalent to:
|
33
|
+
#
|
34
|
+
# routes do
|
35
|
+
# host "example.com", to: :foo
|
36
|
+
# path "/foo", to: :bar
|
37
|
+
# end
|
38
|
+
end
|
39
|
+
{% endhighlight %}
|
40
|
+
|
41
|
+
* Dispatches `https://example.com/foo` to `:foo`.
|
42
|
+
* Dispatches `https://example.com` to `:foo`.
|
43
|
+
* Dispatches `https://yahoo.com/foo` to `:bar`.
|
44
|
+
|
45
|
+
---
|
46
|
+
|
47
|
+
### Nesting routes (child rules)
|
48
|
+
|
49
|
+
A route matches if it has a child rule that matches. This applies recursively.
|
50
|
+
|
51
|
+
{% highlight ruby %}
|
52
|
+
class DummyJob < Wayfarer::Job
|
53
|
+
route.host "example.com", to: :foo do
|
54
|
+
path "/foo"
|
55
|
+
end
|
56
|
+
|
57
|
+
# Is equivalent to:
|
58
|
+
#
|
59
|
+
# route.host "example.com", path: "/foo", to: :foo
|
60
|
+
# route.path "/foo", host: "example.com", to: :foo
|
61
|
+
end
|
62
|
+
{% endhighlight %}
|
63
|
+
|
64
|
+
* Dispatches `https://example.com/foo` to `:foo`.
|
65
|
+
* Does not dispatch `https://example.com`.
|
66
|
+
* Does not dispatch `https://yahoo.com/foo`.
|
67
|
+
|
68
|
+
---
|
69
|
+
|
70
|
+
### Deepest routes override actions
|
71
|
+
|
72
|
+
{% highlight ruby %}
|
73
|
+
class DummyJob < Wayfarer::Job
|
74
|
+
route.host "example.com", to: :foo do
|
75
|
+
path "/foo", to: :bar
|
76
|
+
end
|
77
|
+
end
|
78
|
+
{% endhighlight %}
|
79
|
+
|
80
|
+
* Dispatches `https://example.com/foo` to `:bar`.
|
81
|
+
* Does not dispatch `https://example.com`.
|
82
|
+
* Does not dispatch `https://yahoo.com/foo`.
|
83
|
+
|
84
|
+
---
|
85
|
+
|
86
|
+
### Forbidding rules
|
87
|
+
|
88
|
+
{% highlight ruby %}
|
89
|
+
class DummyJob < Wayfarer::Job
|
90
|
+
route.forbid.path "/foo"
|
91
|
+
route.host "example.com", to: :foo
|
92
|
+
end
|
93
|
+
{% endhighlight %}
|
94
|
+
|
95
|
+
* Dispatches `https://example.com` to `:bar`.
|
96
|
+
* Does not dispatch `https://example.com/foo`.
|