wayfarer 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +8 -0
- data/.rbenv-gemsets +1 -0
- data/.rspec +3 -0
- data/.rubocop.yml +21 -0
- data/.ruby-version +1 -0
- data/.travis.yml +5 -0
- data/.yardopts +3 -0
- data/Changelog.md +10 -0
- data/Gemfile +11 -0
- data/LICENSE +19 -0
- data/README.md +21 -0
- data/Rakefile +114 -0
- data/benchmark/frontiers.rb +143 -0
- data/bin/wayfarer +116 -0
- data/docs/.gitignore +2 -0
- data/docs/_config.yml +15 -0
- data/docs/_includes/base.html +7 -0
- data/docs/_includes/head.html +10 -0
- data/docs/_includes/navigation.html +187 -0
- data/docs/_layouts/default.html +42 -0
- data/docs/_sass/base.scss +439 -0
- data/docs/_sass/variables.scss +24 -0
- data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +19 -0
- data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +425 -0
- data/docs/_sass/vendor/bourbon/_bourbon.scss +90 -0
- data/docs/_sass/vendor/bourbon/addons/_border-color.scss +29 -0
- data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +48 -0
- data/docs/_sass/vendor/bourbon/addons/_border-style.scss +28 -0
- data/docs/_sass/vendor/bourbon/addons/_border-width.scss +28 -0
- data/docs/_sass/vendor/bourbon/addons/_buttons.scss +69 -0
- data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +25 -0
- data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +30 -0
- data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +31 -0
- data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +27 -0
- data/docs/_sass/vendor/bourbon/addons/_margin.scss +29 -0
- data/docs/_sass/vendor/bourbon/addons/_padding.scss +29 -0
- data/docs/_sass/vendor/bourbon/addons/_position.scss +51 -0
- data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +66 -0
- data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +27 -0
- data/docs/_sass/vendor/bourbon/addons/_size.scss +56 -0
- data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +118 -0
- data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +34 -0
- data/docs/_sass/vendor/bourbon/addons/_triangle.scss +63 -0
- data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +29 -0
- data/docs/_sass/vendor/bourbon/css3/_animation.scss +61 -0
- data/docs/_sass/vendor/bourbon/css3/_appearance.scss +5 -0
- data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +5 -0
- data/docs/_sass/vendor/bourbon/css3/_background-image.scss +44 -0
- data/docs/_sass/vendor/bourbon/css3/_background.scss +57 -0
- data/docs/_sass/vendor/bourbon/css3/_border-image.scss +61 -0
- data/docs/_sass/vendor/bourbon/css3/_calc.scss +6 -0
- data/docs/_sass/vendor/bourbon/css3/_columns.scss +67 -0
- data/docs/_sass/vendor/bourbon/css3/_filter.scss +6 -0
- data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +327 -0
- data/docs/_sass/vendor/bourbon/css3/_font-face.scss +29 -0
- data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +6 -0
- data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +12 -0
- data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +6 -0
- data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +15 -0
- data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +38 -0
- data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +40 -0
- data/docs/_sass/vendor/bourbon/css3/_perspective.scss +12 -0
- data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +10 -0
- data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +40 -0
- data/docs/_sass/vendor/bourbon/css3/_selection.scss +44 -0
- data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +27 -0
- data/docs/_sass/vendor/bourbon/css3/_transform.scss +21 -0
- data/docs/_sass/vendor/bourbon/css3/_transition.scss +81 -0
- data/docs/_sass/vendor/bourbon/css3/_user-select.scss +5 -0
- data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +16 -0
- data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +25 -0
- data/docs/_sass/vendor/bourbon/functions/_contains.scss +31 -0
- data/docs/_sass/vendor/bourbon/functions/_is-length.scss +16 -0
- data/docs/_sass/vendor/bourbon/functions/_is-light.scss +26 -0
- data/docs/_sass/vendor/bourbon/functions/_is-number.scss +16 -0
- data/docs/_sass/vendor/bourbon/functions/_is-size.scss +23 -0
- data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +74 -0
- data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +24 -0
- data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +26 -0
- data/docs/_sass/vendor/bourbon/functions/_shade.scss +24 -0
- data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +22 -0
- data/docs/_sass/vendor/bourbon/functions/_tint.scss +24 -0
- data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +37 -0
- data/docs/_sass/vendor/bourbon/functions/_unpack.scss +32 -0
- data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +26 -0
- data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +108 -0
- data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +53 -0
- data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +24 -0
- data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +35 -0
- data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +51 -0
- data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +77 -0
- data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +41 -0
- data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +74 -0
- data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +55 -0
- data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +28 -0
- data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +31 -0
- data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +15 -0
- data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +55 -0
- data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +7 -0
- data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +8 -0
- data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +9 -0
- data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +1 -0
- data/docs/_sass/vendor/neat/_neat-helpers.scss +11 -0
- data/docs/_sass/vendor/neat/_neat.scss +23 -0
- data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +49 -0
- data/docs/_sass/vendor/neat/functions/_private.scss +114 -0
- data/docs/_sass/vendor/neat/grid/_box-sizing.scss +15 -0
- data/docs/_sass/vendor/neat/grid/_direction-context.scss +33 -0
- data/docs/_sass/vendor/neat/grid/_display-context.scss +28 -0
- data/docs/_sass/vendor/neat/grid/_fill-parent.scss +22 -0
- data/docs/_sass/vendor/neat/grid/_media.scss +92 -0
- data/docs/_sass/vendor/neat/grid/_omega.scss +87 -0
- data/docs/_sass/vendor/neat/grid/_outer-container.scss +34 -0
- data/docs/_sass/vendor/neat/grid/_pad.scss +25 -0
- data/docs/_sass/vendor/neat/grid/_private.scss +35 -0
- data/docs/_sass/vendor/neat/grid/_row.scss +52 -0
- data/docs/_sass/vendor/neat/grid/_shift.scss +50 -0
- data/docs/_sass/vendor/neat/grid/_span-columns.scss +94 -0
- data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +97 -0
- data/docs/_sass/vendor/neat/grid/_visual-grid.scss +42 -0
- data/docs/_sass/vendor/neat/mixins/_clearfix.scss +25 -0
- data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +13 -0
- data/docs/_sass/vendor/neat/settings/_grid.scss +51 -0
- data/docs/_sass/vendor/neat/settings/_visual-grid.scss +27 -0
- data/docs/_sass/vendor/normalize-3.0.2.scss +427 -0
- data/docs/_sass/vendor/pygments.scss +356 -0
- data/docs/automating_browsers/capybara.md +70 -0
- data/docs/css/screen.scss +7 -0
- data/docs/guides/callbacks.md +45 -0
- data/docs/guides/cli.md +52 -0
- data/docs/guides/configuration.md +184 -0
- data/docs/guides/error_handling.md +46 -0
- data/docs/guides/frontiers.md +93 -0
- data/docs/guides/halting.md +23 -0
- data/docs/guides/job_queues.md +26 -0
- data/docs/guides/locals.md +36 -0
- data/docs/guides/logging.md +22 -0
- data/docs/guides/page_objects.md +67 -0
- data/docs/guides/peeking.md +46 -0
- data/docs/guides/selenium_capybara.md +100 -0
- data/docs/guides/tutorial.md +452 -0
- data/docs/index.md +82 -0
- data/docs/js/navigation.js +11 -0
- data/docs/misc/contributing.md +20 -0
- data/docs/misc/testing.md +11 -0
- data/docs/recipes/authentication.md +23 -0
- data/docs/recipes/csv.md +29 -0
- data/docs/recipes/javascript.md +20 -0
- data/docs/recipes/multiple_uris.md +18 -0
- data/docs/recipes/screenshots.md +20 -0
- data/docs/routing/custom_rules.md +16 -0
- data/docs/routing/filetypes_rules.md +21 -0
- data/docs/routing/host_rules.md +24 -0
- data/docs/routing/path_rules.md +33 -0
- data/docs/routing/protocol_rules.md +17 -0
- data/docs/routing/query_rules.md +69 -0
- data/docs/routing/routes.md +96 -0
- data/docs/routing/uri_rules.md +18 -0
- data/examples/collect_github_issues.rb +65 -0
- data/examples/find_foobar_on_wikipedia.rb +23 -0
- data/lib/wayfarer/configuration.rb +86 -0
- data/lib/wayfarer/crawl.rb +79 -0
- data/lib/wayfarer/crawl_observer.rb +103 -0
- data/lib/wayfarer/dispatcher.rb +104 -0
- data/lib/wayfarer/finders.rb +61 -0
- data/lib/wayfarer/frontiers/frontier.rb +79 -0
- data/lib/wayfarer/frontiers/memory_bloomfilter.rb +32 -0
- data/lib/wayfarer/frontiers/memory_frontier.rb +76 -0
- data/lib/wayfarer/frontiers/memory_trie_frontier.rb +39 -0
- data/lib/wayfarer/frontiers/normalize_uris.rb +48 -0
- data/lib/wayfarer/frontiers/redis_bloomfilter.rb +34 -0
- data/lib/wayfarer/frontiers/redis_frontier.rb +83 -0
- data/lib/wayfarer/http_adapters/adapter_pool.rb +62 -0
- data/lib/wayfarer/http_adapters/net_http_adapter.rb +77 -0
- data/lib/wayfarer/http_adapters/selenium_adapter.rb +80 -0
- data/lib/wayfarer/job.rb +211 -0
- data/lib/wayfarer/locals.rb +40 -0
- data/lib/wayfarer/page.rb +94 -0
- data/lib/wayfarer/parsers/json_parser.rb +20 -0
- data/lib/wayfarer/parsers/xml_parser.rb +27 -0
- data/lib/wayfarer/processor.rb +103 -0
- data/lib/wayfarer/routing/custom_rule.rb +21 -0
- data/lib/wayfarer/routing/filetypes_rule.rb +20 -0
- data/lib/wayfarer/routing/host_rule.rb +19 -0
- data/lib/wayfarer/routing/path_rule.rb +54 -0
- data/lib/wayfarer/routing/protocol_rule.rb +21 -0
- data/lib/wayfarer/routing/query_rule.rb +59 -0
- data/lib/wayfarer/routing/router.rb +71 -0
- data/lib/wayfarer/routing/rule.rb +114 -0
- data/lib/wayfarer/routing/uri_rule.rb +21 -0
- data/lib/wayfarer.rb +68 -0
- data/spec/configuration_spec.rb +26 -0
- data/spec/crawl_spec.rb +48 -0
- data/spec/finders_spec.rb +49 -0
- data/spec/frontiers/memory_bloomfilter_spec.rb +6 -0
- data/spec/frontiers/memory_frontier_spec.rb +6 -0
- data/spec/frontiers/memory_trie_frontier_spec.rb +6 -0
- data/spec/frontiers/normalize_uris_spec.rb +59 -0
- data/spec/frontiers/redis_bloomfilter_spec.rb +6 -0
- data/spec/frontiers/redis_frontier_spec.rb +6 -0
- data/spec/http_adapters/adapter_pool_spec.rb +33 -0
- data/spec/http_adapters/net_http_adapter_spec.rb +83 -0
- data/spec/http_adapters/selenium_adapter_spec.rb +53 -0
- data/spec/integration/callbacks_spec.rb +42 -0
- data/spec/integration/locals_spec.rb +106 -0
- data/spec/integration/peeking_spec.rb +61 -0
- data/spec/job_spec.rb +122 -0
- data/spec/page_spec.rb +38 -0
- data/spec/parsers/json_parser_spec.rb +30 -0
- data/spec/parsers/xml_parser_spec.rb +24 -0
- data/spec/processor_spec.rb +31 -0
- data/spec/routing/custom_rule_spec.rb +26 -0
- data/spec/routing/filetypes_rule_spec.rb +40 -0
- data/spec/routing/host_rule_spec.rb +48 -0
- data/spec/routing/path_rule_spec.rb +66 -0
- data/spec/routing/protocol_rule_spec.rb +26 -0
- data/spec/routing/query_rule_spec.rb +124 -0
- data/spec/routing/router_spec.rb +67 -0
- data/spec/routing/rule_spec.rb +251 -0
- data/spec/routing/uri_rule_spec.rb +24 -0
- data/spec/shared/frontier.rb +96 -0
- data/spec/spec_helpers.rb +62 -0
- data/spec/wayfarer_spec.rb +24 -0
- data/support/static/finders.html +38 -0
- data/support/static/graph/details/a.html +10 -0
- data/support/static/graph/details/b.html +10 -0
- data/support/static/graph/index.html +20 -0
- data/support/static/json/dummy.json +13 -0
- data/support/static/links/links.html +28 -0
- data/support/static/xml/dummy.xml +120 -0
- data/support/test_app.rb +45 -0
- data/wayfarer-jruby.gemspec +49 -0
- data/wayfarer.gemspec +53 -0
- metadata +697 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>Things</title>
|
|
5
|
+
|
|
6
|
+
<!-- Stylesheet -->
|
|
7
|
+
<link rel="stylesheet" href="/a.css" id="stylesheet-a">
|
|
8
|
+
<link rel="stylesheet" href="b.css" id="stylesheet-b">
|
|
9
|
+
<link rel="stylesheet" href="http://google.com/c.css" id="stylesheet-c">
|
|
10
|
+
|
|
11
|
+
<!-- Duplicate stylesheets -->
|
|
12
|
+
<link rel="stylesheet" href="/a.css">
|
|
13
|
+
<link rel="stylesheet" href="b.css">
|
|
14
|
+
<link rel="stylesheet" href="http://google.com/c.css">
|
|
15
|
+
</head>
|
|
16
|
+
|
|
17
|
+
<body>
|
|
18
|
+
<ul>
|
|
19
|
+
<li><a href="/foo">Foo</a></li>
|
|
20
|
+
<li><a href="/bar">Bar</a></li>
|
|
21
|
+
<li><a href="/baz">Baz</a></li>
|
|
22
|
+
|
|
23
|
+
<li><a href="foo">Foo</a></li>
|
|
24
|
+
<li><a href="bar">Bar</a></li>
|
|
25
|
+
<li><a href="baz">Baz</a></li>
|
|
26
|
+
|
|
27
|
+
<!-- Duplicates -->
|
|
28
|
+
<li><a href="foo">Foo</a></li>
|
|
29
|
+
<li><a href="bar">Bar</a></li>
|
|
30
|
+
<li><a href="baz">Baz</a></li>
|
|
31
|
+
|
|
32
|
+
<!-- Absolute links -->
|
|
33
|
+
<li><a href="http://google.com">Foobar</a></li>
|
|
34
|
+
<li><a href="http://yahoo.com">Yahoo</a></li>
|
|
35
|
+
<li><a href="http://aol.com">AOL</a></li>
|
|
36
|
+
</ul>
|
|
37
|
+
</body>
|
|
38
|
+
</html>
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>Index</title>
|
|
5
|
+
</head>
|
|
6
|
+
|
|
7
|
+
<body>
|
|
8
|
+
<a href="details/a.html">Detail A</a>
|
|
9
|
+
<a href="details/b.html">Detail B</a>
|
|
10
|
+
|
|
11
|
+
<!-- Erroneous links -->
|
|
12
|
+
<a href="/status_code/400">400</a>
|
|
13
|
+
<a href="/status_code/403">403</a>
|
|
14
|
+
<a href="/status_code/404">404</a>
|
|
15
|
+
<a href="http://bro.ken">Broken</a>
|
|
16
|
+
|
|
17
|
+
<!-- Redirect loop -->
|
|
18
|
+
<a href="/redirect_loop">Redirect loop</a>
|
|
19
|
+
</body>
|
|
20
|
+
</html>
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>Things</title>
|
|
5
|
+
</head>
|
|
6
|
+
|
|
7
|
+
<body>
|
|
8
|
+
<ul>
|
|
9
|
+
<li><a href="/foo">Foo</a></li>
|
|
10
|
+
<li><a href="/bar">Bar</a></li>
|
|
11
|
+
<li><a href="/baz">Baz</a></li>
|
|
12
|
+
|
|
13
|
+
<li><a href="foo">Foo</a></li>
|
|
14
|
+
<li><a href="bar">Bar</a></li>
|
|
15
|
+
<li><a href="baz">Baz</a></li>
|
|
16
|
+
|
|
17
|
+
<!-- Duplicates -->
|
|
18
|
+
<li><a href="foo">Foo</a></li>
|
|
19
|
+
<li><a href="bar">Bar</a></li>
|
|
20
|
+
<li><a href="baz">Baz</a></li>
|
|
21
|
+
|
|
22
|
+
<!-- Absolute links -->
|
|
23
|
+
<li><a href="http://google.com">Foobar</a></li>
|
|
24
|
+
<li><a href="http://yahoo.com">Yahoo</a></li>
|
|
25
|
+
<li><a href="http://aol.com">AOL</a></li>
|
|
26
|
+
</ul>
|
|
27
|
+
</body>
|
|
28
|
+
</html>
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<catalog>
|
|
3
|
+
<book id="bk101">
|
|
4
|
+
<author>Gambardella, Matthew</author>
|
|
5
|
+
<title>XML Developer's Guide</title>
|
|
6
|
+
<genre>Computer</genre>
|
|
7
|
+
<price>44.95</price>
|
|
8
|
+
<publish_date>2000-10-01</publish_date>
|
|
9
|
+
<description>An in-depth look at creating applications
|
|
10
|
+
with XML.</description>
|
|
11
|
+
</book>
|
|
12
|
+
<book id="bk102">
|
|
13
|
+
<author>Ralls, Kim</author>
|
|
14
|
+
<title>Midnight Rain</title>
|
|
15
|
+
<genre>Fantasy</genre>
|
|
16
|
+
<price>5.95</price>
|
|
17
|
+
<publish_date>2000-12-16</publish_date>
|
|
18
|
+
<description>A former architect battles corporate zombies,
|
|
19
|
+
an evil sorceress, and her own childhood to become queen
|
|
20
|
+
of the world.</description>
|
|
21
|
+
</book>
|
|
22
|
+
<book id="bk103">
|
|
23
|
+
<author>Corets, Eva</author>
|
|
24
|
+
<title>Maeve Ascendant</title>
|
|
25
|
+
<genre>Fantasy</genre>
|
|
26
|
+
<price>5.95</price>
|
|
27
|
+
<publish_date>2000-11-17</publish_date>
|
|
28
|
+
<description>After the collapse of a nanotechnology
|
|
29
|
+
society in England, the young survivors lay the
|
|
30
|
+
foundation for a new society.</description>
|
|
31
|
+
</book>
|
|
32
|
+
<book id="bk104">
|
|
33
|
+
<author>Corets, Eva</author>
|
|
34
|
+
<title>Oberon's Legacy</title>
|
|
35
|
+
<genre>Fantasy</genre>
|
|
36
|
+
<price>5.95</price>
|
|
37
|
+
<publish_date>2001-03-10</publish_date>
|
|
38
|
+
<description>In post-apocalypse England, the mysterious
|
|
39
|
+
agent known only as Oberon helps to create a new life
|
|
40
|
+
for the inhabitants of London. Sequel to Maeve
|
|
41
|
+
Ascendant.</description>
|
|
42
|
+
</book>
|
|
43
|
+
<book id="bk105">
|
|
44
|
+
<author>Corets, Eva</author>
|
|
45
|
+
<title>The Sundered Grail</title>
|
|
46
|
+
<genre>Fantasy</genre>
|
|
47
|
+
<price>5.95</price>
|
|
48
|
+
<publish_date>2001-09-10</publish_date>
|
|
49
|
+
<description>The two daughters of Maeve, half-sisters,
|
|
50
|
+
battle one another for control of England. Sequel to
|
|
51
|
+
Oberon's Legacy.</description>
|
|
52
|
+
</book>
|
|
53
|
+
<book id="bk106">
|
|
54
|
+
<author>Randall, Cynthia</author>
|
|
55
|
+
<title>Lover Birds</title>
|
|
56
|
+
<genre>Romance</genre>
|
|
57
|
+
<price>4.95</price>
|
|
58
|
+
<publish_date>2000-09-02</publish_date>
|
|
59
|
+
<description>When Carla meets Paul at an ornithology
|
|
60
|
+
conference, tempers fly as feathers get ruffled.</description>
|
|
61
|
+
</book>
|
|
62
|
+
<book id="bk107">
|
|
63
|
+
<author>Thurman, Paula</author>
|
|
64
|
+
<title>Splish Splash</title>
|
|
65
|
+
<genre>Romance</genre>
|
|
66
|
+
<price>4.95</price>
|
|
67
|
+
<publish_date>2000-11-02</publish_date>
|
|
68
|
+
<description>A deep sea diver finds true love twenty
|
|
69
|
+
thousand leagues beneath the sea.</description>
|
|
70
|
+
</book>
|
|
71
|
+
<book id="bk108">
|
|
72
|
+
<author>Knorr, Stefan</author>
|
|
73
|
+
<title>Creepy Crawlies</title>
|
|
74
|
+
<genre>Horror</genre>
|
|
75
|
+
<price>4.95</price>
|
|
76
|
+
<publish_date>2000-12-06</publish_date>
|
|
77
|
+
<description>An anthology of horror stories about roaches,
|
|
78
|
+
centipedes, scorpions and other insects.</description>
|
|
79
|
+
</book>
|
|
80
|
+
<book id="bk109">
|
|
81
|
+
<author>Kress, Peter</author>
|
|
82
|
+
<title>Paradox Lost</title>
|
|
83
|
+
<genre>Science Fiction</genre>
|
|
84
|
+
<price>6.95</price>
|
|
85
|
+
<publish_date>2000-11-02</publish_date>
|
|
86
|
+
<description>After an inadvertant trip through a Heisenberg
|
|
87
|
+
Uncertainty Device, James Salway discovers the problems
|
|
88
|
+
of being quantum.</description>
|
|
89
|
+
</book>
|
|
90
|
+
<book id="bk110">
|
|
91
|
+
<author>O'Brien, Tim</author>
|
|
92
|
+
<title>Microsoft .NET: The Programming Bible</title>
|
|
93
|
+
<genre>Computer</genre>
|
|
94
|
+
<price>36.95</price>
|
|
95
|
+
<publish_date>2000-12-09</publish_date>
|
|
96
|
+
<description>Microsoft's .NET initiative is explored in
|
|
97
|
+
detail in this deep programmer's reference.</description>
|
|
98
|
+
</book>
|
|
99
|
+
<book id="bk111">
|
|
100
|
+
<author>O'Brien, Tim</author>
|
|
101
|
+
<title>MSXML3: A Comprehensive Guide</title>
|
|
102
|
+
<genre>Computer</genre>
|
|
103
|
+
<price>36.95</price>
|
|
104
|
+
<publish_date>2000-12-01</publish_date>
|
|
105
|
+
<description>The Microsoft MSXML3 parser is covered in
|
|
106
|
+
detail, with attention to XML DOM interfaces, XSLT processing,
|
|
107
|
+
SAX and more.</description>
|
|
108
|
+
</book>
|
|
109
|
+
<book id="bk112">
|
|
110
|
+
<author>Galos, Mike</author>
|
|
111
|
+
<title>Visual Studio 7: A Comprehensive Guide</title>
|
|
112
|
+
<genre>Computer</genre>
|
|
113
|
+
<price>49.95</price>
|
|
114
|
+
<publish_date>2001-04-16</publish_date>
|
|
115
|
+
<description>Microsoft Visual Studio 7 is explored in depth,
|
|
116
|
+
looking at how Visual Basic, Visual C++, C#, and ASP+ are
|
|
117
|
+
integrated into a comprehensive development
|
|
118
|
+
environment.</description>
|
|
119
|
+
</book>
|
|
120
|
+
</catalog>
|
data/support/test_app.rb
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "sinatra"
|
|
3
|
+
|
|
4
|
+
class TestApp < Sinatra::Base
|
|
5
|
+
set :root, File.dirname(__FILE__)
|
|
6
|
+
set :public_folder, -> { File.join(root, "static") }
|
|
7
|
+
|
|
8
|
+
get "/status_code/:code" do
|
|
9
|
+
status params[:code]
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
get "/hello_world" do
|
|
13
|
+
headers "hello" => "world"
|
|
14
|
+
"Hello world!"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
get "/redirect_loop" do
|
|
18
|
+
redirect to "/redirect_loop"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
get "/redirect" do
|
|
22
|
+
n = params[:times].to_i
|
|
23
|
+
n.zero? ? "You arrived!" : (redirect to "/redirect?times=#{n - 1}")
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
get "/malformed_redirect" do
|
|
27
|
+
redirect to "hptt://bro.ken"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
get "/json/:file" do
|
|
31
|
+
content_type "application/json"
|
|
32
|
+
send_file(static_file("json/#{params[:file]}"))
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
get "/xml/:file" do
|
|
36
|
+
content_type "application/xml"
|
|
37
|
+
send_file(static_file("xml/#{params[:file]}"))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def static_file(file_path)
|
|
43
|
+
File.expand_path(file_path, settings.public_folder)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Gem::Specification.new do |s|
|
|
4
|
+
s.name = "wayfarer-jruby"
|
|
5
|
+
s.version = "0.0.3"
|
|
6
|
+
s.license = "MIT"
|
|
7
|
+
|
|
8
|
+
s.homepage = "http://github.com/bauerd/wayfarer"
|
|
9
|
+
s.description = "Versatile web crawling with JRuby"
|
|
10
|
+
s.summary = s.description
|
|
11
|
+
|
|
12
|
+
s.date = "2017-05-31"
|
|
13
|
+
s.authors = ["Dominic Bauer"]
|
|
14
|
+
s.email = "bauerdominic@gmail.com"
|
|
15
|
+
|
|
16
|
+
s.files = `git ls-files`.split("\n")
|
|
17
|
+
s.test_files = `git ls-files -- {spec}/*`.split("\n")
|
|
18
|
+
|
|
19
|
+
s.require_paths = ["lib"]
|
|
20
|
+
|
|
21
|
+
s.executables << "wayfarer"
|
|
22
|
+
|
|
23
|
+
s.add_runtime_dependency "connection_pool", "~> 2.2"
|
|
24
|
+
s.add_runtime_dependency "nokogiri", "~> 1.6"
|
|
25
|
+
s.add_runtime_dependency "selenium-webdriver", "~> 3.4"
|
|
26
|
+
s.add_runtime_dependency "selenium-emulated_features", "2.0.1"
|
|
27
|
+
s.add_runtime_dependency "net-http-persistent", "~> 2.9"
|
|
28
|
+
s.add_runtime_dependency "mime-types", "~> 2.6"
|
|
29
|
+
s.add_runtime_dependency "mustermann", "~> 0.4"
|
|
30
|
+
s.add_runtime_dependency "activejob", "~> 4.2"
|
|
31
|
+
s.add_runtime_dependency "thor", "~> 0.19"
|
|
32
|
+
s.add_runtime_dependency "activesupport", "~> 4.2"
|
|
33
|
+
s.add_runtime_dependency "capybara", "~> 2.5"
|
|
34
|
+
s.add_runtime_dependency "chronic", "~> 0.10"
|
|
35
|
+
s.add_runtime_dependency "redis", "~> 3.2"
|
|
36
|
+
s.add_runtime_dependency "concurrent-ruby", "~> 1.0"
|
|
37
|
+
s.add_runtime_dependency "hooks", "~> 0.4"
|
|
38
|
+
s.add_runtime_dependency "normalize_url", "~> 0.0.6"
|
|
39
|
+
|
|
40
|
+
s.add_development_dependency "pry", "~> 0.10"
|
|
41
|
+
s.add_development_dependency "rake", "~> 10.4"
|
|
42
|
+
s.add_development_dependency "rspec", "~> 3.3"
|
|
43
|
+
s.add_development_dependency "sinatra", "~> 1.4"
|
|
44
|
+
s.add_development_dependency "rubocop", "~> 0.34"
|
|
45
|
+
s.add_development_dependency "yard", "~> 0.9"
|
|
46
|
+
s.add_development_dependency "bourbon", "~> 4.3"
|
|
47
|
+
s.add_development_dependency "neat", "~> 1.8"
|
|
48
|
+
s.add_development_dependency "faker", "~> 1.7"
|
|
49
|
+
end
|
data/wayfarer.gemspec
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
Gem::Specification.new do |s|
|
|
3
|
+
s.name = "wayfarer"
|
|
4
|
+
s.version = "0.0.3"
|
|
5
|
+
s.license = "MIT"
|
|
6
|
+
|
|
7
|
+
s.homepage = "http://github.com/bauerd/wayfarer"
|
|
8
|
+
s.description = "Versatile web crawling with Ruby"
|
|
9
|
+
s.summary = s.description
|
|
10
|
+
|
|
11
|
+
s.date = "2017-05-31"
|
|
12
|
+
s.authors = ["Dominic Bauer"]
|
|
13
|
+
s.email = "bauerdominic@gmail.com"
|
|
14
|
+
|
|
15
|
+
s.files = `git ls-files`.split("\n")
|
|
16
|
+
s.test_files = `git ls-files -- {spec}/*`.split("\n")
|
|
17
|
+
|
|
18
|
+
s.require_paths = ["lib"]
|
|
19
|
+
|
|
20
|
+
s.executables << "wayfarer"
|
|
21
|
+
|
|
22
|
+
s.add_runtime_dependency "connection_pool", "~> 2.2"
|
|
23
|
+
s.add_runtime_dependency "nokogiri", "~> 1.6"
|
|
24
|
+
s.add_runtime_dependency "selenium-webdriver", "~> 3.4"
|
|
25
|
+
s.add_runtime_dependency "selenium-emulated_features", "2.0.1"
|
|
26
|
+
s.add_runtime_dependency "net-http-persistent", "~> 2.9"
|
|
27
|
+
s.add_runtime_dependency "mime-types", "~> 2.6"
|
|
28
|
+
s.add_runtime_dependency "pismo", "~> 0.7"
|
|
29
|
+
s.add_runtime_dependency "mustermann", "~> 0.4"
|
|
30
|
+
s.add_runtime_dependency "activejob", "~> 4.2"
|
|
31
|
+
s.add_runtime_dependency "thor", "~> 0.19"
|
|
32
|
+
s.add_runtime_dependency "activesupport", "~> 4.2"
|
|
33
|
+
s.add_runtime_dependency "capybara", "~> 2.5"
|
|
34
|
+
s.add_runtime_dependency "chronic", "~> 0.10"
|
|
35
|
+
s.add_runtime_dependency "redis", "~> 3.2"
|
|
36
|
+
s.add_runtime_dependency "concurrent-ruby", "~> 1.0"
|
|
37
|
+
s.add_runtime_dependency "hooks", "~> 0.4"
|
|
38
|
+
s.add_runtime_dependency "bloomfilter-rb", "~> 2.1"
|
|
39
|
+
s.add_runtime_dependency "oj", "~> 2.12"
|
|
40
|
+
s.add_runtime_dependency "fast_trie", "~> 0.5"
|
|
41
|
+
s.add_runtime_dependency "parallel", "~> 1.11"
|
|
42
|
+
s.add_runtime_dependency "normalize_url", "~> 0.0.6"
|
|
43
|
+
|
|
44
|
+
s.add_development_dependency "pry", "~> 0.10"
|
|
45
|
+
s.add_development_dependency "rake", "~> 10.4"
|
|
46
|
+
s.add_development_dependency "rspec", "~> 3.3"
|
|
47
|
+
s.add_development_dependency "sinatra", "~> 1.4"
|
|
48
|
+
s.add_development_dependency "rubocop", "~> 0.34"
|
|
49
|
+
s.add_development_dependency "yard", "~> 0.9"
|
|
50
|
+
s.add_development_dependency "bourbon", "~> 4.3"
|
|
51
|
+
s.add_development_dependency "neat", "~> 1.8"
|
|
52
|
+
s.add_development_dependency "faker", "~> 1.7"
|
|
53
|
+
end
|