wayfarer 0.0.3 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/ci.yaml +32 -0
- data/.gitignore +3 -4
- data/.rubocop.yml +25 -9
- data/.ruby-version +1 -1
- data/Dockerfile +5 -0
- data/Gemfile +1 -7
- data/Gemfile.lock +212 -0
- data/RELEASING.md +17 -0
- data/Rakefile +38 -90
- data/bin/wayfarer +1 -111
- data/docker-compose.yml +32 -0
- data/docs/cookbook/querying_html.md +42 -0
- data/docs/cookbook/screenshots.md +27 -0
- data/docs/cookbook/user_agent.md +7 -0
- data/docs/guides/browser_automation/capybara.md +3 -0
- data/docs/guides/browser_automation/ferrum.md +37 -0
- data/docs/guides/browser_automation/selenium.md +59 -0
- data/docs/guides/callbacks.md +27 -34
- data/docs/guides/configuration.md +20 -171
- data/docs/guides/error_handling.md +18 -33
- data/docs/guides/jobs.md +75 -0
- data/docs/guides/networking.md +20 -0
- data/docs/guides/pages.md +52 -0
- data/docs/guides/performance.md +22 -0
- data/docs/guides/tasks.md +38 -0
- data/docs/index.md +34 -68
- data/docs/reference/api/base.md +162 -0
- data/docs/reference/api/route.md +182 -0
- data/docs/reference/cli.md +139 -0
- data/docs/reference/environment_variables.md +85 -0
- data/lib/wayfarer/base.rb +60 -0
- data/lib/wayfarer/cli/base.rb +23 -0
- data/lib/wayfarer/cli/generate.rb +17 -0
- data/lib/wayfarer/cli/job.rb +58 -0
- data/lib/wayfarer/cli/route.rb +27 -0
- data/lib/wayfarer/cli/route_printer.rb +116 -0
- data/lib/wayfarer/cli/runner.rb +34 -0
- data/lib/wayfarer/cli/templates/Gemfile.tt +5 -0
- data/lib/wayfarer/cli/templates/job.rb.tt +8 -0
- data/lib/wayfarer/config.rb +67 -0
- data/lib/wayfarer/gc.rb +19 -0
- data/lib/wayfarer/middleware/chain.rb +19 -0
- data/lib/wayfarer/middleware/dedup.rb +25 -0
- data/lib/wayfarer/middleware/fetch.rb +32 -0
- data/lib/wayfarer/middleware/normalize.rb +25 -0
- data/lib/wayfarer/middleware/router.rb +21 -0
- data/lib/wayfarer/middleware/stage.rb +23 -0
- data/lib/wayfarer/middleware/worker.rb +47 -0
- data/lib/wayfarer/networking/ferrum.rb +70 -0
- data/lib/wayfarer/networking/healer.rb +21 -0
- data/lib/wayfarer/networking/net_http.rb +52 -0
- data/lib/wayfarer/networking/pool.rb +34 -0
- data/lib/wayfarer/networking/result.rb +18 -0
- data/lib/wayfarer/networking/selenium.rb +70 -0
- data/lib/wayfarer/page.rb +15 -71
- data/lib/wayfarer/parsing/json.rb +17 -0
- data/lib/wayfarer/parsing/xml.rb +17 -0
- data/lib/wayfarer/redis/barrier.rb +36 -0
- data/lib/wayfarer/redis/connection.rb +13 -0
- data/lib/wayfarer/redis/counter.rb +29 -0
- data/lib/wayfarer/redis/pool.rb +18 -0
- data/lib/wayfarer/redis/version.rb +19 -0
- data/lib/wayfarer/routing/custom_matcher.rb +21 -0
- data/lib/wayfarer/routing/dsl.rb +57 -0
- data/lib/wayfarer/routing/host_matcher.rb +23 -0
- data/lib/wayfarer/routing/path_finder.rb +46 -0
- data/lib/wayfarer/routing/path_matcher.rb +46 -0
- data/lib/wayfarer/routing/{query_rule.rb → query_matcher.rb} +24 -16
- data/lib/wayfarer/routing/result.rb +15 -0
- data/lib/wayfarer/routing/root_route.rb +7 -0
- data/lib/wayfarer/routing/route.rb +41 -0
- data/lib/wayfarer/routing/scheme_matcher.rb +21 -0
- data/lib/wayfarer/routing/suffix_matcher.rb +21 -0
- data/lib/wayfarer/routing/target_route.rb +7 -0
- data/lib/wayfarer/routing/url_matcher.rb +21 -0
- data/lib/wayfarer/serializer.rb +17 -0
- data/lib/wayfarer/stringify.rb +41 -0
- data/lib/wayfarer/task.rb +34 -0
- data/lib/wayfarer.rb +47 -58
- data/mkdocs.yml +47 -0
- data/requirements.txt +1 -0
- data/spec/base_spec.rb +219 -0
- data/spec/cli/generate_spec.rb +39 -0
- data/spec/cli/job_spec.rb +74 -0
- data/spec/cli/version_spec.rb +13 -0
- data/spec/config_spec.rb +144 -0
- data/spec/factories/queue/chain.rb +11 -0
- data/spec/factories/queue/middleware.rb +15 -0
- data/spec/factories/queue/page.rb +78 -0
- data/spec/factories/queue/task.rb +12 -0
- data/spec/fixtures/dummy_job.rb +7 -0
- data/spec/gc_spec.rb +61 -0
- data/spec/middleware/chain_spec.rb +96 -0
- data/spec/middleware/dedup_spec.rb +76 -0
- data/spec/middleware/fetch_spec.rb +72 -0
- data/spec/middleware/normalize_spec.rb +28 -0
- data/spec/middleware/router_spec.rb +46 -0
- data/spec/middleware/stage_spec.rb +39 -0
- data/spec/middleware/worker_spec.rb +90 -0
- data/spec/networking/adapter.rb +135 -0
- data/spec/networking/ferrum_spec.rb +28 -0
- data/spec/networking/healer_spec.rb +46 -0
- data/spec/networking/net_http_spec.rb +37 -0
- data/spec/networking/pool_spec.rb +42 -0
- data/spec/networking/selenium_spec.rb +28 -0
- data/spec/page_spec.rb +21 -12
- data/spec/{parsers/json_parser_spec.rb → parsing/json_spec.rb} +5 -4
- data/spec/{parsers/xml_parser_spec.rb → parsing/xml_spec.rb} +3 -2
- data/spec/redis/barrier_spec.rb +78 -0
- data/spec/redis/counter_spec.rb +32 -0
- data/spec/redis/pool_spec.rb +18 -0
- data/spec/redis/version_spec.rb +13 -0
- data/spec/routing/custom_matcher_spec.rb +31 -0
- data/spec/routing/dsl_spec.rb +98 -0
- data/spec/routing/host_matcher_spec.rb +49 -0
- data/spec/routing/integration_spec.rb +110 -0
- data/spec/routing/path_finder_spec.rb +33 -0
- data/spec/routing/path_matcher_spec.rb +43 -0
- data/spec/routing/{query_rule_spec.rb → query_matcher_spec.rb} +39 -26
- data/spec/routing/root_route_spec.rb +29 -0
- data/spec/routing/route_spec.rb +43 -0
- data/spec/routing/scheme_matcher_spec.rb +25 -0
- data/spec/routing/{filetypes_rule_spec.rb → suffix_matcher_spec.rb} +14 -13
- data/spec/routing/uri_matcher_spec.rb +27 -0
- data/spec/spec_helpers.rb +65 -38
- data/spec/stringify_spec.rb +23 -0
- data/{support → spec/support}/static/finders.html +0 -0
- data/{support → spec/support}/static/graph/details/a.html +0 -0
- data/{support → spec/support}/static/graph/details/b.html +0 -0
- data/{support → spec/support}/static/graph/index.html +0 -0
- data/{support → spec/support}/static/json/dummy.json +0 -0
- data/{support → spec/support}/static/links/links.html +0 -0
- data/{support → spec/support}/static/xml/dummy.xml +0 -0
- data/{support → spec/support}/test_app.rb +9 -2
- data/spec/task_spec.rb +27 -0
- data/spec/wayfarer_spec.rb +2 -13
- data/wayfarer.gemspec +39 -42
- metadata +191 -368
- data/.travis.yml +0 -5
- data/Changelog.md +0 -10
- data/README.md +0 -21
- data/benchmark/frontiers.rb +0 -143
- data/docs/.gitignore +0 -2
- data/docs/_config.yml +0 -15
- data/docs/_includes/base.html +0 -7
- data/docs/_includes/head.html +0 -10
- data/docs/_includes/navigation.html +0 -187
- data/docs/_layouts/default.html +0 -42
- data/docs/_sass/base.scss +0 -439
- data/docs/_sass/variables.scss +0 -24
- data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +0 -19
- data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +0 -425
- data/docs/_sass/vendor/bourbon/_bourbon.scss +0 -90
- data/docs/_sass/vendor/bourbon/addons/_border-color.scss +0 -29
- data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +0 -48
- data/docs/_sass/vendor/bourbon/addons/_border-style.scss +0 -28
- data/docs/_sass/vendor/bourbon/addons/_border-width.scss +0 -28
- data/docs/_sass/vendor/bourbon/addons/_buttons.scss +0 -69
- data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +0 -25
- data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +0 -30
- data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +0 -31
- data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +0 -27
- data/docs/_sass/vendor/bourbon/addons/_margin.scss +0 -29
- data/docs/_sass/vendor/bourbon/addons/_padding.scss +0 -29
- data/docs/_sass/vendor/bourbon/addons/_position.scss +0 -51
- data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +0 -66
- data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +0 -27
- data/docs/_sass/vendor/bourbon/addons/_size.scss +0 -56
- data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +0 -118
- data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +0 -34
- data/docs/_sass/vendor/bourbon/addons/_triangle.scss +0 -63
- data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +0 -29
- data/docs/_sass/vendor/bourbon/css3/_animation.scss +0 -61
- data/docs/_sass/vendor/bourbon/css3/_appearance.scss +0 -5
- data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +0 -5
- data/docs/_sass/vendor/bourbon/css3/_background-image.scss +0 -44
- data/docs/_sass/vendor/bourbon/css3/_background.scss +0 -57
- data/docs/_sass/vendor/bourbon/css3/_border-image.scss +0 -61
- data/docs/_sass/vendor/bourbon/css3/_calc.scss +0 -6
- data/docs/_sass/vendor/bourbon/css3/_columns.scss +0 -67
- data/docs/_sass/vendor/bourbon/css3/_filter.scss +0 -6
- data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +0 -327
- data/docs/_sass/vendor/bourbon/css3/_font-face.scss +0 -29
- data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +0 -6
- data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +0 -12
- data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +0 -6
- data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +0 -15
- data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +0 -38
- data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +0 -40
- data/docs/_sass/vendor/bourbon/css3/_perspective.scss +0 -12
- data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +0 -10
- data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +0 -40
- data/docs/_sass/vendor/bourbon/css3/_selection.scss +0 -44
- data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +0 -27
- data/docs/_sass/vendor/bourbon/css3/_transform.scss +0 -21
- data/docs/_sass/vendor/bourbon/css3/_transition.scss +0 -81
- data/docs/_sass/vendor/bourbon/css3/_user-select.scss +0 -5
- data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +0 -16
- data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +0 -25
- data/docs/_sass/vendor/bourbon/functions/_contains.scss +0 -31
- data/docs/_sass/vendor/bourbon/functions/_is-length.scss +0 -16
- data/docs/_sass/vendor/bourbon/functions/_is-light.scss +0 -26
- data/docs/_sass/vendor/bourbon/functions/_is-number.scss +0 -16
- data/docs/_sass/vendor/bourbon/functions/_is-size.scss +0 -23
- data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +0 -74
- data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +0 -24
- data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +0 -26
- data/docs/_sass/vendor/bourbon/functions/_shade.scss +0 -24
- data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +0 -22
- data/docs/_sass/vendor/bourbon/functions/_tint.scss +0 -24
- data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +0 -37
- data/docs/_sass/vendor/bourbon/functions/_unpack.scss +0 -32
- data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +0 -26
- data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +0 -108
- data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +0 -53
- data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +0 -24
- data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +0 -35
- data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +0 -51
- data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +0 -77
- data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +0 -41
- data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +0 -74
- data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +0 -55
- data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +0 -28
- data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +0 -31
- data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +0 -15
- data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +0 -55
- data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +0 -7
- data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +0 -8
- data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +0 -9
- data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +0 -1
- data/docs/_sass/vendor/neat/_neat-helpers.scss +0 -11
- data/docs/_sass/vendor/neat/_neat.scss +0 -23
- data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +0 -49
- data/docs/_sass/vendor/neat/functions/_private.scss +0 -114
- data/docs/_sass/vendor/neat/grid/_box-sizing.scss +0 -15
- data/docs/_sass/vendor/neat/grid/_direction-context.scss +0 -33
- data/docs/_sass/vendor/neat/grid/_display-context.scss +0 -28
- data/docs/_sass/vendor/neat/grid/_fill-parent.scss +0 -22
- data/docs/_sass/vendor/neat/grid/_media.scss +0 -92
- data/docs/_sass/vendor/neat/grid/_omega.scss +0 -87
- data/docs/_sass/vendor/neat/grid/_outer-container.scss +0 -34
- data/docs/_sass/vendor/neat/grid/_pad.scss +0 -25
- data/docs/_sass/vendor/neat/grid/_private.scss +0 -35
- data/docs/_sass/vendor/neat/grid/_row.scss +0 -52
- data/docs/_sass/vendor/neat/grid/_shift.scss +0 -50
- data/docs/_sass/vendor/neat/grid/_span-columns.scss +0 -94
- data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +0 -97
- data/docs/_sass/vendor/neat/grid/_visual-grid.scss +0 -42
- data/docs/_sass/vendor/neat/mixins/_clearfix.scss +0 -25
- data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +0 -13
- data/docs/_sass/vendor/neat/settings/_grid.scss +0 -51
- data/docs/_sass/vendor/neat/settings/_visual-grid.scss +0 -27
- data/docs/_sass/vendor/normalize-3.0.2.scss +0 -427
- data/docs/_sass/vendor/pygments.scss +0 -356
- data/docs/automating_browsers/capybara.md +0 -70
- data/docs/css/screen.scss +0 -7
- data/docs/guides/cli.md +0 -52
- data/docs/guides/frontiers.md +0 -93
- data/docs/guides/halting.md +0 -23
- data/docs/guides/job_queues.md +0 -26
- data/docs/guides/locals.md +0 -36
- data/docs/guides/logging.md +0 -22
- data/docs/guides/page_objects.md +0 -67
- data/docs/guides/peeking.md +0 -46
- data/docs/guides/selenium_capybara.md +0 -100
- data/docs/guides/tutorial.md +0 -452
- data/docs/js/navigation.js +0 -11
- data/docs/misc/contributing.md +0 -20
- data/docs/misc/testing.md +0 -11
- data/docs/recipes/authentication.md +0 -23
- data/docs/recipes/csv.md +0 -29
- data/docs/recipes/javascript.md +0 -20
- data/docs/recipes/multiple_uris.md +0 -18
- data/docs/recipes/screenshots.md +0 -20
- data/docs/routing/custom_rules.md +0 -16
- data/docs/routing/filetypes_rules.md +0 -21
- data/docs/routing/host_rules.md +0 -24
- data/docs/routing/path_rules.md +0 -33
- data/docs/routing/protocol_rules.md +0 -17
- data/docs/routing/query_rules.md +0 -69
- data/docs/routing/routes.md +0 -96
- data/docs/routing/uri_rules.md +0 -18
- data/examples/collect_github_issues.rb +0 -65
- data/examples/find_foobar_on_wikipedia.rb +0 -23
- data/lib/wayfarer/configuration.rb +0 -86
- data/lib/wayfarer/crawl.rb +0 -79
- data/lib/wayfarer/crawl_observer.rb +0 -103
- data/lib/wayfarer/dispatcher.rb +0 -104
- data/lib/wayfarer/finders.rb +0 -61
- data/lib/wayfarer/frontiers/frontier.rb +0 -79
- data/lib/wayfarer/frontiers/memory_bloomfilter.rb +0 -32
- data/lib/wayfarer/frontiers/memory_frontier.rb +0 -76
- data/lib/wayfarer/frontiers/memory_trie_frontier.rb +0 -39
- data/lib/wayfarer/frontiers/normalize_uris.rb +0 -48
- data/lib/wayfarer/frontiers/redis_bloomfilter.rb +0 -34
- data/lib/wayfarer/frontiers/redis_frontier.rb +0 -83
- data/lib/wayfarer/http_adapters/adapter_pool.rb +0 -62
- data/lib/wayfarer/http_adapters/net_http_adapter.rb +0 -77
- data/lib/wayfarer/http_adapters/selenium_adapter.rb +0 -80
- data/lib/wayfarer/job.rb +0 -211
- data/lib/wayfarer/locals.rb +0 -40
- data/lib/wayfarer/parsers/json_parser.rb +0 -20
- data/lib/wayfarer/parsers/xml_parser.rb +0 -27
- data/lib/wayfarer/processor.rb +0 -103
- data/lib/wayfarer/routing/custom_rule.rb +0 -21
- data/lib/wayfarer/routing/filetypes_rule.rb +0 -20
- data/lib/wayfarer/routing/host_rule.rb +0 -19
- data/lib/wayfarer/routing/path_rule.rb +0 -54
- data/lib/wayfarer/routing/protocol_rule.rb +0 -21
- data/lib/wayfarer/routing/router.rb +0 -71
- data/lib/wayfarer/routing/rule.rb +0 -114
- data/lib/wayfarer/routing/uri_rule.rb +0 -21
- data/spec/configuration_spec.rb +0 -26
- data/spec/crawl_spec.rb +0 -48
- data/spec/finders_spec.rb +0 -49
- data/spec/frontiers/memory_bloomfilter_spec.rb +0 -6
- data/spec/frontiers/memory_frontier_spec.rb +0 -6
- data/spec/frontiers/memory_trie_frontier_spec.rb +0 -6
- data/spec/frontiers/normalize_uris_spec.rb +0 -59
- data/spec/frontiers/redis_bloomfilter_spec.rb +0 -6
- data/spec/frontiers/redis_frontier_spec.rb +0 -6
- data/spec/http_adapters/adapter_pool_spec.rb +0 -33
- data/spec/http_adapters/net_http_adapter_spec.rb +0 -83
- data/spec/http_adapters/selenium_adapter_spec.rb +0 -53
- data/spec/integration/callbacks_spec.rb +0 -42
- data/spec/integration/locals_spec.rb +0 -106
- data/spec/integration/peeking_spec.rb +0 -61
- data/spec/job_spec.rb +0 -122
- data/spec/processor_spec.rb +0 -31
- data/spec/routing/custom_rule_spec.rb +0 -26
- data/spec/routing/host_rule_spec.rb +0 -48
- data/spec/routing/path_rule_spec.rb +0 -66
- data/spec/routing/protocol_rule_spec.rb +0 -26
- data/spec/routing/router_spec.rb +0 -67
- data/spec/routing/rule_spec.rb +0 -251
- data/spec/routing/uri_rule_spec.rb +0 -24
- data/spec/shared/frontier.rb +0 -96
- data/wayfarer-jruby.gemspec +0 -49
@@ -1,25 +1,26 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require "spec_helpers"
|
3
4
|
|
4
|
-
describe Wayfarer::Routing::
|
5
|
-
subject(:
|
5
|
+
describe Wayfarer::Routing::QueryMatcher do
|
6
|
+
subject(:matcher) { Wayfarer::Routing::QueryMatcher.new(constraints) }
|
6
7
|
|
7
8
|
describe "String constraints" do
|
8
9
|
let(:constraints) { Hash[arg: "foo"] }
|
9
10
|
|
10
11
|
context "with matching query field value" do
|
11
|
-
let(:
|
12
|
+
let(:url) { URI("http://example.com?arg=foo") }
|
12
13
|
|
13
14
|
it "returns true" do
|
14
|
-
expect(
|
15
|
+
expect(matcher).to match(url)
|
15
16
|
end
|
16
17
|
end
|
17
18
|
|
18
19
|
context "with mismatching query field value" do
|
19
|
-
let(:
|
20
|
+
let(:url) { URI("http://example.com?arg=bar") }
|
20
21
|
|
21
22
|
it "returns false" do
|
22
|
-
expect(
|
23
|
+
expect(matcher).not_to match(url)
|
23
24
|
end
|
24
25
|
end
|
25
26
|
end
|
@@ -28,26 +29,26 @@ describe Wayfarer::Routing::QueryRule do
|
|
28
29
|
let(:constraints) { Hash[arg: 0] }
|
29
30
|
|
30
31
|
context "with matching query field value" do
|
31
|
-
let(:
|
32
|
+
let(:url) { URI("http://example.com?arg=0") }
|
32
33
|
|
33
34
|
it "returns true" do
|
34
|
-
expect(
|
35
|
+
expect(matcher).to match(url)
|
35
36
|
end
|
36
37
|
end
|
37
38
|
|
38
39
|
context "with mismatching query field value" do
|
39
|
-
let(:
|
40
|
+
let(:url) { URI("http://example.com?arg=1") }
|
40
41
|
|
41
42
|
it "returns false" do
|
42
|
-
expect(
|
43
|
+
expect(matcher).not_to match(url)
|
43
44
|
end
|
44
45
|
end
|
45
46
|
|
46
47
|
context "with non-numeric query field value" do
|
47
|
-
let(:
|
48
|
+
let(:url) { URI("http://example.com?arg=foo") }
|
48
49
|
|
49
50
|
it "returns false" do
|
50
|
-
expect(
|
51
|
+
expect(matcher).not_to match(url)
|
51
52
|
end
|
52
53
|
end
|
53
54
|
end
|
@@ -56,18 +57,18 @@ describe Wayfarer::Routing::QueryRule do
|
|
56
57
|
let(:constraints) { Hash[arg: /foo/] }
|
57
58
|
|
58
59
|
context "with matching query field value" do
|
59
|
-
let(:
|
60
|
+
let(:url) { URI("http://example.com?arg=foo") }
|
60
61
|
|
61
62
|
it "returns true" do
|
62
|
-
expect(
|
63
|
+
expect(matcher).to match(url)
|
63
64
|
end
|
64
65
|
end
|
65
66
|
|
66
67
|
context "with mismatching query field value" do
|
67
|
-
let(:
|
68
|
+
let(:url) { URI("http://example.com?arg=bar") }
|
68
69
|
|
69
70
|
it "returns false" do
|
70
|
-
expect(
|
71
|
+
expect(matcher).not_to match(url)
|
71
72
|
end
|
72
73
|
end
|
73
74
|
end
|
@@ -76,26 +77,26 @@ describe Wayfarer::Routing::QueryRule do
|
|
76
77
|
let(:constraints) { Hash[arg: 1..10] }
|
77
78
|
|
78
79
|
context "with matching query field value" do
|
79
|
-
let(:
|
80
|
+
let(:url) { URI("http://example.com?arg=5") }
|
80
81
|
|
81
82
|
it "returns true" do
|
82
|
-
expect(
|
83
|
+
expect(matcher).to match(url)
|
83
84
|
end
|
84
85
|
end
|
85
86
|
|
86
87
|
context "with mismatching query field value" do
|
87
|
-
let(:
|
88
|
+
let(:url) { URI("http://example.com?arg=11") }
|
88
89
|
|
89
90
|
it "returns false" do
|
90
|
-
expect(
|
91
|
+
expect(matcher).not_to match(url)
|
91
92
|
end
|
92
93
|
end
|
93
94
|
|
94
95
|
context "with non-numeric query field value" do
|
95
|
-
let(:
|
96
|
+
let(:url) { URI("http://example.com?arg=foo") }
|
96
97
|
|
97
98
|
it "returns false" do
|
98
|
-
expect(
|
99
|
+
expect(matcher).not_to match(url)
|
99
100
|
end
|
100
101
|
end
|
101
102
|
end
|
@@ -106,19 +107,31 @@ describe Wayfarer::Routing::QueryRule do
|
|
106
107
|
end
|
107
108
|
|
108
109
|
context "with matching query field values" do
|
109
|
-
let(:
|
110
|
+
let(:url) { URI("http://example.com?foo=4&bar=bazqux&qux=zot&toto=2") }
|
110
111
|
|
111
112
|
it "returns true" do
|
112
|
-
expect(
|
113
|
+
expect(matcher).to match(url)
|
113
114
|
end
|
114
115
|
end
|
115
116
|
|
116
117
|
context "with mismatching query field values" do
|
117
|
-
let(:
|
118
|
+
let(:url) { URI("http://example.com?foo=bar&bar=qux&qux=6toto=0") }
|
118
119
|
|
119
120
|
it "returns false" do
|
120
|
-
expect(
|
121
|
+
expect(matcher).not_to match(url)
|
121
122
|
end
|
122
123
|
end
|
123
124
|
end
|
125
|
+
|
126
|
+
describe "#params" do
|
127
|
+
let(:constraints) do
|
128
|
+
Hash[foo: 1..5, bar: /baz/, qux: "zot", toto: 2]
|
129
|
+
end
|
130
|
+
|
131
|
+
it "returns the correct parameters" do
|
132
|
+
url = URI("http://example.com?foo=4&bar=bazqux&qux=zot&toto=2")
|
133
|
+
expect(matcher.params(url)).to \
|
134
|
+
eq("foo" => "4", "bar" => "bazqux", "qux" => "zot", "toto" => "2")
|
135
|
+
end
|
136
|
+
end
|
124
137
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Routing::Route do
|
6
|
+
let(:route) { Wayfarer::Routing::RootRoute.new }
|
7
|
+
|
8
|
+
describe "#matcher" do
|
9
|
+
subject(:matcher) { route.matcher }
|
10
|
+
|
11
|
+
context "with child routes" do
|
12
|
+
before { route.host("http://google.com") }
|
13
|
+
|
14
|
+
it "returns true" do
|
15
|
+
%w[http://example.com http://w3c.org http://google.com].each do |url|
|
16
|
+
expect(matcher).to match(URI(url))
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context "without child routes" do
|
22
|
+
it "returns false" do
|
23
|
+
%w[http://example.com http://w3c.org http://google.com].each do |url|
|
24
|
+
expect(matcher).not_to match(URI(url))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Routing::Route do
|
6
|
+
subject(:root) { Wayfarer::Routing::RootRoute.new }
|
7
|
+
|
8
|
+
describe "#invoke" do
|
9
|
+
context "with matching matchers" do
|
10
|
+
it "returns a Match" do
|
11
|
+
root.host "example.com", to: :overridden do
|
12
|
+
path "/:a/:b" do
|
13
|
+
to :overrider, query: { page: 42 }
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
result = root.invoke(URI("http://example.com/foo/bar?page=42"))
|
18
|
+
|
19
|
+
expect(result).to be_a(Wayfarer::Routing::Result::Match)
|
20
|
+
expect(result.action).to be(:overrider)
|
21
|
+
expect(result.params).to eq({
|
22
|
+
"a" => "foo",
|
23
|
+
"b" => "bar",
|
24
|
+
"page" => "42"
|
25
|
+
})
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
context "with mismatching matchers" do
|
30
|
+
it "returns a Mismatch" do
|
31
|
+
root.host "example.com", to: :overridden do
|
32
|
+
path "/:a/:b" do
|
33
|
+
query({ page: 42 }, { to: :overrider })
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
result = root.invoke(URI("http://w3c.org/foo/bar?page=42"))
|
38
|
+
|
39
|
+
expect(result).to be_a(Wayfarer::Routing::Result::Mismatch)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Routing::SchemeMatcher do
|
6
|
+
describe "#match" do
|
7
|
+
subject(:matcher) { Wayfarer::Routing::SchemeMatcher.new(:http) }
|
8
|
+
|
9
|
+
context "with matching URL" do
|
10
|
+
let(:url) { URI("http://example.com") }
|
11
|
+
|
12
|
+
it "returns true" do
|
13
|
+
expect(matcher).to match(url)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
context "with mismatching URL" do
|
18
|
+
let(:url) { URI("https://example.com") }
|
19
|
+
|
20
|
+
it "returns true" do
|
21
|
+
expect(matcher).not_to match(url)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -1,38 +1,39 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require "spec_helpers"
|
3
4
|
|
4
|
-
describe Wayfarer::Routing::
|
5
|
-
subject(:
|
5
|
+
describe Wayfarer::Routing::SuffixMatcher do
|
6
|
+
subject(:matcher) { Wayfarer::Routing::SuffixMatcher.new(".png") }
|
6
7
|
|
7
|
-
describe "#
|
8
|
-
context "with matching
|
8
|
+
describe "#match" do
|
9
|
+
context "with matching URL" do
|
9
10
|
it "returns true" do
|
10
|
-
|
11
|
+
urls = %w[
|
11
12
|
http://example.com/foo.png
|
12
13
|
https://example.com/a/b.png/c.png
|
13
14
|
http://example.com/foobar.html.png
|
14
15
|
https://example.com/foo.png/bar.png
|
15
|
-
|
16
|
+
].map { |u| URI(u) }
|
16
17
|
|
17
|
-
|
18
|
-
expect(
|
18
|
+
urls.each do |url|
|
19
|
+
expect(matcher).to match(url)
|
19
20
|
end
|
20
21
|
end
|
21
22
|
end
|
22
23
|
|
23
|
-
context "with mismatching
|
24
|
+
context "with mismatching URL" do
|
24
25
|
it "returns false" do
|
25
|
-
|
26
|
+
urls = %w[
|
26
27
|
http://example.png
|
27
28
|
http://example.com.png
|
28
29
|
https://example.com/a/b.png/c
|
29
30
|
http://example.com.html
|
30
31
|
http://example.com/foobar.html
|
31
32
|
https://example.com/foo.png/bar.png.html
|
32
|
-
|
33
|
+
].map { |u| URI(u) }
|
33
34
|
|
34
|
-
|
35
|
-
expect(
|
35
|
+
urls.each do |url|
|
36
|
+
expect(matcher).not_to match(url)
|
36
37
|
end
|
37
38
|
end
|
38
39
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Routing::URLMatcher do
|
6
|
+
subject(:matcher) do
|
7
|
+
Wayfarer::Routing::URLMatcher.new("http://example.com/foo/bar")
|
8
|
+
end
|
9
|
+
|
10
|
+
describe "#match" do
|
11
|
+
context "with matching URL" do
|
12
|
+
let(:url) { URI("http://example.com/foo/bar") }
|
13
|
+
|
14
|
+
it "returns true" do
|
15
|
+
expect(matcher).to match(url)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
context "with mismatching URL" do
|
20
|
+
let(:url) { URI("http://example.com/foo/qux") }
|
21
|
+
|
22
|
+
it "returns true" do
|
23
|
+
expect(matcher).not_to match(url)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/spec/spec_helpers.rb
CHANGED
@@ -1,62 +1,89 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require_relative "../lib/wayfarer"
|
3
|
-
|
4
|
-
require("bundler") && Bundler.require
|
5
2
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
include Wayfarer::Frontiers
|
10
|
-
|
11
|
-
if RUBY_PLATFORM == "java"
|
12
|
-
Wayfarer::Frontiers::MemoryBloomfilter = \
|
13
|
-
Wayfarer::Frontiers::MemoryTrieFrontier = \
|
14
|
-
Wayfarer::Frontiers::RedisBloomfilter = nil
|
15
|
-
end
|
3
|
+
require "factory_bot"
|
4
|
+
require "pry"
|
5
|
+
require "sinatra"
|
16
6
|
|
17
|
-
|
7
|
+
require_relative "../lib/wayfarer"
|
8
|
+
require_relative "support/test_app"
|
18
9
|
|
19
10
|
module SpecHelpers
|
20
|
-
def
|
21
|
-
|
22
|
-
end
|
23
|
-
|
24
|
-
def test_app(path)
|
25
|
-
URI("http://localhost:9876#{path}")
|
11
|
+
def test_app_path(path)
|
12
|
+
File.join(test_app_host, path)
|
26
13
|
end
|
27
14
|
|
28
|
-
def
|
29
|
-
|
15
|
+
def test_app_host
|
16
|
+
if ENV["CI"]
|
17
|
+
"http://test:9876"
|
18
|
+
else
|
19
|
+
"http://localhost:9876"
|
20
|
+
end
|
30
21
|
end
|
31
22
|
|
32
|
-
def
|
33
|
-
|
23
|
+
def redis_host
|
24
|
+
if ENV["CI"]
|
25
|
+
"redis://redis:6379"
|
26
|
+
else
|
27
|
+
"redis://localhost:6379"
|
28
|
+
end
|
34
29
|
end
|
35
30
|
|
36
|
-
def
|
37
|
-
|
31
|
+
def write_file(path, contents)
|
32
|
+
FileUtils.mkdir_p(Pathname.new(path).dirname)
|
33
|
+
IO.write(path, contents)
|
38
34
|
end
|
39
35
|
end
|
40
36
|
|
41
37
|
RSpec.configure do |config|
|
38
|
+
config.include(FactoryBot::Syntax::Methods)
|
39
|
+
config.include(ActiveJob::TestHelper)
|
42
40
|
config.include(SpecHelpers)
|
43
41
|
|
44
|
-
config.before do
|
45
|
-
|
46
|
-
Wayfarer.config.selenium_argv = [:phantomjs]
|
47
|
-
end
|
42
|
+
config.before(:suite) do
|
43
|
+
FactoryBot.find_definitions
|
48
44
|
|
49
|
-
|
50
|
-
|
51
|
-
end
|
45
|
+
mutex = Mutex.new
|
46
|
+
cvar = ConditionVariable.new
|
52
47
|
|
53
|
-
|
54
|
-
|
48
|
+
Thread.new do
|
49
|
+
Rack::Handler::WEBrick.run(TestApp,
|
50
|
+
Host: "0.0.0.0",
|
51
|
+
Port: 9876,
|
52
|
+
Logger: WEBrick::Log.new("/dev/null"),
|
53
|
+
AccessLog: [],
|
54
|
+
StartCallback: proc { cvar.signal })
|
55
|
+
end
|
56
|
+
|
57
|
+
mutex.lock
|
58
|
+
cvar.wait(mutex)
|
55
59
|
end
|
56
60
|
|
57
|
-
config.before
|
58
|
-
|
61
|
+
config.before do
|
62
|
+
Wayfarer.config = Wayfarer::Config.from_environment
|
63
|
+
Wayfarer.config.redis_url = redis_host
|
64
|
+
|
65
|
+
::Redis.new(url: redis_host).flushall
|
66
|
+
|
67
|
+
ActiveJob::Base.queue_adapter = :test
|
68
|
+
ActiveJob::Base.logger = Logger.new(nil)
|
69
|
+
ActiveJob::Base.queue_adapter.enqueued_jobs.clear
|
70
|
+
ActiveJob::Base.queue_adapter.performed_jobs.clear
|
71
|
+
|
72
|
+
if ENV["CI"]
|
73
|
+
# For unknown reasons, Wayfarer::Networking::Ferrum#renew can take a very
|
74
|
+
# long time to instantiate a new ::Ferrum::Browser and requires the 60s
|
75
|
+
# timeout. TODO: Figure out what's going on
|
76
|
+
Wayfarer.config.ferrum_options = { url: "http://chrome:3000", timeout: 60 }
|
77
|
+
Wayfarer.config.selenium_argv = [:firefox, { url: "http://firefox:4444" }]
|
78
|
+
end
|
59
79
|
end
|
60
80
|
|
61
|
-
config.
|
81
|
+
config.around(cli: true) do |example|
|
82
|
+
origin = Dir.pwd
|
83
|
+
tmp = Dir.mktmpdir
|
84
|
+
Dir.chdir(tmp)
|
85
|
+
example.run
|
86
|
+
FileUtils.rm_r(tmp)
|
87
|
+
Dir.chdir(origin)
|
88
|
+
end
|
62
89
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Stringify do
|
6
|
+
let(:klass) do
|
7
|
+
Class.new(Struct.new(:a, :b, :c, :d)) do
|
8
|
+
include Wayfarer::Stringify
|
9
|
+
|
10
|
+
stringify :a, :c, :d
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
subject(:instance) { klass.new(1, 2.0, :three, "four") }
|
15
|
+
|
16
|
+
before { stub_const("Foobar", klass) }
|
17
|
+
|
18
|
+
describe "#to_s" do
|
19
|
+
it "returns the expected string" do
|
20
|
+
expect(instance.to_s).to eq('#<Foobar a=1, c=:three, d="four">')
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
@@ -1,5 +1,4 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require "sinatra"
|
3
2
|
|
4
3
|
class TestApp < Sinatra::Base
|
5
4
|
set :root, File.dirname(__FILE__)
|
@@ -9,11 +8,19 @@ class TestApp < Sinatra::Base
|
|
9
8
|
status params[:code]
|
10
9
|
end
|
11
10
|
|
11
|
+
get "/headers/:header" do
|
12
|
+
request.env[params[:header]]
|
13
|
+
end
|
14
|
+
|
12
15
|
get "/hello_world" do
|
13
16
|
headers "hello" => "world"
|
14
17
|
"Hello world!"
|
15
18
|
end
|
16
19
|
|
20
|
+
get "/body/:content" do
|
21
|
+
params[:content]
|
22
|
+
end
|
23
|
+
|
17
24
|
get "/redirect_loop" do
|
18
25
|
redirect to "/redirect_loop"
|
19
26
|
end
|
@@ -37,7 +44,7 @@ class TestApp < Sinatra::Base
|
|
37
44
|
send_file(static_file("xml/#{params[:file]}"))
|
38
45
|
end
|
39
46
|
|
40
|
-
|
47
|
+
private
|
41
48
|
|
42
49
|
def static_file(file_path)
|
43
50
|
File.expand_path(file_path, settings.public_folder)
|
data/spec/task_spec.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Task do
|
6
|
+
subject(:task) { build(:task) }
|
7
|
+
|
8
|
+
describe "#metadata" do
|
9
|
+
it "returns an OpenStruct" do
|
10
|
+
expect(task.metadata).to be_an(OpenStruct)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe "#barrier" do
|
15
|
+
it "returns its batch barrier" do
|
16
|
+
expect(task.barrier).to be_a(Wayfarer::Redis::Barrier)
|
17
|
+
expect(task.barrier.batch).to eq(task.batch)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe "#counter" do
|
22
|
+
it "returns its batch counter" do
|
23
|
+
expect(task.counter).to be_a(Wayfarer::Redis::Counter)
|
24
|
+
expect(task.counter.batch).to eq(task.batch)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
data/spec/wayfarer_spec.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require "spec_helpers"
|
3
4
|
|
4
5
|
describe Wayfarer do
|
@@ -6,19 +7,7 @@ describe Wayfarer do
|
|
6
7
|
it("is present") { expect(defined? Wayfarer::VERSION).not_to be nil }
|
7
8
|
end
|
8
9
|
|
9
|
-
describe "::logger, ::logger=" do
|
10
|
-
it "returns and alters the logger" do
|
11
|
-
expect(Wayfarer.logger).to be_a Logger
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
10
|
describe "::config" do
|
16
|
-
it "
|
17
|
-
Wayfarer.config.foo = :bar
|
18
|
-
expect(Wayfarer.config.foo).to be :bar
|
19
|
-
|
20
|
-
Wayfarer.config { |config| config.baz = :qux }
|
21
|
-
expect(Wayfarer.config.baz).to be :qux
|
22
|
-
end
|
11
|
+
it("returns a config") { expect(Wayfarer.config).to be_a(Wayfarer::Config) }
|
23
12
|
end
|
24
13
|
end
|