wayfarer 0.0.3 → 0.4.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/ci.yaml +32 -0
- data/.gitignore +3 -4
- data/.rubocop.yml +25 -9
- data/.ruby-version +1 -1
- data/Dockerfile +5 -0
- data/Gemfile +1 -7
- data/Gemfile.lock +221 -0
- data/RELEASING.md +17 -0
- data/Rakefile +38 -90
- data/bin/wayfarer +1 -111
- data/docker-compose.yml +32 -0
- data/docs/cookbook/batch_routing.md +22 -0
- data/docs/cookbook/consent_screen.md +36 -0
- data/docs/cookbook/executing_javascript.md +41 -0
- data/docs/cookbook/querying_html.md +42 -0
- data/docs/cookbook/screenshots.md +27 -0
- data/docs/cookbook/user_agent.md +7 -0
- data/docs/guides/browser_automation/capybara.md +69 -0
- data/docs/guides/browser_automation/custom_adapters.md +100 -0
- data/docs/guides/browser_automation/ferrum.md +39 -0
- data/docs/guides/browser_automation/selenium.md +63 -0
- data/docs/guides/callbacks.md +131 -31
- data/docs/guides/configuration.md +24 -169
- data/docs/guides/debugging.md +17 -0
- data/docs/guides/error_handling.md +30 -45
- data/docs/guides/jobs.md +101 -0
- data/docs/guides/navigation.md +73 -0
- data/docs/guides/networking.md +94 -0
- data/docs/guides/pages.md +52 -0
- data/docs/guides/performance.md +130 -0
- data/docs/guides/reliability.md +41 -0
- data/docs/guides/routing/steering.md +30 -0
- data/docs/guides/tasks.md +14 -0
- data/docs/index.md +40 -66
- data/docs/reference/api/base.md +48 -0
- data/docs/reference/api/route.md +182 -0
- data/docs/reference/cli.md +61 -0
- data/docs/reference/configuration_keys.md +42 -0
- data/docs/reference/environment_variables.md +83 -0
- data/lib/wayfarer/base.rb +50 -0
- data/lib/wayfarer/callbacks.rb +71 -0
- data/lib/wayfarer/cli/base.rb +27 -0
- data/lib/wayfarer/cli/generate.rb +17 -0
- data/lib/wayfarer/cli/job.rb +60 -0
- data/lib/wayfarer/cli/route.rb +29 -0
- data/lib/wayfarer/cli/route_printer.rb +116 -0
- data/lib/wayfarer/cli/runner.rb +34 -0
- data/lib/wayfarer/cli/templates/Gemfile.tt +5 -0
- data/lib/wayfarer/cli/templates/job.rb.tt +10 -0
- data/lib/wayfarer/config/capybara.rb +10 -0
- data/lib/wayfarer/config/ferrum.rb +11 -0
- data/lib/wayfarer/config/networking.rb +26 -0
- data/lib/wayfarer/config/redis.rb +14 -0
- data/lib/wayfarer/config/root.rb +11 -0
- data/lib/wayfarer/config/selenium.rb +21 -0
- data/lib/wayfarer/config/strconv.rb +45 -0
- data/lib/wayfarer/config/struct.rb +72 -0
- data/lib/wayfarer/gc.rb +15 -0
- data/lib/wayfarer/middleware/chain.rb +19 -0
- data/lib/wayfarer/middleware/dedup.rb +25 -0
- data/lib/wayfarer/middleware/fetch.rb +47 -0
- data/lib/wayfarer/middleware/normalize.rb +25 -0
- data/lib/wayfarer/middleware/router.rb +53 -0
- data/lib/wayfarer/middleware/stage.rb +23 -0
- data/lib/wayfarer/middleware/worker.rb +30 -0
- data/lib/wayfarer/networking/capybara.rb +28 -0
- data/lib/wayfarer/networking/context.rb +36 -0
- data/lib/wayfarer/networking/ferrum.rb +35 -0
- data/lib/wayfarer/networking/http.rb +34 -0
- data/lib/wayfarer/networking/pool.rb +40 -0
- data/lib/wayfarer/networking/result.rb +18 -0
- data/lib/wayfarer/networking/selenium.rb +43 -0
- data/lib/wayfarer/networking/strategy.rb +38 -0
- data/lib/wayfarer/page.rb +17 -74
- data/lib/wayfarer/parsing/json.rb +17 -0
- data/lib/wayfarer/parsing/xml.rb +17 -0
- data/lib/wayfarer/redis/.#barrier.rb +1 -0
- data/lib/wayfarer/redis/barrier.rb +36 -0
- data/lib/wayfarer/redis/connection.rb +13 -0
- data/lib/wayfarer/redis/counter.rb +29 -0
- data/lib/wayfarer/redis/pool.rb +20 -0
- data/lib/wayfarer/redis/version.rb +19 -0
- data/lib/wayfarer/routing/dsl.rb +57 -0
- data/lib/wayfarer/routing/matchers/custom.rb +25 -0
- data/lib/wayfarer/routing/matchers/host.rb +19 -0
- data/lib/wayfarer/routing/matchers/path.rb +49 -0
- data/lib/wayfarer/routing/matchers/query.rb +63 -0
- data/lib/wayfarer/routing/matchers/scheme.rb +17 -0
- data/lib/wayfarer/routing/matchers/suffix.rb +17 -0
- data/lib/wayfarer/routing/matchers/url.rb +17 -0
- data/lib/wayfarer/routing/path_finder.rb +46 -0
- data/lib/wayfarer/routing/result.rb +15 -0
- data/lib/wayfarer/routing/root_route.rb +7 -0
- data/lib/wayfarer/routing/route.rb +47 -0
- data/lib/wayfarer/routing/router.rb +10 -54
- data/lib/wayfarer/routing/target_route.rb +7 -0
- data/lib/wayfarer/serializer.rb +17 -0
- data/lib/wayfarer/stringify.rb +47 -0
- data/lib/wayfarer/task.rb +34 -0
- data/lib/wayfarer.rb +48 -57
- data/mkdocs.yml +47 -0
- data/requirements.txt +1 -0
- data/spec/base_spec.rb +233 -0
- data/spec/callbacks_spec.rb +102 -0
- data/spec/cli/generate_spec.rb +39 -0
- data/spec/cli/job_spec.rb +74 -0
- data/spec/cli/version_spec.rb +13 -0
- data/spec/config/capybara_spec.rb +18 -0
- data/spec/config/ferrum_spec.rb +24 -0
- data/spec/config/networking_spec.rb +73 -0
- data/spec/config/redis_spec.rb +32 -0
- data/spec/config/root_spec.rb +31 -0
- data/spec/config/selenium_spec.rb +56 -0
- data/spec/config/strconv_spec.rb +58 -0
- data/spec/config/struct_spec.rb +66 -0
- data/spec/factories/middleware.rb +15 -0
- data/spec/factories/page.rb +78 -0
- data/spec/factories/task.rb +12 -0
- data/spec/fixtures/dummy_job.rb +7 -0
- data/spec/gc_spec.rb +63 -0
- data/spec/middleware/chain_spec.rb +96 -0
- data/spec/middleware/dedup_spec.rb +76 -0
- data/spec/middleware/fetch_spec.rb +100 -0
- data/spec/middleware/normalize_spec.rb +28 -0
- data/spec/middleware/router_spec.rb +80 -0
- data/spec/middleware/stage_spec.rb +39 -0
- data/spec/middleware/worker_spec.rb +117 -0
- data/spec/networking/capybara_spec.rb +12 -0
- data/spec/networking/context_spec.rb +127 -0
- data/spec/networking/ferrum_spec.rb +12 -0
- data/spec/networking/http_spec.rb +12 -0
- data/spec/networking/pool_spec.rb +67 -0
- data/spec/networking/selenium_spec.rb +12 -0
- data/spec/networking/strategy.rb +170 -0
- data/spec/page_spec.rb +21 -12
- data/spec/{parsers/json_parser_spec.rb → parsing/json_spec.rb} +5 -4
- data/spec/{parsers/xml_parser_spec.rb → parsing/xml_spec.rb} +3 -2
- data/spec/redis/barrier_spec.rb +78 -0
- data/spec/redis/counter_spec.rb +32 -0
- data/spec/redis/pool_spec.rb +18 -0
- data/spec/redis/version_spec.rb +13 -0
- data/spec/routing/dsl_spec.rb +98 -0
- data/spec/routing/integration_spec.rb +110 -0
- data/spec/routing/matchers/custom_spec.rb +31 -0
- data/spec/routing/matchers/host_spec.rb +49 -0
- data/spec/routing/matchers/path_spec.rb +43 -0
- data/spec/routing/matchers/query_spec.rb +137 -0
- data/spec/routing/matchers/scheme_spec.rb +25 -0
- data/spec/routing/{filetypes_rule_spec.rb → matchers/suffix_spec.rb} +14 -13
- data/spec/routing/matchers/uri_spec.rb +27 -0
- data/spec/routing/path_finder_spec.rb +33 -0
- data/spec/routing/root_route_spec.rb +29 -0
- data/spec/routing/route_spec.rb +43 -0
- data/spec/routing/router_spec.rb +13 -56
- data/spec/spec_helpers.rb +73 -38
- data/spec/stringify_spec.rb +23 -0
- data/{support → spec/support}/static/finders.html +0 -0
- data/{support → spec/support}/static/graph/details/a.html +0 -0
- data/{support → spec/support}/static/graph/details/b.html +0 -0
- data/{support → spec/support}/static/graph/index.html +0 -0
- data/{support → spec/support}/static/json/dummy.json +0 -0
- data/{support → spec/support}/static/links/links.html +0 -0
- data/{support → spec/support}/static/xml/dummy.xml +0 -0
- data/{support → spec/support}/test_app.rb +9 -2
- data/spec/task_spec.rb +27 -0
- data/spec/wayfarer_spec.rb +2 -13
- data/wayfarer.gemspec +40 -42
- metadata +234 -361
- data/.travis.yml +0 -5
- data/Changelog.md +0 -10
- data/README.md +0 -21
- data/benchmark/frontiers.rb +0 -143
- data/docs/.gitignore +0 -2
- data/docs/_config.yml +0 -15
- data/docs/_includes/base.html +0 -7
- data/docs/_includes/head.html +0 -10
- data/docs/_includes/navigation.html +0 -187
- data/docs/_layouts/default.html +0 -42
- data/docs/_sass/base.scss +0 -439
- data/docs/_sass/variables.scss +0 -24
- data/docs/_sass/vendor/bourbon/_bourbon-deprecate.scss +0 -19
- data/docs/_sass/vendor/bourbon/_bourbon-deprecated-upcoming.scss +0 -425
- data/docs/_sass/vendor/bourbon/_bourbon.scss +0 -90
- data/docs/_sass/vendor/bourbon/addons/_border-color.scss +0 -29
- data/docs/_sass/vendor/bourbon/addons/_border-radius.scss +0 -48
- data/docs/_sass/vendor/bourbon/addons/_border-style.scss +0 -28
- data/docs/_sass/vendor/bourbon/addons/_border-width.scss +0 -28
- data/docs/_sass/vendor/bourbon/addons/_buttons.scss +0 -69
- data/docs/_sass/vendor/bourbon/addons/_clearfix.scss +0 -25
- data/docs/_sass/vendor/bourbon/addons/_ellipsis.scss +0 -30
- data/docs/_sass/vendor/bourbon/addons/_font-stacks.scss +0 -31
- data/docs/_sass/vendor/bourbon/addons/_hide-text.scss +0 -27
- data/docs/_sass/vendor/bourbon/addons/_margin.scss +0 -29
- data/docs/_sass/vendor/bourbon/addons/_padding.scss +0 -29
- data/docs/_sass/vendor/bourbon/addons/_position.scss +0 -51
- data/docs/_sass/vendor/bourbon/addons/_prefixer.scss +0 -66
- data/docs/_sass/vendor/bourbon/addons/_retina-image.scss +0 -27
- data/docs/_sass/vendor/bourbon/addons/_size.scss +0 -56
- data/docs/_sass/vendor/bourbon/addons/_text-inputs.scss +0 -118
- data/docs/_sass/vendor/bourbon/addons/_timing-functions.scss +0 -34
- data/docs/_sass/vendor/bourbon/addons/_triangle.scss +0 -63
- data/docs/_sass/vendor/bourbon/addons/_word-wrap.scss +0 -29
- data/docs/_sass/vendor/bourbon/css3/_animation.scss +0 -61
- data/docs/_sass/vendor/bourbon/css3/_appearance.scss +0 -5
- data/docs/_sass/vendor/bourbon/css3/_backface-visibility.scss +0 -5
- data/docs/_sass/vendor/bourbon/css3/_background-image.scss +0 -44
- data/docs/_sass/vendor/bourbon/css3/_background.scss +0 -57
- data/docs/_sass/vendor/bourbon/css3/_border-image.scss +0 -61
- data/docs/_sass/vendor/bourbon/css3/_calc.scss +0 -6
- data/docs/_sass/vendor/bourbon/css3/_columns.scss +0 -67
- data/docs/_sass/vendor/bourbon/css3/_filter.scss +0 -6
- data/docs/_sass/vendor/bourbon/css3/_flex-box.scss +0 -327
- data/docs/_sass/vendor/bourbon/css3/_font-face.scss +0 -29
- data/docs/_sass/vendor/bourbon/css3/_font-feature-settings.scss +0 -6
- data/docs/_sass/vendor/bourbon/css3/_hidpi-media-query.scss +0 -12
- data/docs/_sass/vendor/bourbon/css3/_hyphens.scss +0 -6
- data/docs/_sass/vendor/bourbon/css3/_image-rendering.scss +0 -15
- data/docs/_sass/vendor/bourbon/css3/_keyframes.scss +0 -38
- data/docs/_sass/vendor/bourbon/css3/_linear-gradient.scss +0 -40
- data/docs/_sass/vendor/bourbon/css3/_perspective.scss +0 -12
- data/docs/_sass/vendor/bourbon/css3/_placeholder.scss +0 -10
- data/docs/_sass/vendor/bourbon/css3/_radial-gradient.scss +0 -40
- data/docs/_sass/vendor/bourbon/css3/_selection.scss +0 -44
- data/docs/_sass/vendor/bourbon/css3/_text-decoration.scss +0 -27
- data/docs/_sass/vendor/bourbon/css3/_transform.scss +0 -21
- data/docs/_sass/vendor/bourbon/css3/_transition.scss +0 -81
- data/docs/_sass/vendor/bourbon/css3/_user-select.scss +0 -5
- data/docs/_sass/vendor/bourbon/functions/_assign-inputs.scss +0 -16
- data/docs/_sass/vendor/bourbon/functions/_contains-falsy.scss +0 -25
- data/docs/_sass/vendor/bourbon/functions/_contains.scss +0 -31
- data/docs/_sass/vendor/bourbon/functions/_is-length.scss +0 -16
- data/docs/_sass/vendor/bourbon/functions/_is-light.scss +0 -26
- data/docs/_sass/vendor/bourbon/functions/_is-number.scss +0 -16
- data/docs/_sass/vendor/bourbon/functions/_is-size.scss +0 -23
- data/docs/_sass/vendor/bourbon/functions/_modular-scale.scss +0 -74
- data/docs/_sass/vendor/bourbon/functions/_px-to-em.scss +0 -24
- data/docs/_sass/vendor/bourbon/functions/_px-to-rem.scss +0 -26
- data/docs/_sass/vendor/bourbon/functions/_shade.scss +0 -24
- data/docs/_sass/vendor/bourbon/functions/_strip-units.scss +0 -22
- data/docs/_sass/vendor/bourbon/functions/_tint.scss +0 -24
- data/docs/_sass/vendor/bourbon/functions/_transition-property-name.scss +0 -37
- data/docs/_sass/vendor/bourbon/functions/_unpack.scss +0 -32
- data/docs/_sass/vendor/bourbon/helpers/_convert-units.scss +0 -26
- data/docs/_sass/vendor/bourbon/helpers/_directional-values.scss +0 -108
- data/docs/_sass/vendor/bourbon/helpers/_font-source-declaration.scss +0 -53
- data/docs/_sass/vendor/bourbon/helpers/_gradient-positions-parser.scss +0 -24
- data/docs/_sass/vendor/bourbon/helpers/_linear-angle-parser.scss +0 -35
- data/docs/_sass/vendor/bourbon/helpers/_linear-gradient-parser.scss +0 -51
- data/docs/_sass/vendor/bourbon/helpers/_linear-positions-parser.scss +0 -77
- data/docs/_sass/vendor/bourbon/helpers/_linear-side-corner-parser.scss +0 -41
- data/docs/_sass/vendor/bourbon/helpers/_radial-arg-parser.scss +0 -74
- data/docs/_sass/vendor/bourbon/helpers/_radial-gradient-parser.scss +0 -55
- data/docs/_sass/vendor/bourbon/helpers/_radial-positions-parser.scss +0 -28
- data/docs/_sass/vendor/bourbon/helpers/_render-gradients.scss +0 -31
- data/docs/_sass/vendor/bourbon/helpers/_shape-size-stripper.scss +0 -15
- data/docs/_sass/vendor/bourbon/helpers/_str-to-num.scss +0 -55
- data/docs/_sass/vendor/bourbon/settings/_asset-pipeline.scss +0 -7
- data/docs/_sass/vendor/bourbon/settings/_deprecation-warnings.scss +0 -8
- data/docs/_sass/vendor/bourbon/settings/_prefixer.scss +0 -9
- data/docs/_sass/vendor/bourbon/settings/_px-to-em.scss +0 -1
- data/docs/_sass/vendor/neat/_neat-helpers.scss +0 -11
- data/docs/_sass/vendor/neat/_neat.scss +0 -23
- data/docs/_sass/vendor/neat/functions/_new-breakpoint.scss +0 -49
- data/docs/_sass/vendor/neat/functions/_private.scss +0 -114
- data/docs/_sass/vendor/neat/grid/_box-sizing.scss +0 -15
- data/docs/_sass/vendor/neat/grid/_direction-context.scss +0 -33
- data/docs/_sass/vendor/neat/grid/_display-context.scss +0 -28
- data/docs/_sass/vendor/neat/grid/_fill-parent.scss +0 -22
- data/docs/_sass/vendor/neat/grid/_media.scss +0 -92
- data/docs/_sass/vendor/neat/grid/_omega.scss +0 -87
- data/docs/_sass/vendor/neat/grid/_outer-container.scss +0 -34
- data/docs/_sass/vendor/neat/grid/_pad.scss +0 -25
- data/docs/_sass/vendor/neat/grid/_private.scss +0 -35
- data/docs/_sass/vendor/neat/grid/_row.scss +0 -52
- data/docs/_sass/vendor/neat/grid/_shift.scss +0 -50
- data/docs/_sass/vendor/neat/grid/_span-columns.scss +0 -94
- data/docs/_sass/vendor/neat/grid/_to-deprecate.scss +0 -97
- data/docs/_sass/vendor/neat/grid/_visual-grid.scss +0 -42
- data/docs/_sass/vendor/neat/mixins/_clearfix.scss +0 -25
- data/docs/_sass/vendor/neat/settings/_disable-warnings.scss +0 -13
- data/docs/_sass/vendor/neat/settings/_grid.scss +0 -51
- data/docs/_sass/vendor/neat/settings/_visual-grid.scss +0 -27
- data/docs/_sass/vendor/normalize-3.0.2.scss +0 -427
- data/docs/_sass/vendor/pygments.scss +0 -356
- data/docs/automating_browsers/capybara.md +0 -70
- data/docs/css/screen.scss +0 -7
- data/docs/guides/cli.md +0 -52
- data/docs/guides/frontiers.md +0 -93
- data/docs/guides/halting.md +0 -23
- data/docs/guides/job_queues.md +0 -26
- data/docs/guides/locals.md +0 -36
- data/docs/guides/logging.md +0 -22
- data/docs/guides/page_objects.md +0 -67
- data/docs/guides/peeking.md +0 -46
- data/docs/guides/selenium_capybara.md +0 -100
- data/docs/guides/tutorial.md +0 -452
- data/docs/js/navigation.js +0 -11
- data/docs/misc/contributing.md +0 -20
- data/docs/misc/testing.md +0 -11
- data/docs/recipes/authentication.md +0 -23
- data/docs/recipes/csv.md +0 -29
- data/docs/recipes/javascript.md +0 -20
- data/docs/recipes/multiple_uris.md +0 -18
- data/docs/recipes/screenshots.md +0 -20
- data/docs/routing/custom_rules.md +0 -16
- data/docs/routing/filetypes_rules.md +0 -21
- data/docs/routing/host_rules.md +0 -24
- data/docs/routing/path_rules.md +0 -33
- data/docs/routing/protocol_rules.md +0 -17
- data/docs/routing/query_rules.md +0 -69
- data/docs/routing/routes.md +0 -96
- data/docs/routing/uri_rules.md +0 -18
- data/examples/collect_github_issues.rb +0 -65
- data/examples/find_foobar_on_wikipedia.rb +0 -23
- data/lib/wayfarer/configuration.rb +0 -86
- data/lib/wayfarer/crawl.rb +0 -79
- data/lib/wayfarer/crawl_observer.rb +0 -103
- data/lib/wayfarer/dispatcher.rb +0 -104
- data/lib/wayfarer/finders.rb +0 -61
- data/lib/wayfarer/frontiers/frontier.rb +0 -79
- data/lib/wayfarer/frontiers/memory_bloomfilter.rb +0 -32
- data/lib/wayfarer/frontiers/memory_frontier.rb +0 -76
- data/lib/wayfarer/frontiers/memory_trie_frontier.rb +0 -39
- data/lib/wayfarer/frontiers/normalize_uris.rb +0 -48
- data/lib/wayfarer/frontiers/redis_bloomfilter.rb +0 -34
- data/lib/wayfarer/frontiers/redis_frontier.rb +0 -83
- data/lib/wayfarer/http_adapters/adapter_pool.rb +0 -62
- data/lib/wayfarer/http_adapters/net_http_adapter.rb +0 -77
- data/lib/wayfarer/http_adapters/selenium_adapter.rb +0 -80
- data/lib/wayfarer/job.rb +0 -211
- data/lib/wayfarer/locals.rb +0 -40
- data/lib/wayfarer/parsers/json_parser.rb +0 -20
- data/lib/wayfarer/parsers/xml_parser.rb +0 -27
- data/lib/wayfarer/processor.rb +0 -103
- data/lib/wayfarer/routing/custom_rule.rb +0 -21
- data/lib/wayfarer/routing/filetypes_rule.rb +0 -20
- data/lib/wayfarer/routing/host_rule.rb +0 -19
- data/lib/wayfarer/routing/path_rule.rb +0 -54
- data/lib/wayfarer/routing/protocol_rule.rb +0 -21
- data/lib/wayfarer/routing/query_rule.rb +0 -59
- data/lib/wayfarer/routing/rule.rb +0 -114
- data/lib/wayfarer/routing/uri_rule.rb +0 -21
- data/spec/configuration_spec.rb +0 -26
- data/spec/crawl_spec.rb +0 -48
- data/spec/finders_spec.rb +0 -49
- data/spec/frontiers/memory_bloomfilter_spec.rb +0 -6
- data/spec/frontiers/memory_frontier_spec.rb +0 -6
- data/spec/frontiers/memory_trie_frontier_spec.rb +0 -6
- data/spec/frontiers/normalize_uris_spec.rb +0 -59
- data/spec/frontiers/redis_bloomfilter_spec.rb +0 -6
- data/spec/frontiers/redis_frontier_spec.rb +0 -6
- data/spec/http_adapters/adapter_pool_spec.rb +0 -33
- data/spec/http_adapters/net_http_adapter_spec.rb +0 -83
- data/spec/http_adapters/selenium_adapter_spec.rb +0 -53
- data/spec/integration/callbacks_spec.rb +0 -42
- data/spec/integration/locals_spec.rb +0 -106
- data/spec/integration/peeking_spec.rb +0 -61
- data/spec/job_spec.rb +0 -122
- data/spec/processor_spec.rb +0 -31
- data/spec/routing/custom_rule_spec.rb +0 -26
- data/spec/routing/host_rule_spec.rb +0 -48
- data/spec/routing/path_rule_spec.rb +0 -66
- data/spec/routing/protocol_rule_spec.rb +0 -26
- data/spec/routing/query_rule_spec.rb +0 -124
- data/spec/routing/rule_spec.rb +0 -251
- data/spec/routing/uri_rule_spec.rb +0 -24
- data/spec/shared/frontier.rb +0 -96
- data/wayfarer-jruby.gemspec +0 -49
@@ -0,0 +1,83 @@
|
|
1
|
+
# Environment Variables
|
2
|
+
|
3
|
+
## String formats
|
4
|
+
|
5
|
+
Environment variable values can be parsed to Hash or Array at runtime
|
6
|
+
with the following syntaxes:
|
7
|
+
|
8
|
+
* Hash: Variable string `a:1,b:2,c:3` parses to `{a:1, b:2, c:3}` at runtime
|
9
|
+
* Array: Variable string `a,b,c` parses to `[:a, :b, :c]` at runtime
|
10
|
+
|
11
|
+
## Variables
|
12
|
+
|
13
|
+
### `WAYFARER_AGENT`
|
14
|
+
: Either `ferrum`, `selenium` or `http`.
|
15
|
+
|
16
|
+
* Type: String
|
17
|
+
* Key: `config.agent`
|
18
|
+
* Default value: `:http`
|
19
|
+
|
20
|
+
### `WAYFARER_POOL_SIZE`
|
21
|
+
: Number of user agents to maintain.
|
22
|
+
|
23
|
+
* Type: Integer
|
24
|
+
* Key: `config.pool_size`
|
25
|
+
* Default value: `1`
|
26
|
+
|
27
|
+
### `WAYFARER_POOL_TIMEOUT`
|
28
|
+
: How long a user agent may remain checked out until the owning job
|
29
|
+
fails.
|
30
|
+
|
31
|
+
* Type: Integer
|
32
|
+
* Key: `config.agent_pool_timeout`
|
33
|
+
* Default value: `1`
|
34
|
+
|
35
|
+
---
|
36
|
+
|
37
|
+
### `WAYFARER_FERRUM_OPTIONS`
|
38
|
+
: Key/value options passed to `Ferrum::Browser.new`.
|
39
|
+
|
40
|
+
* Type: Hash
|
41
|
+
* Key: `config.ferrum_options`
|
42
|
+
* Default value: `{}`
|
43
|
+
|
44
|
+
---
|
45
|
+
|
46
|
+
### `WAYFARER_SELENIUM_DRIVER`
|
47
|
+
: Driver passed to `Selenium::WebDriver.for`.
|
48
|
+
|
49
|
+
* Type: Symbol
|
50
|
+
* Key: `config.selenium_driver`
|
51
|
+
* Default value: `:chrome`
|
52
|
+
|
53
|
+
---
|
54
|
+
|
55
|
+
### `WAYFARER_SELENIUM_OPTIONS`
|
56
|
+
: Options passed to `Selenium::WebDriver.for`.
|
57
|
+
|
58
|
+
* Type: Hash
|
59
|
+
* Key: `config.selenium_options`
|
60
|
+
* Default value: `{}`
|
61
|
+
|
62
|
+
---
|
63
|
+
|
64
|
+
### `WAYFARER_SELENIUM_CLIENT_TIMEOUT`
|
65
|
+
: Selenium HTTP client timeout (seconds).
|
66
|
+
|
67
|
+
* Type: Integer
|
68
|
+
* Key: `config.selenium_client_timeout`
|
69
|
+
* Default value: `60`
|
70
|
+
|
71
|
+
---
|
72
|
+
|
73
|
+
### `WAYFARER_HTTP_HEADERS`
|
74
|
+
: HTTP request headers used when retrieving pages.
|
75
|
+
|
76
|
+
* Type: Hash
|
77
|
+
* Key: `config.http_headers`
|
78
|
+
* Default value: `{}`
|
79
|
+
|
80
|
+
!!! attention "Partial support"
|
81
|
+
|
82
|
+
Selenium does not support configuring HTTP request headers.
|
83
|
+
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
class Base < ActiveJob::Base
|
5
|
+
include Wayfarer::Middleware::Worker
|
6
|
+
extend Forwardable
|
7
|
+
|
8
|
+
after_enqueue do |_job|
|
9
|
+
task.counter.increment
|
10
|
+
end
|
11
|
+
|
12
|
+
after_perform { |job| GC.new(job).run }
|
13
|
+
|
14
|
+
rescue_from(StandardError) { GC.new(self).run }
|
15
|
+
|
16
|
+
def self.retry_on(*argv)
|
17
|
+
super(*argv) do |job, error|
|
18
|
+
job.task.barrier.seen?(job.task.url)
|
19
|
+
GC.new(job).run
|
20
|
+
yield job, error if block_given?
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.discard_on(*argv)
|
25
|
+
super(*argv) do |job, error|
|
26
|
+
job.task.barrier.seen?(job.task.url)
|
27
|
+
GC.new(job).run
|
28
|
+
yield job, error if block_given?
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.crawl_later(url, batch: SecureRandom.uuid)
|
33
|
+
perform_later(Task.new(url, batch))
|
34
|
+
end
|
35
|
+
|
36
|
+
def retry_job(...)
|
37
|
+
super(...) # increments the counter by re-enqueuing the job
|
38
|
+
task.counter.decrement
|
39
|
+
end
|
40
|
+
|
41
|
+
def perform(task)
|
42
|
+
task.job = self
|
43
|
+
chain.call(task)
|
44
|
+
end
|
45
|
+
|
46
|
+
def task
|
47
|
+
arguments.first
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module Callbacks
|
5
|
+
TERMINATOR = ->(_target, result) { result.call == false }
|
6
|
+
OPTIONS = { terminator: TERMINATOR, skip_after_callbacks_if_terminated: true }.freeze
|
7
|
+
|
8
|
+
ConditionalCallback = Struct.new(:job, :filters) do
|
9
|
+
def run(method, &block)
|
10
|
+
return if only && !applies?(only)
|
11
|
+
return if except && applies?(except)
|
12
|
+
|
13
|
+
return job.send(method) if method
|
14
|
+
|
15
|
+
job.instance_eval(&block)
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def applies?(condition)
|
21
|
+
case condition
|
22
|
+
when Symbol then condition == action
|
23
|
+
when Enumerable then condition&.include?(action)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def only
|
28
|
+
filters[:only]
|
29
|
+
end
|
30
|
+
|
31
|
+
def except
|
32
|
+
filters[:except]
|
33
|
+
end
|
34
|
+
|
35
|
+
def action
|
36
|
+
task.metadata.action
|
37
|
+
end
|
38
|
+
|
39
|
+
def task
|
40
|
+
job.task
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.included(base)
|
45
|
+
base.include(ActiveSupport::Callbacks)
|
46
|
+
base.extend(ClassMethods)
|
47
|
+
|
48
|
+
base.class_eval do
|
49
|
+
define_callbacks(:fetch, OPTIONS)
|
50
|
+
define_callbacks(:action, OPTIONS)
|
51
|
+
define_callbacks(:batch, OPTIONS)
|
52
|
+
|
53
|
+
define(:fetch, :before)
|
54
|
+
define(:action, :before)
|
55
|
+
define(:batch, :after)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
module ClassMethods
|
60
|
+
private
|
61
|
+
|
62
|
+
def define(name, stage)
|
63
|
+
define_singleton_method([stage, name].join("_")) do |method = nil, **filters, &block|
|
64
|
+
set_callback(name, stage, **filters) do |job|
|
65
|
+
ConditionalCallback.new(job, filters).run(method, &block)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module CLI
|
5
|
+
class Base < Thor
|
6
|
+
include Thor::Actions
|
7
|
+
|
8
|
+
def self.source_root
|
9
|
+
File.expand_path("templates", __dir__)
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def mock_redis
|
15
|
+
Wayfarer.config.redis.factory = ->(_) { MockRedis.new }
|
16
|
+
end
|
17
|
+
|
18
|
+
def load_environment
|
19
|
+
Wayfarer::CLI::Runner.loader.setup
|
20
|
+
end
|
21
|
+
|
22
|
+
def free_agent_pool
|
23
|
+
Wayfarer::Networking::Pool.instance.free
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module CLI
|
5
|
+
class Generate < Base
|
6
|
+
desc "project NAME",
|
7
|
+
"Generate new project directory NAME"
|
8
|
+
def project(name)
|
9
|
+
@name = name
|
10
|
+
|
11
|
+
empty_directory name
|
12
|
+
template "Gemfile.tt", "#{name}/Gemfile"
|
13
|
+
template "job.rb.tt", "#{name}/app/jobs/#{name}.rb"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module CLI
|
5
|
+
class Job < Base
|
6
|
+
desc "perform JOB URL",
|
7
|
+
"Perform JOB with URL"
|
8
|
+
option :mock_redis, type: :boolean
|
9
|
+
option :batch, type: :string, default: SecureRandom.uuid
|
10
|
+
def perform(job, url)
|
11
|
+
load_environment
|
12
|
+
mock_redis if options[:mock_redis]
|
13
|
+
|
14
|
+
url = Addressable::URI.parse(url)
|
15
|
+
job = job.classify.constantize.new
|
16
|
+
task = Wayfarer::Task.new(url, options[:batch])
|
17
|
+
job.arguments.push(task)
|
18
|
+
job.perform(task)
|
19
|
+
GC.new(job).run
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "enqueue JOB URL",
|
23
|
+
"Enqueue JOB with URL"
|
24
|
+
option :batch, type: :string, default: SecureRandom.uuid
|
25
|
+
def enqueue(job, url)
|
26
|
+
load_environment
|
27
|
+
mock_redis if options[:mock_redis]
|
28
|
+
|
29
|
+
url = Addressable::URI.parse(url)
|
30
|
+
job = job.classify.constantize
|
31
|
+
job.crawl_later(url, batch: options[:batch])
|
32
|
+
end
|
33
|
+
|
34
|
+
desc "execute JOB URL",
|
35
|
+
"Execute JOB with async adapter"
|
36
|
+
option :mock_redis, type: :boolean
|
37
|
+
option :batch, type: :string, default: SecureRandom.uuid
|
38
|
+
option :min_threads, type: :numeric, default: 1
|
39
|
+
option :max_threads, type: :numeric, default: 1
|
40
|
+
def execute(job, url)
|
41
|
+
load_environment
|
42
|
+
mock_redis if options[:mock_redis]
|
43
|
+
|
44
|
+
url = Addressable::URI.parse(url)
|
45
|
+
job = job.classify.constantize
|
46
|
+
|
47
|
+
job.queue_adapter = ActiveJob::QueueAdapters::AsyncAdapter.new(min_threads: options[:min_threads],
|
48
|
+
max_threads: options[:max_threads])
|
49
|
+
scheduler = job.queue_adapter.instance_variable_get(:@scheduler)
|
50
|
+
executor = scheduler.instance_variable_get(:@async_executor)
|
51
|
+
|
52
|
+
job.crawl_later(url, batch: options[:batch])
|
53
|
+
|
54
|
+
sleep(1) while executor.scheduled_task_count > executor.completed_task_count
|
55
|
+
|
56
|
+
free_agent_pool
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# # frozen_string_literal: true
|
4
|
+
|
5
|
+
module Wayfarer
|
6
|
+
module CLI
|
7
|
+
class Route < Base
|
8
|
+
desc "result JOB URL",
|
9
|
+
"Invoke JOB's router with URL"
|
10
|
+
def result(job, url)
|
11
|
+
load_environment
|
12
|
+
url = Addressable::URI.parse(url)
|
13
|
+
job = job.classify.constantize
|
14
|
+
job.router.invoke(url, job.new.steer)
|
15
|
+
say Wayfarer::Routing::PathFinder.result(job.router.root, url)
|
16
|
+
end
|
17
|
+
|
18
|
+
desc "tree JOB URL",
|
19
|
+
"Visualize JOB's routing tree for URL"
|
20
|
+
def tree(job, url)
|
21
|
+
load_environment
|
22
|
+
url = Addressable::URI.parse(url)
|
23
|
+
job = job.classify.constantize
|
24
|
+
job.router.invoke(url, job.new.steer)
|
25
|
+
Wayfarer::CLI::RoutePrinter.print(job.router.root, url)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module CLI
|
5
|
+
class RoutePrinter < Thor::Shell::Color
|
6
|
+
attr_reader :url,
|
7
|
+
:path_finder,
|
8
|
+
:output
|
9
|
+
|
10
|
+
INDENT = " "
|
11
|
+
REGULAR_SEGMENT = "│ "
|
12
|
+
JUNCTION_SEGMENT = "├──"
|
13
|
+
CORNER_SEGMENT = "└──"
|
14
|
+
|
15
|
+
def self.print(route, url)
|
16
|
+
route.accept(new(url))
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(url)
|
20
|
+
@url = url
|
21
|
+
@path_finder = Wayfarer::Routing::PathFinder.new(url)
|
22
|
+
super()
|
23
|
+
end
|
24
|
+
|
25
|
+
def visit(route)
|
26
|
+
route.accept(path_finder) unless route.parent
|
27
|
+
return true if route.is_a?(Wayfarer::Routing::RootRoute)
|
28
|
+
|
29
|
+
puts [segments(route), label(route)].join("")[3..]
|
30
|
+
true
|
31
|
+
end
|
32
|
+
|
33
|
+
def segments(route)
|
34
|
+
current = segment(route)
|
35
|
+
parents = parents(route).map { |parent| parent_segment(parent) }
|
36
|
+
[parents, current].join
|
37
|
+
end
|
38
|
+
|
39
|
+
def parent_segment(parent)
|
40
|
+
if trailer?(parent)
|
41
|
+
INDENT
|
42
|
+
else
|
43
|
+
REGULAR_SEGMENT
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def segment(route)
|
48
|
+
if trailer?(route)
|
49
|
+
CORNER_SEGMENT
|
50
|
+
else
|
51
|
+
JUNCTION_SEGMENT
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def label(route)
|
56
|
+
[highlight_matcher(route, matcher_label(route)),
|
57
|
+
highlight_options(route, options(route)),
|
58
|
+
highlight_options(route, params(route))].compact.join(" ")
|
59
|
+
end
|
60
|
+
|
61
|
+
def highlight_matcher(route, string)
|
62
|
+
if path_finder.path.include?(route)
|
63
|
+
set_color(string, :green, :bold)
|
64
|
+
elsif route.matcher.match(url)
|
65
|
+
set_color(string, :green)
|
66
|
+
else
|
67
|
+
set_color(string, :red)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def highlight_options(route, string)
|
72
|
+
return string unless path_finder.path.include?(route)
|
73
|
+
|
74
|
+
set_color(string, :green, :bold)
|
75
|
+
end
|
76
|
+
|
77
|
+
def matcher_label(route)
|
78
|
+
return "Target" if route.is_a?(Wayfarer::Routing::TargetRoute)
|
79
|
+
|
80
|
+
route.matcher.class.name.demodulize
|
81
|
+
end
|
82
|
+
|
83
|
+
def options(route)
|
84
|
+
return "" if route.is_a?(Wayfarer::Routing::RootRoute)
|
85
|
+
|
86
|
+
case (matcher = route.matcher)
|
87
|
+
when Wayfarer::Routing::Matchers::Host then matcher.host
|
88
|
+
when Wayfarer::Routing::Matchers::Path then matcher.path
|
89
|
+
when Wayfarer::Routing::Matchers::Query then matcher.fields
|
90
|
+
when Wayfarer::Routing::Matchers::Custom then "##{route.action}"
|
91
|
+
when Wayfarer::Routing::Matchers::Scheme then matcher.scheme
|
92
|
+
when Wayfarer::Routing::Matchers::Suffix then matcher.suffix
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def params(route)
|
97
|
+
params = route.matcher.params(url)
|
98
|
+
"=> #{params.symbolize_keys}" if params.any?
|
99
|
+
end
|
100
|
+
|
101
|
+
private
|
102
|
+
|
103
|
+
def parents(route, current = [])
|
104
|
+
return current unless route.parent
|
105
|
+
|
106
|
+
parents(route.parent, [route.parent, *current])
|
107
|
+
end
|
108
|
+
|
109
|
+
def trailer?(route)
|
110
|
+
return true unless route.parent
|
111
|
+
|
112
|
+
route.parent.children.last == route
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module CLI
|
5
|
+
class Runner < Thor
|
6
|
+
def self.exit_on_failure?
|
7
|
+
true
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.loader
|
11
|
+
# TODO: Raises if $PWD/app/jobs does not exist
|
12
|
+
|
13
|
+
@loader ||= Zeitwerk::Loader.new.tap do |load|
|
14
|
+
load.inflector = Zeitwerk::GemInflector.new(__FILE__)
|
15
|
+
load.push_dir("#{Dir.pwd}/app/jobs")
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
desc "version", "Print version"
|
20
|
+
def version
|
21
|
+
say Wayfarer::VERSION::STRING
|
22
|
+
end
|
23
|
+
|
24
|
+
desc "route", "Routing"
|
25
|
+
subcommand "route", Wayfarer::CLI::Route
|
26
|
+
|
27
|
+
desc "generate", "Generators"
|
28
|
+
subcommand "generate", Wayfarer::CLI::Generate
|
29
|
+
|
30
|
+
desc "job", "Jobs"
|
31
|
+
subcommand "job", Wayfarer::CLI::Job
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module Config
|
5
|
+
Networking = Struct.new(agent: {
|
6
|
+
env_key: "WAYFARER_NETWORK_AGENT",
|
7
|
+
type: Symbol,
|
8
|
+
default: :http
|
9
|
+
},
|
10
|
+
pool_size: {
|
11
|
+
env_key: "WAYFARER_NETWORK_POOL_SIZE",
|
12
|
+
type: Integer,
|
13
|
+
default: 1
|
14
|
+
},
|
15
|
+
pool_timeout: {
|
16
|
+
env_key: "WAYFARER_NETWORK_POOL_TIMEOUT",
|
17
|
+
type: Integer,
|
18
|
+
default: 10
|
19
|
+
},
|
20
|
+
http_headers: {
|
21
|
+
env_key: "WAYFARER_NETWORK_HTTP_HEADERS",
|
22
|
+
type: Hash,
|
23
|
+
default: {}
|
24
|
+
})
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module Config
|
5
|
+
Redis = Struct.new(url: {
|
6
|
+
env_key: "WAYFARER_REDIS_URL",
|
7
|
+
type: String,
|
8
|
+
default: "redis://localhost:6379"
|
9
|
+
},
|
10
|
+
factory: {
|
11
|
+
default: ->(redis) { ::Redis.new(url: redis.url) }
|
12
|
+
})
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module Config
|
5
|
+
Root = Struct.new(ferrum: { default: Wayfarer::Config::Ferrum.new },
|
6
|
+
network: { default: Wayfarer::Config::Networking.new },
|
7
|
+
redis: { default: Wayfarer::Config::Redis.new },
|
8
|
+
selenium: { default: Wayfarer::Config::Selenium.new },
|
9
|
+
capybara: { default: Wayfarer::Config::Capybara.new })
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module Config
|
5
|
+
Selenium = Struct.new(driver: {
|
6
|
+
env_key: "WAYFARER_SELENIUM_DRIVER",
|
7
|
+
type: Symbol,
|
8
|
+
default: :chrome
|
9
|
+
},
|
10
|
+
options: {
|
11
|
+
env_key: "WAYFARER_SELENIUM_OPTIONS",
|
12
|
+
type: Hash,
|
13
|
+
default: {}
|
14
|
+
},
|
15
|
+
client_timeout: {
|
16
|
+
env_key: "WAYFARER_SELENIUM_CLIENT_TIMEOUT",
|
17
|
+
type: Integer,
|
18
|
+
default: 60 # seconds
|
19
|
+
})
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Wayfarer
|
4
|
+
module Config
|
5
|
+
module Strconv
|
6
|
+
module_function
|
7
|
+
|
8
|
+
def parse(str, type = nil)
|
9
|
+
return primitive(str) unless type
|
10
|
+
|
11
|
+
case type.name
|
12
|
+
when "Hash" then hash(str)
|
13
|
+
when "Array" then array(str)
|
14
|
+
when "Symbol" then str.to_sym
|
15
|
+
when "Integer" then Integer(str)
|
16
|
+
else str
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def hash(str)
|
21
|
+
array(str).reduce({}) do |acc, pair|
|
22
|
+
k, v = pair.split(":", 2)
|
23
|
+
next acc unless k && v
|
24
|
+
|
25
|
+
acc.merge({ parse(k, Symbol) => primitive(v) })
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def array(str)
|
30
|
+
str.split(",").map(&:strip)
|
31
|
+
end
|
32
|
+
|
33
|
+
def primitive(str)
|
34
|
+
return true if str == "true"
|
35
|
+
return false if str == "false"
|
36
|
+
|
37
|
+
begin
|
38
|
+
parse(str, Integer)
|
39
|
+
rescue StandardError
|
40
|
+
str
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|