wayfarer 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.env +17 -0
- data/.github/workflows/lint.yaml +27 -0
- data/.github/workflows/release.yaml +30 -0
- data/.github/workflows/tests.yaml +21 -0
- data/.gitignore +5 -1
- data/.rubocop.yml +36 -0
- data/.vale.ini +8 -0
- data/.yardopts +1 -3
- data/Dockerfile +6 -4
- data/Gemfile +24 -0
- data/Gemfile.lock +274 -164
- data/Rakefile +7 -51
- data/bin/wayfarer +1 -1
- data/docker-compose.yml +23 -13
- data/docs/cookbook/consent_screen.md +2 -2
- data/docs/cookbook/executing_javascript.md +3 -3
- data/docs/cookbook/navigation.md +12 -12
- data/docs/cookbook/querying_html.md +3 -3
- data/docs/cookbook/screenshots.md +2 -2
- data/docs/guides/callbacks.md +25 -125
- data/docs/guides/cli.md +71 -0
- data/docs/guides/configuration.md +10 -35
- data/docs/guides/development.md +67 -0
- data/docs/guides/handlers.md +60 -0
- data/docs/guides/index.md +1 -0
- data/docs/guides/jobs.md +142 -31
- data/docs/guides/navigation.md +1 -1
- data/docs/guides/networking/capybara.md +13 -22
- data/docs/guides/networking/custom_adapters.md +103 -41
- data/docs/guides/networking/ferrum.md +4 -4
- data/docs/guides/networking/http.md +9 -13
- data/docs/guides/networking/selenium.md +10 -11
- data/docs/guides/pages.md +78 -10
- data/docs/guides/redis.md +10 -0
- data/docs/guides/routing.md +156 -0
- data/docs/guides/tasks.md +53 -9
- data/docs/guides/tutorial.md +66 -0
- data/docs/guides/user_agents.md +115 -0
- data/docs/index.md +17 -40
- data/lib/wayfarer/base.rb +125 -46
- data/lib/wayfarer/batch_completion.rb +60 -0
- data/lib/wayfarer/callbacks.rb +22 -48
- data/lib/wayfarer/cli/route_printer.rb +85 -89
- data/lib/wayfarer/cli.rb +103 -0
- data/lib/wayfarer/gc.rb +18 -6
- data/lib/wayfarer/handler.rb +15 -7
- data/lib/wayfarer/kv.rb +28 -0
- data/lib/wayfarer/logging.rb +38 -0
- data/lib/wayfarer/middleware/base.rb +2 -0
- data/lib/wayfarer/middleware/batch_completion.rb +19 -0
- data/lib/wayfarer/middleware/chain.rb +7 -1
- data/lib/wayfarer/middleware/content_type.rb +59 -0
- data/lib/wayfarer/middleware/controller.rb +19 -15
- data/lib/wayfarer/middleware/dedup.rb +22 -13
- data/lib/wayfarer/middleware/dispatch.rb +17 -4
- data/lib/wayfarer/middleware/normalize.rb +7 -14
- data/lib/wayfarer/middleware/redis.rb +15 -0
- data/lib/wayfarer/middleware/router.rb +33 -35
- data/lib/wayfarer/middleware/stage.rb +5 -5
- data/lib/wayfarer/middleware/uri_parser.rb +31 -0
- data/lib/wayfarer/middleware/user_agent.rb +49 -0
- data/lib/wayfarer/networking/capybara.rb +1 -1
- data/lib/wayfarer/networking/context.rb +14 -3
- data/lib/wayfarer/networking/ferrum.rb +1 -4
- data/lib/wayfarer/networking/follow.rb +14 -7
- data/lib/wayfarer/networking/http.rb +1 -1
- data/lib/wayfarer/networking/pool.rb +23 -13
- data/lib/wayfarer/networking/selenium.rb +15 -7
- data/lib/wayfarer/networking/strategy.rb +2 -2
- data/lib/wayfarer/page.rb +34 -14
- data/lib/wayfarer/parsing/xml.rb +6 -6
- data/lib/wayfarer/parsing.rb +21 -0
- data/lib/wayfarer/redis/barrier.rb +26 -21
- data/lib/wayfarer/redis/counter.rb +18 -9
- data/lib/wayfarer/redis/pool.rb +1 -1
- data/lib/wayfarer/redis/resettable.rb +19 -0
- data/lib/wayfarer/routing/dsl.rb +166 -30
- data/lib/wayfarer/routing/hash_stack.rb +33 -0
- data/lib/wayfarer/routing/matchers/custom.rb +8 -5
- data/lib/wayfarer/routing/matchers/{suffix.rb → empty_params.rb} +2 -6
- data/lib/wayfarer/routing/matchers/host.rb +15 -9
- data/lib/wayfarer/routing/matchers/path.rb +11 -31
- data/lib/wayfarer/routing/matchers/query.rb +41 -17
- data/lib/wayfarer/routing/matchers/result.rb +12 -0
- data/lib/wayfarer/routing/matchers/scheme.rb +13 -5
- data/lib/wayfarer/routing/matchers/url.rb +13 -5
- data/lib/wayfarer/routing/path_consumer.rb +130 -0
- data/lib/wayfarer/routing/path_finder.rb +151 -23
- data/lib/wayfarer/routing/result.rb +1 -1
- data/lib/wayfarer/routing/root_route.rb +17 -1
- data/lib/wayfarer/routing/route.rb +66 -19
- data/lib/wayfarer/routing/serializable.rb +28 -0
- data/lib/wayfarer/routing/sub_route.rb +53 -0
- data/lib/wayfarer/routing/target_route.rb +17 -1
- data/lib/wayfarer/stringify.rb +21 -30
- data/lib/wayfarer/task.rb +9 -17
- data/lib/wayfarer/uri/normalization.rb +120 -0
- data/lib/wayfarer.rb +72 -5
- data/mise.toml +2 -0
- data/mkdocs.yml +44 -8
- data/rake/docs.rake +26 -0
- data/rake/lint.rake +9 -0
- data/rake/release.rake +23 -0
- data/rake/tests.rake +32 -0
- data/requirements.txt +1 -1
- data/spec/factories/job.rb +8 -0
- data/spec/factories/middleware.rb +2 -2
- data/spec/factories/path_finder.rb +11 -0
- data/spec/factories/redis.rb +19 -0
- data/spec/factories/task.rb +46 -2
- data/spec/spec_helpers.rb +55 -51
- data/spec/support/active_job_helpers.rb +8 -0
- data/spec/support/integration_helpers.rb +21 -0
- data/spec/support/redis_helpers.rb +9 -0
- data/spec/support/test_app.rb +66 -37
- data/spec/wayfarer/base_spec.rb +200 -0
- data/spec/wayfarer/batch_completion_spec.rb +142 -0
- data/spec/wayfarer/cli/job_spec.rb +88 -0
- data/spec/wayfarer/cli/routing_spec.rb +322 -0
- data/spec/{cli → wayfarer/cli}/version_spec.rb +1 -1
- data/spec/wayfarer/gc_spec.rb +29 -0
- data/spec/wayfarer/handler_spec.rb +9 -0
- data/spec/wayfarer/integration/callbacks_spec.rb +200 -0
- data/spec/wayfarer/integration/content_type_spec.rb +37 -0
- data/spec/wayfarer/integration/custom_routing_spec.rb +51 -0
- data/spec/wayfarer/integration/gc_spec.rb +40 -0
- data/spec/wayfarer/integration/handler_spec.rb +65 -0
- data/spec/wayfarer/integration/page_spec.rb +79 -0
- data/spec/wayfarer/integration/params_spec.rb +64 -0
- data/spec/wayfarer/integration/parsing_spec.rb +99 -0
- data/spec/wayfarer/integration/retry_spec.rb +112 -0
- data/spec/wayfarer/integration/stage_spec.rb +58 -0
- data/spec/wayfarer/middleware/batch_completion_spec.rb +33 -0
- data/spec/{middleware → wayfarer/middleware}/chain_spec.rb +24 -19
- data/spec/wayfarer/middleware/content_type_spec.rb +83 -0
- data/spec/{middleware → wayfarer/middleware}/controller_spec.rb +24 -22
- data/spec/wayfarer/middleware/dedup_spec.rb +66 -0
- data/spec/wayfarer/middleware/normalize_spec.rb +32 -0
- data/spec/wayfarer/middleware/router_spec.rb +102 -0
- data/spec/wayfarer/middleware/stage_spec.rb +63 -0
- data/spec/wayfarer/middleware/uri_parser_spec.rb +63 -0
- data/spec/wayfarer/middleware/user_agent_spec.rb +158 -0
- data/spec/wayfarer/networking/capybara_spec.rb +13 -0
- data/spec/{networking → wayfarer/networking}/context_spec.rb +46 -38
- data/spec/wayfarer/networking/ferrum_spec.rb +13 -0
- data/spec/{networking → wayfarer/networking}/follow_spec.rb +11 -6
- data/spec/wayfarer/networking/http_spec.rb +12 -0
- data/spec/{networking → wayfarer/networking}/pool_spec.rb +16 -14
- data/spec/wayfarer/networking/selenium_spec.rb +12 -0
- data/spec/{networking → wayfarer/networking}/strategy.rb +33 -54
- data/spec/wayfarer/page_spec.rb +69 -0
- data/spec/{parsing → wayfarer/parsing}/json_spec.rb +1 -1
- data/spec/wayfarer/parsing/xml_parse_spec.rb +25 -0
- data/spec/wayfarer/redis/barrier_spec.rb +39 -0
- data/spec/wayfarer/redis/counter_spec.rb +34 -0
- data/spec/{redis → wayfarer/redis}/pool_spec.rb +4 -3
- data/spec/{routing → wayfarer/routing}/dsl_spec.rb +12 -22
- data/spec/wayfarer/routing/hash_stack_spec.rb +63 -0
- data/spec/wayfarer/routing/integration_spec.rb +101 -0
- data/spec/wayfarer/routing/matchers/custom_spec.rb +39 -0
- data/spec/wayfarer/routing/matchers/host_spec.rb +56 -0
- data/spec/wayfarer/routing/matchers/matcher.rb +17 -0
- data/spec/wayfarer/routing/matchers/path_spec.rb +43 -0
- data/spec/wayfarer/routing/matchers/query_spec.rb +123 -0
- data/spec/wayfarer/routing/matchers/scheme_spec.rb +45 -0
- data/spec/wayfarer/routing/matchers/url_spec.rb +33 -0
- data/spec/wayfarer/routing/path_consumer_spec.rb +123 -0
- data/spec/wayfarer/routing/path_finder_spec.rb +409 -0
- data/spec/wayfarer/routing/root_route_spec.rb +51 -0
- data/spec/wayfarer/routing/route_spec.rb +74 -0
- data/spec/wayfarer/routing/sub_route_spec.rb +103 -0
- data/spec/wayfarer/task_spec.rb +13 -0
- data/spec/wayfarer/uri/normalization_spec.rb +98 -0
- data/spec/wayfarer_spec.rb +2 -2
- data/wayfarer.gemspec +18 -28
- metadata +797 -265
- data/.github/workflows/ci.yaml +0 -32
- data/.rbenv-gemsets +0 -1
- data/.ruby-version +0 -1
- data/RELEASING.md +0 -17
- data/docs/cookbook/user_agent.md +0 -7
- data/docs/guides/error_handling.md +0 -53
- data/docs/guides/networking.md +0 -94
- data/docs/guides/performance.md +0 -130
- data/docs/guides/reliability.md +0 -41
- data/docs/guides/routing/steering.md +0 -30
- data/docs/reference/api/base.md +0 -48
- data/docs/reference/cli.md +0 -61
- data/docs/reference/configuration_keys.md +0 -43
- data/docs/reference/environment_variables.md +0 -83
- data/lib/wayfarer/cli/base.rb +0 -45
- data/lib/wayfarer/cli/generate.rb +0 -17
- data/lib/wayfarer/cli/job.rb +0 -56
- data/lib/wayfarer/cli/route.rb +0 -29
- data/lib/wayfarer/cli/runner.rb +0 -34
- data/lib/wayfarer/cli/templates/Gemfile.tt +0 -5
- data/lib/wayfarer/cli/templates/job.rb.tt +0 -10
- data/lib/wayfarer/config/capybara.rb +0 -10
- data/lib/wayfarer/config/ferrum.rb +0 -11
- data/lib/wayfarer/config/networking.rb +0 -29
- data/lib/wayfarer/config/redis.rb +0 -14
- data/lib/wayfarer/config/root.rb +0 -11
- data/lib/wayfarer/config/selenium.rb +0 -21
- data/lib/wayfarer/config/strconv.rb +0 -45
- data/lib/wayfarer/config/struct.rb +0 -72
- data/lib/wayfarer/middleware/fetch.rb +0 -56
- data/lib/wayfarer/redis/connection.rb +0 -13
- data/lib/wayfarer/redis/version.rb +0 -19
- data/lib/wayfarer/routing/router.rb +0 -28
- data/spec/base_spec.rb +0 -224
- data/spec/callbacks_spec.rb +0 -102
- data/spec/cli/generate_spec.rb +0 -39
- data/spec/cli/job_spec.rb +0 -78
- data/spec/config/capybara_spec.rb +0 -18
- data/spec/config/ferrum_spec.rb +0 -24
- data/spec/config/networking_spec.rb +0 -73
- data/spec/config/redis_spec.rb +0 -32
- data/spec/config/root_spec.rb +0 -31
- data/spec/config/selenium_spec.rb +0 -56
- data/spec/config/strconv_spec.rb +0 -58
- data/spec/config/struct_spec.rb +0 -66
- data/spec/fixtures/dummy_job.rb +0 -7
- data/spec/gc_spec.rb +0 -59
- data/spec/handler_spec.rb +0 -11
- data/spec/integration/callbacks_spec.rb +0 -85
- data/spec/integration/page_spec.rb +0 -62
- data/spec/integration/params_spec.rb +0 -56
- data/spec/integration/stage_spec.rb +0 -51
- data/spec/integration/steering_spec.rb +0 -57
- data/spec/middleware/dedup_spec.rb +0 -88
- data/spec/middleware/dispatch_spec.rb +0 -43
- data/spec/middleware/fetch_spec.rb +0 -155
- data/spec/middleware/normalize_spec.rb +0 -29
- data/spec/middleware/router_spec.rb +0 -105
- data/spec/middleware/stage_spec.rb +0 -62
- data/spec/networking/capybara_spec.rb +0 -12
- data/spec/networking/ferrum_spec.rb +0 -12
- data/spec/networking/http_spec.rb +0 -12
- data/spec/networking/selenium_spec.rb +0 -12
- data/spec/page_spec.rb +0 -47
- data/spec/parsing/xml_spec.rb +0 -25
- data/spec/redis/barrier_spec.rb +0 -78
- data/spec/redis/counter_spec.rb +0 -32
- data/spec/redis/version_spec.rb +0 -13
- data/spec/routing/integration_spec.rb +0 -110
- data/spec/routing/matchers/custom_spec.rb +0 -31
- data/spec/routing/matchers/host_spec.rb +0 -49
- data/spec/routing/matchers/path_spec.rb +0 -43
- data/spec/routing/matchers/query_spec.rb +0 -137
- data/spec/routing/matchers/scheme_spec.rb +0 -25
- data/spec/routing/matchers/suffix_spec.rb +0 -41
- data/spec/routing/matchers/uri_spec.rb +0 -27
- data/spec/routing/path_finder_spec.rb +0 -33
- data/spec/routing/root_route_spec.rb +0 -29
- data/spec/routing/route_spec.rb +0 -43
- data/spec/routing/router_spec.rb +0 -24
- data/spec/task_spec.rb +0 -34
- data/spec/{stringify_spec.rb → wayfarer/stringify_spec.rb} +2 -2
data/Gemfile.lock
CHANGED
@@ -1,221 +1,331 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
wayfarer (0.4.
|
5
|
-
activejob (>=
|
4
|
+
wayfarer (0.4.8)
|
5
|
+
activejob (>= 7.1)
|
6
6
|
addressable (~> 2.8)
|
7
|
-
capybara (~> 3.
|
8
|
-
connection_pool (~> 2.
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
selenium-webdriver (~> 3.4)
|
7
|
+
capybara (~> 3.4)
|
8
|
+
connection_pool (~> 2.5)
|
9
|
+
ferrum (~> 0.17)
|
10
|
+
metainspector (~> 5.16)
|
11
|
+
mime-types (~> 3.7)
|
12
|
+
mock_redis (~> 0.52)
|
13
|
+
mustermann (~> 3.0)
|
14
|
+
net-http-persistent (~> 4.0)
|
15
|
+
nokogiri (~> 1.18)
|
16
|
+
redis (~> 5.4)
|
17
|
+
selenium-webdriver (~> 4.35)
|
18
|
+
sorted_set (~> 1.0)
|
20
19
|
thor (~> 1.0)
|
20
|
+
zeitwerk (~> 2.7)
|
21
21
|
|
22
22
|
GEM
|
23
23
|
remote: https://rubygems.org/
|
24
24
|
specs:
|
25
|
-
activejob (
|
26
|
-
activesupport (=
|
25
|
+
activejob (8.0.2.1)
|
26
|
+
activesupport (= 8.0.2.1)
|
27
27
|
globalid (>= 0.3.6)
|
28
|
-
activesupport (
|
29
|
-
|
28
|
+
activesupport (8.0.2.1)
|
29
|
+
base64
|
30
|
+
benchmark (>= 0.3)
|
31
|
+
bigdecimal
|
32
|
+
concurrent-ruby (~> 1.0, >= 1.3.1)
|
33
|
+
connection_pool (>= 2.2.5)
|
34
|
+
drb
|
30
35
|
i18n (>= 1.6, < 2)
|
36
|
+
logger (>= 1.4.2)
|
31
37
|
minitest (>= 5.1)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
+
securerandom (>= 0.3)
|
39
|
+
tzinfo (~> 2.0, >= 2.0.5)
|
40
|
+
uri (>= 0.13.1)
|
41
|
+
addressable (2.8.7)
|
42
|
+
public_suffix (>= 2.0.2, < 7.0)
|
43
|
+
ast (2.4.3)
|
44
|
+
base64 (0.3.0)
|
45
|
+
benchmark (0.4.1)
|
46
|
+
bigdecimal (3.2.2)
|
47
|
+
binding_of_caller (1.0.1)
|
48
|
+
debug_inspector (>= 1.2.0)
|
49
|
+
capybara (3.40.0)
|
38
50
|
addressable
|
39
51
|
matrix
|
40
52
|
mini_mime (>= 0.1.3)
|
41
|
-
nokogiri (~> 1.
|
53
|
+
nokogiri (~> 1.11)
|
42
54
|
rack (>= 1.6.0)
|
43
55
|
rack-test (>= 0.6.3)
|
44
56
|
regexp_parser (>= 1.5, < 3.0)
|
45
57
|
xpath (~> 3.2)
|
46
|
-
childprocess (3.0.0)
|
47
|
-
cliver (0.3.2)
|
48
58
|
coderay (1.1.3)
|
49
|
-
concurrent-ruby (1.
|
50
|
-
connection_pool (2.
|
51
|
-
cuprite (0.
|
52
|
-
capybara (
|
53
|
-
ferrum (~> 0.
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
faraday-
|
66
|
-
|
67
|
-
|
68
|
-
faraday-net_http (~> 1.0)
|
69
|
-
faraday-net_http_persistent (~> 1.0)
|
70
|
-
faraday-patron (~> 1.0)
|
71
|
-
faraday-rack (~> 1.0)
|
72
|
-
faraday-retry (~> 1.0)
|
73
|
-
ruby2_keywords (>= 0.0.4)
|
59
|
+
concurrent-ruby (1.3.5)
|
60
|
+
connection_pool (2.5.3)
|
61
|
+
cuprite (0.17)
|
62
|
+
capybara (~> 3.0)
|
63
|
+
ferrum (~> 0.17.0)
|
64
|
+
date (3.4.1)
|
65
|
+
debug_inspector (1.2.0)
|
66
|
+
diff-lcs (1.6.2)
|
67
|
+
domain_name (0.6.20240107)
|
68
|
+
drb (2.2.3)
|
69
|
+
erb (5.0.2)
|
70
|
+
factory_bot (6.5.5)
|
71
|
+
activesupport (>= 6.1.0)
|
72
|
+
faker (3.5.2)
|
73
|
+
i18n (>= 1.8.11, < 2)
|
74
|
+
faraday (2.13.4)
|
75
|
+
faraday-net_http (>= 2.0, < 3.5)
|
76
|
+
json
|
77
|
+
logger
|
74
78
|
faraday-cookie_jar (0.0.7)
|
75
79
|
faraday (>= 0.8.0)
|
76
80
|
http-cookie (~> 1.0.0)
|
77
|
-
faraday-
|
78
|
-
faraday-em_synchrony (1.0.0)
|
79
|
-
faraday-encoding (0.0.5)
|
81
|
+
faraday-encoding (0.0.6)
|
80
82
|
faraday
|
81
|
-
faraday-
|
82
|
-
|
83
|
+
faraday-follow_redirects (0.3.0)
|
84
|
+
faraday (>= 1, < 3)
|
85
|
+
faraday-gzip (2.0.1)
|
86
|
+
faraday (>= 1.0)
|
87
|
+
zlib (~> 3.0)
|
88
|
+
faraday-http-cache (2.5.1)
|
83
89
|
faraday (>= 0.8)
|
84
|
-
faraday-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
faraday-rack (1.0.0)
|
91
|
-
faraday-retry (1.0.3)
|
92
|
-
faraday_middleware (1.2.0)
|
93
|
-
faraday (~> 1.0)
|
94
|
-
fastimage (2.2.6)
|
95
|
-
ferrum (0.11)
|
90
|
+
faraday-net_http (3.4.1)
|
91
|
+
net-http (>= 0.5.0)
|
92
|
+
faraday-retry (2.3.2)
|
93
|
+
faraday (~> 2.0)
|
94
|
+
fastimage (2.4.0)
|
95
|
+
ferrum (0.17.1)
|
96
96
|
addressable (~> 2.5)
|
97
|
-
|
97
|
+
base64 (~> 0.2)
|
98
98
|
concurrent-ruby (~> 1.1)
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
99
|
+
webrick (~> 1.7)
|
100
|
+
websocket-driver (~> 0.7)
|
101
|
+
ffi (1.17.2-x86_64-darwin)
|
102
|
+
ffi (1.17.2-x86_64-linux-musl)
|
103
|
+
globalid (1.2.1)
|
104
|
+
activesupport (>= 6.1)
|
105
|
+
http-cookie (1.0.8)
|
103
106
|
domain_name (~> 0.5)
|
104
|
-
i18n (1.
|
107
|
+
i18n (1.14.7)
|
105
108
|
concurrent-ruby (~> 1.0)
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
109
|
+
io-console (0.8.1)
|
110
|
+
irb (1.15.2)
|
111
|
+
pp (>= 0.6.0)
|
112
|
+
rdoc (>= 4.0.0)
|
113
|
+
reline (>= 0.4.2)
|
114
|
+
json (2.13.2)
|
115
|
+
language_server-protocol (3.17.0.5)
|
116
|
+
lint_roller (1.1.0)
|
117
|
+
listen (3.9.0)
|
118
|
+
rb-fsevent (~> 0.10, >= 0.10.3)
|
119
|
+
rb-inotify (~> 0.9, >= 0.9.10)
|
120
|
+
logger (1.7.0)
|
121
|
+
matrix (0.4.3)
|
122
|
+
metainspector (5.16.0)
|
123
|
+
addressable (~> 2.8.4)
|
124
|
+
faraday (~> 2.5)
|
110
125
|
faraday-cookie_jar (~> 0.0)
|
111
126
|
faraday-encoding (~> 0.0)
|
112
|
-
faraday-
|
113
|
-
|
127
|
+
faraday-follow_redirects (~> 0.3)
|
128
|
+
faraday-gzip (>= 0.1, < 3.0)
|
129
|
+
faraday-http-cache (~> 2.5)
|
130
|
+
faraday-retry (~> 2.0)
|
114
131
|
fastimage (~> 2.2)
|
115
132
|
nesty (~> 1.0)
|
116
|
-
nokogiri (~> 1.
|
117
|
-
method_source (1.
|
118
|
-
mime-types (3.
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
minitest (5.
|
124
|
-
mock_redis (0.
|
125
|
-
|
126
|
-
|
127
|
-
mustermann (1.1.1)
|
133
|
+
nokogiri (~> 1.18.8)
|
134
|
+
method_source (1.1.0)
|
135
|
+
mime-types (3.7.0)
|
136
|
+
logger
|
137
|
+
mime-types-data (~> 3.2025, >= 3.2025.0507)
|
138
|
+
mime-types-data (3.2025.0819)
|
139
|
+
mini_mime (1.1.5)
|
140
|
+
minitest (5.25.5)
|
141
|
+
mock_redis (0.52.0)
|
142
|
+
redis (~> 5)
|
143
|
+
mustermann (3.0.4)
|
128
144
|
ruby2_keywords (~> 0.0.1)
|
129
145
|
nesty (1.0.2)
|
130
|
-
net-http
|
131
|
-
|
132
|
-
|
133
|
-
|
146
|
+
net-http (0.6.0)
|
147
|
+
uri
|
148
|
+
net-http-persistent (4.0.6)
|
149
|
+
connection_pool (~> 2.2, >= 2.2.4)
|
150
|
+
nio4r (2.7.4)
|
151
|
+
nokogiri (1.18.9-x86_64-darwin)
|
152
|
+
racc (~> 1.4)
|
153
|
+
nokogiri (1.18.9-x86_64-linux-musl)
|
134
154
|
racc (~> 1.4)
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
parser (3.0.2.0)
|
155
|
+
ostruct (0.6.3)
|
156
|
+
parallel (1.27.0)
|
157
|
+
parser (3.3.9.0)
|
139
158
|
ast (~> 2.4.1)
|
140
|
-
|
159
|
+
racc
|
160
|
+
pp (0.6.2)
|
161
|
+
prettyprint
|
162
|
+
prettyprint (0.2.0)
|
163
|
+
prism (1.4.0)
|
164
|
+
proc_to_ast (0.2.0)
|
165
|
+
parser
|
166
|
+
rouge
|
167
|
+
unparser
|
168
|
+
pry (0.15.2)
|
141
169
|
coderay (~> 1.1)
|
142
170
|
method_source (~> 1.0)
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
171
|
+
psych (5.2.6)
|
172
|
+
date
|
173
|
+
stringio
|
174
|
+
public_suffix (6.0.2)
|
175
|
+
puma (6.6.1)
|
176
|
+
nio4r (~> 2.0)
|
177
|
+
racc (1.8.1)
|
178
|
+
rack (3.2.0)
|
179
|
+
rack-protection (4.1.1)
|
180
|
+
base64 (>= 0.1.0)
|
181
|
+
logger (>= 1.6.0)
|
182
|
+
rack (>= 3.0.0, < 4)
|
183
|
+
rack-session (2.1.1)
|
184
|
+
base64 (>= 0.1.0)
|
185
|
+
rack (>= 3.0.0)
|
186
|
+
rack-test (2.2.0)
|
187
|
+
rack (>= 1.3)
|
188
|
+
rackup (2.2.1)
|
189
|
+
rack (>= 3)
|
190
|
+
rainbow (3.1.1)
|
191
|
+
rake (13.3.0)
|
192
|
+
rb-fsevent (0.11.2)
|
193
|
+
rb-inotify (0.11.1)
|
194
|
+
ffi (~> 1.0)
|
195
|
+
rbtree (0.4.6)
|
196
|
+
rdoc (6.14.2)
|
197
|
+
erb
|
198
|
+
psych (>= 4.0.0)
|
199
|
+
redis (5.4.1)
|
200
|
+
redis-client (>= 0.22.0)
|
201
|
+
redis-client (0.25.2)
|
202
|
+
connection_pool
|
203
|
+
regexp_parser (2.11.2)
|
204
|
+
reline (0.6.2)
|
205
|
+
io-console (~> 0.5)
|
206
|
+
rerun (0.14.0)
|
207
|
+
listen (~> 3.0)
|
208
|
+
rexml (3.4.1)
|
209
|
+
rouge (4.6.0)
|
210
|
+
rspec (3.13.1)
|
211
|
+
rspec-core (~> 3.13.0)
|
212
|
+
rspec-expectations (~> 3.13.0)
|
213
|
+
rspec-mocks (~> 3.13.0)
|
214
|
+
rspec-core (3.13.5)
|
215
|
+
rspec-support (~> 3.13.0)
|
216
|
+
rspec-expectations (3.13.5)
|
162
217
|
diff-lcs (>= 1.2.0, < 2.0)
|
163
|
-
rspec-support (~> 3.
|
164
|
-
rspec-mocks (3.
|
218
|
+
rspec-support (~> 3.13.0)
|
219
|
+
rspec-mocks (3.13.5)
|
165
220
|
diff-lcs (>= 1.2.0, < 2.0)
|
166
|
-
rspec-support (~> 3.
|
167
|
-
rspec-
|
168
|
-
|
221
|
+
rspec-support (~> 3.13.0)
|
222
|
+
rspec-parameterized (2.0.0)
|
223
|
+
rspec-parameterized-core (>= 2, < 3)
|
224
|
+
rspec-parameterized-table_syntax (>= 2, < 3)
|
225
|
+
rspec-parameterized-core (2.0.0)
|
226
|
+
parser
|
227
|
+
prism
|
228
|
+
proc_to_ast (>= 0.2.0)
|
229
|
+
rspec (>= 2.13, < 4)
|
230
|
+
unparser
|
231
|
+
rspec-parameterized-table_syntax (2.0.0)
|
232
|
+
binding_of_caller
|
233
|
+
rspec-parameterized-core (>= 2, < 3)
|
234
|
+
rspec-support (3.13.5)
|
235
|
+
rubocop (1.80.0)
|
236
|
+
json (~> 2.3)
|
237
|
+
language_server-protocol (~> 3.17.0.2)
|
238
|
+
lint_roller (~> 1.1.0)
|
169
239
|
parallel (~> 1.10)
|
170
|
-
parser (>=
|
240
|
+
parser (>= 3.3.0.2)
|
171
241
|
rainbow (>= 2.2.2, < 4.0)
|
172
|
-
regexp_parser (>=
|
173
|
-
|
174
|
-
rubocop-ast (>= 0.6.0)
|
242
|
+
regexp_parser (>= 2.9.3, < 3.0)
|
243
|
+
rubocop-ast (>= 1.46.0, < 2.0)
|
175
244
|
ruby-progressbar (~> 1.7)
|
176
|
-
unicode-display_width (>=
|
177
|
-
rubocop-ast (1.
|
178
|
-
parser (>= 3.
|
179
|
-
|
245
|
+
unicode-display_width (>= 2.4.0, < 4.0)
|
246
|
+
rubocop-ast (1.46.0)
|
247
|
+
parser (>= 3.3.7.2)
|
248
|
+
prism (~> 1.4)
|
249
|
+
rubocop-factory_bot (2.27.1)
|
250
|
+
lint_roller (~> 1.1)
|
251
|
+
rubocop (~> 1.72, >= 1.72.1)
|
252
|
+
rubocop-rake (0.7.1)
|
253
|
+
lint_roller (~> 1.1)
|
254
|
+
rubocop (>= 1.72.1)
|
255
|
+
rubocop-rspec (3.6.0)
|
256
|
+
lint_roller (~> 1.1)
|
257
|
+
rubocop (~> 1.72, >= 1.72.1)
|
258
|
+
ruby-progressbar (1.13.0)
|
180
259
|
ruby2_keywords (0.0.5)
|
181
|
-
rubyzip (
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
260
|
+
rubyzip (3.0.2)
|
261
|
+
securerandom (0.4.1)
|
262
|
+
selenium-webdriver (4.35.0)
|
263
|
+
base64 (~> 0.2)
|
264
|
+
logger (~> 1.4)
|
265
|
+
rexml (~> 3.2, >= 3.2.5)
|
266
|
+
rubyzip (>= 1.2.2, < 4.0)
|
267
|
+
websocket (~> 1.0)
|
268
|
+
set (1.1.2)
|
269
|
+
sinatra (4.1.1)
|
270
|
+
logger (>= 1.6.0)
|
271
|
+
mustermann (~> 3.0)
|
272
|
+
rack (>= 3.0.0, < 4)
|
273
|
+
rack-protection (= 4.1.1)
|
274
|
+
rack-session (>= 2.0.0, < 3)
|
275
|
+
tilt (~> 2.0)
|
276
|
+
sorted_set (1.0.3)
|
277
|
+
rbtree
|
278
|
+
set (~> 1.0)
|
279
|
+
stringio (3.1.7)
|
280
|
+
thor (1.4.0)
|
281
|
+
tilt (2.6.1)
|
282
|
+
tzinfo (2.0.6)
|
192
283
|
concurrent-ruby (~> 1.0)
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
284
|
+
unicode-display_width (3.1.5)
|
285
|
+
unicode-emoji (~> 4.0, >= 4.0.4)
|
286
|
+
unicode-emoji (4.0.4)
|
287
|
+
unparser (0.8.0)
|
288
|
+
diff-lcs (~> 1.6)
|
289
|
+
parser (>= 3.3.0)
|
290
|
+
prism (>= 1.4)
|
291
|
+
uri (1.0.3)
|
292
|
+
webrick (1.9.1)
|
293
|
+
websocket (1.2.11)
|
294
|
+
websocket-driver (0.8.0)
|
295
|
+
base64
|
198
296
|
websocket-extensions (>= 0.1.0)
|
199
297
|
websocket-extensions (0.1.5)
|
200
298
|
xpath (3.2.0)
|
201
299
|
nokogiri (~> 1.8)
|
202
|
-
yard (0.9.
|
203
|
-
zeitwerk (2.
|
300
|
+
yard (0.9.37)
|
301
|
+
zeitwerk (2.7.3)
|
302
|
+
zlib (3.2.1)
|
204
303
|
|
205
304
|
PLATFORMS
|
206
|
-
|
305
|
+
x86_64-darwin-24
|
306
|
+
x86_64-linux-musl
|
207
307
|
|
208
308
|
DEPENDENCIES
|
209
|
-
cuprite (~> 0.
|
210
|
-
factory_bot (~> 6.
|
211
|
-
faker (~>
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
309
|
+
cuprite (~> 0.17)
|
310
|
+
factory_bot (~> 6.5)
|
311
|
+
faker (~> 3.5)
|
312
|
+
irb
|
313
|
+
ostruct (~> 0.6)
|
314
|
+
pry (~> 0.15)
|
315
|
+
puma (~> 6.6)
|
316
|
+
rackup (~> 2.2)
|
317
|
+
rake (~> 13.3)
|
318
|
+
rerun
|
319
|
+
rspec (~> 3.13)
|
320
|
+
rspec-parameterized (~> 2.0)
|
321
|
+
rubocop (~> 1.80)
|
322
|
+
rubocop-factory_bot
|
323
|
+
rubocop-rake
|
324
|
+
rubocop-rspec
|
325
|
+
sinatra (~> 4.1)
|
217
326
|
wayfarer!
|
218
|
-
|
327
|
+
webrick
|
328
|
+
yard
|
219
329
|
|
220
330
|
BUNDLED WITH
|
221
|
-
2.1
|
331
|
+
2.7.1
|
data/Rakefile
CHANGED
@@ -1,62 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "irb"
|
4
|
-
require "
|
5
|
-
require "
|
4
|
+
require "open-uri"
|
5
|
+
require "zip"
|
6
6
|
|
7
|
-
|
7
|
+
require "bundler/gem_tasks"
|
8
|
+
require "pry"
|
9
|
+
|
10
|
+
Dir.glob("rake/*.rake").each { |file| load(file) }
|
8
11
|
|
9
12
|
task default: :build
|
10
13
|
|
11
|
-
desc "Start an IRB session"
|
12
14
|
task :console do
|
13
15
|
require_relative "lib/wayfarer"
|
14
|
-
ARGV.clear
|
15
|
-
IRB.start
|
16
|
-
end
|
17
|
-
|
18
|
-
desc "Build gem"
|
19
|
-
task build: %i[clean] do
|
20
|
-
sh "gem build wayfarer.gemspec --verbose"
|
21
|
-
sh "mkdir #{BUILD_DIR}"
|
22
|
-
sh "mv wayfarer-*.gem #{BUILD_DIR}"
|
23
|
-
end
|
24
|
-
|
25
|
-
desc "Remove build directory"
|
26
|
-
task :clean do
|
27
|
-
sh "rm -rf #{BUILD_DIR}"
|
28
|
-
end
|
29
|
-
|
30
|
-
desc "Install gem"
|
31
|
-
task install: %i[build] do
|
32
|
-
sh "gem install #{BUILD_DIR}/*.gem"
|
33
|
-
end
|
34
|
-
|
35
|
-
namespace :test do
|
36
|
-
desc "Run only isolated tests"
|
37
|
-
RSpec::Core::RakeTask.new :isolated do |task|
|
38
|
-
task.rspec_opts = ["--tag ~selenium --tag ~ferrum --tag ~cli"]
|
39
|
-
end
|
40
|
-
|
41
|
-
desc "Run only Selenium tests"
|
42
|
-
RSpec::Core::RakeTask.new :selenium do |task|
|
43
|
-
task.rspec_opts = ["--tag selenium"]
|
44
|
-
end
|
45
|
-
|
46
|
-
desc "Run only Ferrum tests"
|
47
|
-
RSpec::Core::RakeTask.new :ferrum do |task|
|
48
|
-
task.rspec_opts = ["--tag ferrum"]
|
49
|
-
end
|
50
|
-
|
51
|
-
desc "Run only CLI tests"
|
52
|
-
RSpec::Core::RakeTask.new :cli do |task|
|
53
|
-
task.rspec_opts = ["--tag cli"]
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
desc "Run all tests"
|
58
|
-
RSpec::Core::RakeTask.new(:test)
|
59
16
|
|
60
|
-
|
61
|
-
task.formatters = %w[simple]
|
17
|
+
Pry.start
|
62
18
|
end
|
data/bin/wayfarer
CHANGED
data/docker-compose.yml
CHANGED
@@ -1,32 +1,42 @@
|
|
1
|
-
version: "3"
|
2
1
|
services:
|
3
2
|
wayfarer:
|
4
3
|
build: .
|
4
|
+
tty: true
|
5
5
|
volumes:
|
6
|
-
|
6
|
+
- ./:/opt/app
|
7
7
|
ports:
|
8
|
-
|
8
|
+
- "${WAYFARER_PORT}:${WAYFARER_PORT}"
|
9
|
+
hostname: test
|
9
10
|
environment:
|
10
|
-
|
11
|
+
CI: "${CI}"
|
12
|
+
depends_on:
|
13
|
+
- redis
|
14
|
+
- chrome
|
15
|
+
- firefox
|
16
|
+
- docs
|
11
17
|
|
12
18
|
redis:
|
13
|
-
image:
|
19
|
+
image: ${REDIS_IMAGE}:${REDIS_VERSION}
|
14
20
|
|
15
21
|
chrome:
|
16
|
-
image:
|
22
|
+
image: ${CHROME_IMAGE}:${CHROME_VERSION}
|
17
23
|
ports:
|
18
|
-
|
24
|
+
- "${CHROME_PORT}:${CHROME_PORT}"
|
19
25
|
|
20
26
|
firefox:
|
21
|
-
image:
|
27
|
+
image: ${FIREFOX_IMAGE}:${FIREFOX_VERSION}
|
22
28
|
ports:
|
23
|
-
|
29
|
+
- "${FIREFOX_PORT}:${FIREFOX_PORT}"
|
24
30
|
volumes:
|
25
|
-
|
31
|
+
- /dev/shm:/dev/shm
|
26
32
|
|
27
33
|
docs:
|
28
|
-
image:
|
34
|
+
image: ${DOCS_IMAGE}:${DOCS_VERSION}
|
29
35
|
volumes:
|
30
|
-
|
36
|
+
- ./:/docs
|
31
37
|
ports:
|
32
|
-
|
38
|
+
- "${DOCS_PORT}:${DOCS_PORT}"
|
39
|
+
|
40
|
+
networks:
|
41
|
+
default:
|
42
|
+
driver: bridge
|
@@ -6,10 +6,10 @@ iframe, clicked, and makes the live page behind the screen accessible to
|
|
6
6
|
`#index`:
|
7
7
|
|
8
8
|
```ruby
|
9
|
-
Wayfarer.config
|
9
|
+
Wayfarer.config[:network][:agent] = :ferrum
|
10
10
|
|
11
11
|
class DummyJob < Wayfarer::Base
|
12
|
-
route
|
12
|
+
route.to :index, host: "example.com"
|
13
13
|
|
14
14
|
before_action if: :consent_required? do
|
15
15
|
sleep(5) # If the consent form has a loading animation
|
@@ -6,7 +6,7 @@ Executing JavaScript requires automating a browser.
|
|
6
6
|
|
7
7
|
```ruby
|
8
8
|
class DummyJob < Wayfarer::Base
|
9
|
-
route
|
9
|
+
route.to :index
|
10
10
|
|
11
11
|
def index
|
12
12
|
browser.evaluate("[window.scrollX, window.scrollY]")
|
@@ -18,7 +18,7 @@ Executing JavaScript requires automating a browser.
|
|
18
18
|
|
19
19
|
```ruby
|
20
20
|
class DummyJob < Wayfarer::Base
|
21
|
-
route
|
21
|
+
route.to :index
|
22
22
|
|
23
23
|
def index
|
24
24
|
# Mind the explicit return
|
@@ -31,7 +31,7 @@ Executing JavaScript requires automating a browser.
|
|
31
31
|
|
32
32
|
```ruby
|
33
33
|
class DummyJob < Wayfarer::Base
|
34
|
-
route
|
34
|
+
route.to :index
|
35
35
|
|
36
36
|
def index
|
37
37
|
# Capybara does not return value of JavaScript execution
|