wayfarer 0.4.6 → 0.4.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/lint.yaml +25 -0
- data/.github/workflows/release.yaml +29 -0
- data/.github/workflows/tests.yaml +30 -0
- data/.gitignore +4 -0
- data/.rubocop.yml +5 -0
- data/.vale.ini +5 -0
- data/.yardopts +1 -3
- data/Dockerfile +5 -4
- data/Gemfile +3 -0
- data/Gemfile.lock +107 -102
- data/Rakefile +5 -56
- data/bin/wayfarer +1 -1
- data/docker-compose.yml +20 -9
- data/docs/cookbook/consent_screen.md +2 -2
- data/docs/cookbook/executing_javascript.md +3 -3
- data/docs/cookbook/navigation.md +12 -12
- data/docs/cookbook/querying_html.md +3 -3
- data/docs/cookbook/screenshots.md +2 -2
- data/docs/cookbook/user_agent.md +1 -1
- data/docs/design.md +36 -0
- data/docs/guides/callbacks.md +24 -126
- data/docs/guides/configuration.md +8 -8
- data/docs/guides/handlers.md +60 -0
- data/docs/guides/index.md +1 -0
- data/docs/guides/jobs/error_handling.md +40 -0
- data/docs/guides/jobs.md +99 -31
- data/docs/guides/navigation.md +1 -1
- data/docs/guides/networking/capybara.md +13 -22
- data/docs/guides/networking/custom_adapters.md +82 -41
- data/docs/guides/networking/ferrum.md +4 -4
- data/docs/guides/networking/http.md +9 -13
- data/docs/guides/networking/selenium.md +10 -11
- data/docs/guides/pages.md +76 -10
- data/docs/guides/redis.md +10 -0
- data/docs/guides/routing.md +74 -0
- data/docs/guides/tasks.md +33 -9
- data/docs/guides/tutorial.md +60 -0
- data/docs/guides/user_agents.md +113 -0
- data/docs/index.md +17 -40
- data/docs/reference/cli.md +35 -25
- data/docs/reference/configuration.md +36 -0
- data/lib/wayfarer/base.rb +124 -46
- data/lib/wayfarer/batch_completion.rb +56 -0
- data/lib/wayfarer/callbacks.rb +22 -48
- data/lib/wayfarer/cli/route_printer.rb +71 -57
- data/lib/wayfarer/cli.rb +121 -0
- data/lib/wayfarer/gc.rb +13 -6
- data/lib/wayfarer/handler.rb +15 -7
- data/lib/wayfarer/logging.rb +38 -0
- data/lib/wayfarer/middleware/base.rb +2 -0
- data/lib/wayfarer/middleware/batch_completion.rb +19 -0
- data/lib/wayfarer/middleware/content_type.rb +54 -0
- data/lib/wayfarer/middleware/controller.rb +19 -15
- data/lib/wayfarer/middleware/dedup.rb +16 -13
- data/lib/wayfarer/middleware/dispatch.rb +12 -4
- data/lib/wayfarer/middleware/normalize.rb +12 -11
- data/lib/wayfarer/middleware/redis.rb +15 -0
- data/lib/wayfarer/middleware/router.rb +33 -35
- data/lib/wayfarer/middleware/stage.rb +5 -5
- data/lib/wayfarer/middleware/uri_parser.rb +30 -0
- data/lib/wayfarer/middleware/user_agent.rb +49 -0
- data/lib/wayfarer/networking/capybara.rb +1 -1
- data/lib/wayfarer/networking/context.rb +2 -2
- data/lib/wayfarer/networking/ferrum.rb +2 -2
- data/lib/wayfarer/networking/follow.rb +12 -6
- data/lib/wayfarer/networking/http.rb +1 -1
- data/lib/wayfarer/networking/pool.rb +17 -12
- data/lib/wayfarer/networking/selenium.rb +3 -3
- data/lib/wayfarer/networking/strategy.rb +2 -2
- data/lib/wayfarer/page.rb +36 -14
- data/lib/wayfarer/parsing/xml.rb +6 -6
- data/lib/wayfarer/parsing.rb +24 -0
- data/lib/wayfarer/redis/barrier.rb +13 -21
- data/lib/wayfarer/redis/counter.rb +19 -9
- data/lib/wayfarer/redis/pool.rb +1 -1
- data/lib/wayfarer/redis/resettable.rb +19 -0
- data/lib/wayfarer/routing/dsl.rb +1 -0
- data/lib/wayfarer/routing/matchers/path.rb +4 -2
- data/lib/wayfarer/routing/root_route.rb +5 -1
- data/lib/wayfarer/routing/route.rb +4 -14
- data/lib/wayfarer/stringify.rb +22 -30
- data/lib/wayfarer/task.rb +12 -18
- data/lib/wayfarer.rb +28 -1
- data/mkdocs.yml +52 -7
- data/rake/docs.rake +26 -0
- data/rake/lint.rake +105 -0
- data/rake/release.rake +29 -0
- data/rake/tests.rake +28 -0
- data/requirements.txt +1 -1
- data/spec/base_spec.rb +140 -160
- data/spec/batch_completion_spec.rb +104 -0
- data/spec/cli/job_spec.rb +19 -23
- data/spec/cli/routing_spec.rb +101 -0
- data/spec/cli/version_spec.rb +1 -1
- data/spec/factories/task.rb +7 -1
- data/spec/fixtures/dummy_job.rb +5 -3
- data/spec/gc_spec.rb +8 -50
- data/spec/handler_spec.rb +1 -1
- data/spec/integration/callbacks_spec.rb +157 -45
- data/spec/integration/content_type_spec.rb +145 -0
- data/spec/integration/gc_spec.rb +44 -0
- data/spec/integration/handler_spec.rb +66 -0
- data/spec/integration/page_spec.rb +44 -29
- data/spec/integration/params_spec.rb +33 -25
- data/spec/integration/parsing_spec.rb +125 -0
- data/spec/integration/routing_spec.rb +18 -0
- data/spec/integration/stage_spec.rb +27 -20
- data/spec/middleware/batch_completion_spec.rb +34 -0
- data/spec/middleware/chain_spec.rb +8 -8
- data/spec/middleware/content_type_spec.rb +86 -0
- data/spec/middleware/controller_spec.rb +5 -5
- data/spec/middleware/dedup_spec.rb +38 -55
- data/spec/middleware/dispatch_spec.rb +23 -7
- data/spec/middleware/normalize_spec.rb +44 -13
- data/spec/middleware/router_spec.rb +29 -30
- data/spec/middleware/stage_spec.rb +8 -8
- data/spec/middleware/uri_parser_spec.rb +53 -0
- data/spec/middleware/{fetch_spec.rb → user_agent_spec.rb} +28 -27
- data/spec/networking/context_spec.rb +1 -1
- data/spec/networking/follow_spec.rb +2 -2
- data/spec/networking/pool_spec.rb +5 -5
- data/spec/networking/strategy.rb +2 -2
- data/spec/page_spec.rb +42 -20
- data/spec/parsing/xml_spec.rb +11 -12
- data/spec/redis/barrier_spec.rb +8 -48
- data/spec/redis/counter_spec.rb +13 -1
- data/spec/redis/pool_spec.rb +1 -1
- data/spec/spec_helpers.rb +27 -16
- data/spec/support/test_app.rb +8 -0
- data/spec/task_spec.rb +3 -24
- data/spec/wayfarer_spec.rb +1 -1
- data/wayfarer.gemspec +4 -3
- metadata +61 -51
- data/.github/workflows/ci.yaml +0 -32
- data/docs/guides/error_handling.md +0 -53
- data/docs/guides/networking.md +0 -94
- data/docs/guides/performance.md +0 -130
- data/docs/guides/reliability.md +0 -41
- data/docs/guides/routing/steering.md +0 -30
- data/docs/reference/api/base.md +0 -48
- data/docs/reference/configuration_keys.md +0 -43
- data/docs/reference/environment_variables.md +0 -83
- data/lib/wayfarer/cli/base.rb +0 -45
- data/lib/wayfarer/cli/generate.rb +0 -17
- data/lib/wayfarer/cli/job.rb +0 -56
- data/lib/wayfarer/cli/route.rb +0 -29
- data/lib/wayfarer/cli/runner.rb +0 -34
- data/lib/wayfarer/cli/templates/Gemfile.tt +0 -5
- data/lib/wayfarer/cli/templates/job.rb.tt +0 -10
- data/lib/wayfarer/config/capybara.rb +0 -10
- data/lib/wayfarer/config/ferrum.rb +0 -11
- data/lib/wayfarer/config/networking.rb +0 -29
- data/lib/wayfarer/config/redis.rb +0 -14
- data/lib/wayfarer/config/root.rb +0 -11
- data/lib/wayfarer/config/selenium.rb +0 -21
- data/lib/wayfarer/config/strconv.rb +0 -45
- data/lib/wayfarer/config/struct.rb +0 -72
- data/lib/wayfarer/middleware/fetch.rb +0 -56
- data/lib/wayfarer/redis/connection.rb +0 -13
- data/lib/wayfarer/redis/version.rb +0 -19
- data/lib/wayfarer/routing/router.rb +0 -28
- data/spec/callbacks_spec.rb +0 -102
- data/spec/cli/generate_spec.rb +0 -39
- data/spec/config/capybara_spec.rb +0 -18
- data/spec/config/ferrum_spec.rb +0 -24
- data/spec/config/networking_spec.rb +0 -73
- data/spec/config/redis_spec.rb +0 -32
- data/spec/config/root_spec.rb +0 -31
- data/spec/config/selenium_spec.rb +0 -56
- data/spec/config/strconv_spec.rb +0 -58
- data/spec/config/struct_spec.rb +0 -66
- data/spec/integration/steering_spec.rb +0 -57
- data/spec/redis/version_spec.rb +0 -13
- data/spec/routing/router_spec.rb +0 -24
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 98db45c7ede4e4949b4a18c5560477a64982e78f6640e388de2809b5cab02a27
|
4
|
+
data.tar.gz: dc69bf066254600c4b9021234bf4623edd039c87d5cb79166bb7ef174ebe644b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cc001901a22a9d9a84aa16b3946cbc7e0954db1b20a16d3c105411039273138c15ffa60bec57503aa4b73c39b39fde78ad7600697a3f461620316ed05583e91d
|
7
|
+
data.tar.gz: 0be5255180603e2e9d7171aec9f54008b21e1e4ebf6c3e7069535a9d01d59e620aef4c171a84b5d46a754e45043f1ec95c2672847fc03231cff880a3796e48cf
|
@@ -0,0 +1,25 @@
|
|
1
|
+
name: lint
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches:
|
6
|
+
- '*'
|
7
|
+
env:
|
8
|
+
CI: true
|
9
|
+
|
10
|
+
jobs:
|
11
|
+
lint:
|
12
|
+
runs-on: ubuntu-latest
|
13
|
+
steps:
|
14
|
+
- uses: actions/checkout@v2
|
15
|
+
|
16
|
+
- name: Start services
|
17
|
+
run: docker-compose up -d wayfarer
|
18
|
+
|
19
|
+
- name: RuboCop
|
20
|
+
if: success() || failure()
|
21
|
+
run: docker-compose run --rm --name test wayfarer bundle exec rake lint:rubocop
|
22
|
+
|
23
|
+
- name: Vale
|
24
|
+
if: success() || failure()
|
25
|
+
run: docker-compose run --rm --name test wayfarer bundle exec rake lint:vale
|
@@ -0,0 +1,29 @@
|
|
1
|
+
name: release
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: [master]
|
6
|
+
|
7
|
+
jobs:
|
8
|
+
release:
|
9
|
+
runs-on: ubuntu-latest
|
10
|
+
steps:
|
11
|
+
- uses: actions/checkout@v2
|
12
|
+
|
13
|
+
- run: docker compose up -d
|
14
|
+
|
15
|
+
- run: docker compose exec -T wayfarer git config --global --add safe.directory /opt/app
|
16
|
+
|
17
|
+
- name: Release gem
|
18
|
+
env:
|
19
|
+
GIT_AUTHOR_NAME: Dominic Bauer
|
20
|
+
GIT_AUTHOR_EMAIL: bauerd@users.noreply.github.com
|
21
|
+
GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }}
|
22
|
+
run: |
|
23
|
+
docker compose exec -T \
|
24
|
+
-e GEM_HOST_API_KEY="${{ env.GEM_HOST_API_KEY }}" \
|
25
|
+
-e GIT_AUTHOR_NAME="${{ env.GIT_AUTHOR_NAME }}" \
|
26
|
+
-e GIT_AUTHOR_EMAIL="${{ env.GIT_AUTHOR_EMAIL }}" \
|
27
|
+
-e GIT_COMMITTER_NAME="${{ env.GIT_AUTHOR_NAME }}" \
|
28
|
+
-e GIT_COMMITTER_EMAIL="${{ env.GIT_AUTHOR_EMAIL }}" \
|
29
|
+
wayfarer bundle exec rake release
|
@@ -0,0 +1,30 @@
|
|
1
|
+
name: tests
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: ["*"]
|
6
|
+
|
7
|
+
env:
|
8
|
+
CI: true
|
9
|
+
|
10
|
+
jobs:
|
11
|
+
tests:
|
12
|
+
runs-on: ubuntu-latest
|
13
|
+
|
14
|
+
steps:
|
15
|
+
- uses: actions/checkout@v2
|
16
|
+
|
17
|
+
- name: Set up Docker Compose
|
18
|
+
run: docker-compose up -d
|
19
|
+
|
20
|
+
- name: Isolated tests
|
21
|
+
run: docker-compose exec -T wayfarer bundle exec rake test:isolated
|
22
|
+
|
23
|
+
- name: Ferrum tests
|
24
|
+
run: docker-compose exec -T wayfarer bundle exec rake test:ferrum
|
25
|
+
|
26
|
+
- name: Selenium tests
|
27
|
+
run: docker-compose exec -T wayfarer bundle exec rake test:selenium
|
28
|
+
|
29
|
+
- name: CLI tests
|
30
|
+
run: docker-compose exec -T wayfarer bundle exec rake test:cli
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
@@ -27,12 +27,16 @@ Style/ClassAndModuleChildren:
|
|
27
27
|
Style/Documentation:
|
28
28
|
Enabled: false
|
29
29
|
|
30
|
+
Style/NumericPredicate:
|
31
|
+
EnforcedStyle: comparison
|
32
|
+
|
30
33
|
Metrics/MethodLength:
|
31
34
|
Max: 20
|
32
35
|
|
33
36
|
Metrics/BlockLength:
|
34
37
|
Exclude:
|
35
38
|
- "spec/**/*"
|
39
|
+
- "**/*.rake"
|
36
40
|
|
37
41
|
Metrics/AbcSize:
|
38
42
|
Max: 30
|
@@ -45,3 +49,4 @@ Metrics/CyclomaticComplexity:
|
|
45
49
|
Lint/ConstantDefinitionInBlock:
|
46
50
|
Exclude:
|
47
51
|
- "spec/**/*"
|
52
|
+
|
data/.vale.ini
ADDED
data/.yardopts
CHANGED
data/Dockerfile
CHANGED
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
wayfarer (0.4.
|
5
|
-
activejob (>=
|
4
|
+
wayfarer (0.4.7)
|
5
|
+
activejob (>= 7.1)
|
6
6
|
addressable (~> 2.8)
|
7
7
|
capybara (~> 3.0)
|
8
8
|
connection_pool (~> 2.2)
|
@@ -18,23 +18,30 @@ PATH
|
|
18
18
|
redis (~> 4.4, < 4.5)
|
19
19
|
selenium-webdriver (~> 3.4)
|
20
20
|
thor (~> 1.0)
|
21
|
+
zeitwerk (~> 2.4)
|
21
22
|
|
22
23
|
GEM
|
23
24
|
remote: https://rubygems.org/
|
24
25
|
specs:
|
25
|
-
activejob (
|
26
|
-
activesupport (=
|
26
|
+
activejob (7.1.3.1)
|
27
|
+
activesupport (= 7.1.3.1)
|
27
28
|
globalid (>= 0.3.6)
|
28
|
-
activesupport (
|
29
|
+
activesupport (7.1.3.1)
|
30
|
+
base64
|
31
|
+
bigdecimal
|
29
32
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
33
|
+
connection_pool (>= 2.2.5)
|
34
|
+
drb
|
30
35
|
i18n (>= 1.6, < 2)
|
31
36
|
minitest (>= 5.1)
|
37
|
+
mutex_m
|
32
38
|
tzinfo (~> 2.0)
|
33
|
-
|
34
|
-
|
35
|
-
public_suffix (>= 2.0.2, < 5.0)
|
39
|
+
addressable (2.8.6)
|
40
|
+
public_suffix (>= 2.0.2, < 6.0)
|
36
41
|
ast (2.4.2)
|
37
|
-
|
42
|
+
base64 (0.2.0)
|
43
|
+
bigdecimal (3.1.6)
|
44
|
+
capybara (3.39.2)
|
38
45
|
addressable
|
39
46
|
matrix
|
40
47
|
mini_mime (>= 0.1.3)
|
@@ -44,127 +51,125 @@ GEM
|
|
44
51
|
regexp_parser (>= 1.5, < 3.0)
|
45
52
|
xpath (~> 3.2)
|
46
53
|
childprocess (3.0.0)
|
47
|
-
cliver (0.3.2)
|
48
54
|
coderay (1.1.3)
|
49
|
-
concurrent-ruby (1.
|
50
|
-
connection_pool (2.
|
51
|
-
cuprite (0.
|
52
|
-
capybara (
|
53
|
-
ferrum (~> 0.
|
54
|
-
diff-lcs (1.
|
55
|
+
concurrent-ruby (1.2.3)
|
56
|
+
connection_pool (2.4.1)
|
57
|
+
cuprite (0.15)
|
58
|
+
capybara (~> 3.0)
|
59
|
+
ferrum (~> 0.14.0)
|
60
|
+
diff-lcs (1.5.1)
|
55
61
|
docile (1.4.0)
|
56
|
-
domain_name (0.
|
57
|
-
|
58
|
-
|
62
|
+
domain_name (0.6.20240107)
|
63
|
+
drb (2.2.0)
|
64
|
+
ruby2_keywords
|
65
|
+
factory_bot (6.4.5)
|
59
66
|
activesupport (>= 5.0.0)
|
60
67
|
faker (1.9.6)
|
61
68
|
i18n (>= 0.7)
|
62
|
-
faraday (
|
63
|
-
|
64
|
-
faraday-
|
65
|
-
faraday-excon (~> 1.1)
|
66
|
-
faraday-httpclient (~> 1.0)
|
67
|
-
faraday-multipart (~> 1.0)
|
68
|
-
faraday-net_http (~> 1.0)
|
69
|
-
faraday-net_http_persistent (~> 1.0)
|
70
|
-
faraday-patron (~> 1.0)
|
71
|
-
faraday-rack (~> 1.0)
|
72
|
-
faraday-retry (~> 1.0)
|
69
|
+
faraday (2.8.1)
|
70
|
+
base64
|
71
|
+
faraday-net_http (>= 2.0, < 3.1)
|
73
72
|
ruby2_keywords (>= 0.0.4)
|
74
73
|
faraday-cookie_jar (0.0.7)
|
75
74
|
faraday (>= 0.8.0)
|
76
75
|
http-cookie (~> 1.0.0)
|
77
|
-
faraday-em_http (1.0.0)
|
78
|
-
faraday-em_synchrony (1.0.0)
|
79
76
|
faraday-encoding (0.0.5)
|
80
77
|
faraday
|
81
|
-
faraday-
|
82
|
-
|
78
|
+
faraday-follow_redirects (0.3.0)
|
79
|
+
faraday (>= 1, < 3)
|
80
|
+
faraday-gzip (1.0.0)
|
81
|
+
faraday (>= 1.0)
|
82
|
+
zlib (~> 2.1)
|
83
|
+
faraday-http-cache (2.5.1)
|
83
84
|
faraday (>= 0.8)
|
84
|
-
faraday-
|
85
|
-
faraday-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
faraday-patron (1.0.0)
|
90
|
-
faraday-rack (1.0.0)
|
91
|
-
faraday-retry (1.0.3)
|
92
|
-
faraday_middleware (1.2.0)
|
93
|
-
faraday (~> 1.0)
|
94
|
-
fastimage (2.2.6)
|
95
|
-
ferrum (0.11)
|
85
|
+
faraday-net_http (3.0.2)
|
86
|
+
faraday-retry (2.2.1)
|
87
|
+
faraday (~> 2.0)
|
88
|
+
fastimage (2.3.1)
|
89
|
+
ferrum (0.14)
|
96
90
|
addressable (~> 2.5)
|
97
|
-
cliver (~> 0.3)
|
98
91
|
concurrent-ruby (~> 1.1)
|
92
|
+
webrick (~> 1.7)
|
99
93
|
websocket-driver (>= 0.6, < 0.8)
|
100
|
-
|
101
|
-
|
102
|
-
|
94
|
+
ffi (1.16.3)
|
95
|
+
globalid (1.2.1)
|
96
|
+
activesupport (>= 6.1)
|
97
|
+
http-cookie (1.0.6)
|
103
98
|
domain_name (~> 0.5)
|
104
|
-
i18n (1.
|
99
|
+
i18n (1.14.1)
|
105
100
|
concurrent-ruby (~> 1.0)
|
101
|
+
listen (3.9.0)
|
102
|
+
rb-fsevent (~> 0.10, >= 0.10.3)
|
103
|
+
rb-inotify (~> 0.9, >= 0.9.10)
|
106
104
|
matrix (0.4.2)
|
107
|
-
metainspector (5.
|
108
|
-
addressable (~> 2.
|
109
|
-
faraday (~>
|
105
|
+
metainspector (5.15.0)
|
106
|
+
addressable (~> 2.8.4)
|
107
|
+
faraday (~> 2.5)
|
110
108
|
faraday-cookie_jar (~> 0.0)
|
111
109
|
faraday-encoding (~> 0.0)
|
112
|
-
faraday-
|
113
|
-
|
110
|
+
faraday-follow_redirects (~> 0.3)
|
111
|
+
faraday-gzip (>= 0.1, < 2.0)
|
112
|
+
faraday-http-cache (~> 2.5)
|
113
|
+
faraday-retry (~> 2.0)
|
114
114
|
fastimage (~> 2.2)
|
115
115
|
nesty (~> 1.0)
|
116
|
-
nokogiri (~> 1.
|
116
|
+
nokogiri (~> 1.13)
|
117
117
|
method_source (1.0.0)
|
118
|
-
mime-types (3.
|
118
|
+
mime-types (3.5.2)
|
119
119
|
mime-types-data (~> 3.2015)
|
120
|
-
mime-types-data (3.
|
121
|
-
mini_mime (1.1.
|
122
|
-
mini_portile2 (2.
|
123
|
-
minitest (5.
|
124
|
-
mock_redis (0.
|
125
|
-
|
126
|
-
multipart-post (2.1.1)
|
127
|
-
mustermann (1.1.1)
|
120
|
+
mime-types-data (3.2024.0507)
|
121
|
+
mini_mime (1.1.5)
|
122
|
+
mini_portile2 (2.8.5)
|
123
|
+
minitest (5.22.2)
|
124
|
+
mock_redis (0.41.0)
|
125
|
+
mustermann (1.1.2)
|
128
126
|
ruby2_keywords (~> 0.0.1)
|
127
|
+
mutex_m (0.2.0)
|
129
128
|
nesty (1.0.2)
|
130
129
|
net-http-persistent (3.1.0)
|
131
130
|
connection_pool (~> 2.2)
|
132
|
-
nokogiri (1.
|
133
|
-
mini_portile2 (~> 2.
|
131
|
+
nokogiri (1.15.5)
|
132
|
+
mini_portile2 (~> 2.8.2)
|
134
133
|
racc (~> 1.4)
|
135
134
|
normalize_url (0.0.6)
|
136
135
|
addressable (~> 2.3)
|
137
|
-
parallel (1.
|
138
|
-
parser (3.0.
|
136
|
+
parallel (1.24.0)
|
137
|
+
parser (3.3.0.5)
|
139
138
|
ast (~> 2.4.1)
|
140
|
-
|
139
|
+
racc
|
140
|
+
pry (0.14.2)
|
141
141
|
coderay (~> 1.1)
|
142
142
|
method_source (~> 1.0)
|
143
|
-
public_suffix (
|
144
|
-
racc (1.
|
143
|
+
public_suffix (5.0.4)
|
144
|
+
racc (1.7.3)
|
145
145
|
rack (1.6.13)
|
146
146
|
rack-protection (1.5.5)
|
147
147
|
rack
|
148
|
-
rack-test (
|
149
|
-
rack (>= 1.
|
150
|
-
rainbow (3.
|
148
|
+
rack-test (2.1.0)
|
149
|
+
rack (>= 1.3)
|
150
|
+
rainbow (3.1.1)
|
151
151
|
rake (10.5.0)
|
152
|
+
rb-fsevent (0.11.2)
|
153
|
+
rb-inotify (0.10.1)
|
154
|
+
ffi (~> 1.0)
|
152
155
|
redis (4.4.0)
|
153
|
-
regexp_parser (2.
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
rspec-
|
159
|
-
|
160
|
-
rspec-
|
161
|
-
rspec-
|
156
|
+
regexp_parser (2.9.0)
|
157
|
+
rerun (0.14.0)
|
158
|
+
listen (~> 3.0)
|
159
|
+
rexml (3.2.6)
|
160
|
+
rspec (3.13.0)
|
161
|
+
rspec-core (~> 3.13.0)
|
162
|
+
rspec-expectations (~> 3.13.0)
|
163
|
+
rspec-mocks (~> 3.13.0)
|
164
|
+
rspec-core (3.13.0)
|
165
|
+
rspec-support (~> 3.13.0)
|
166
|
+
rspec-expectations (3.13.0)
|
162
167
|
diff-lcs (>= 1.2.0, < 2.0)
|
163
|
-
rspec-support (~> 3.
|
164
|
-
rspec-mocks (3.
|
168
|
+
rspec-support (~> 3.13.0)
|
169
|
+
rspec-mocks (3.13.0)
|
165
170
|
diff-lcs (>= 1.2.0, < 2.0)
|
166
|
-
rspec-support (~> 3.
|
167
|
-
rspec-support (3.
|
171
|
+
rspec-support (~> 3.13.0)
|
172
|
+
rspec-support (3.13.1)
|
168
173
|
rubocop (0.93.1)
|
169
174
|
parallel (~> 1.10)
|
170
175
|
parser (>= 2.7.1.5)
|
@@ -174,9 +179,9 @@ GEM
|
|
174
179
|
rubocop-ast (>= 0.6.0)
|
175
180
|
ruby-progressbar (~> 1.7)
|
176
181
|
unicode-display_width (>= 1.4.0, < 2.0)
|
177
|
-
rubocop-ast (1.
|
178
|
-
parser (>= 3.
|
179
|
-
ruby-progressbar (1.
|
182
|
+
rubocop-ast (1.30.0)
|
183
|
+
parser (>= 3.2.1.0)
|
184
|
+
ruby-progressbar (1.13.0)
|
180
185
|
ruby2_keywords (0.0.5)
|
181
186
|
rubyzip (2.3.2)
|
182
187
|
selenium-webdriver (3.142.7)
|
@@ -186,21 +191,20 @@ GEM
|
|
186
191
|
rack (~> 1.5)
|
187
192
|
rack-protection (~> 1.4)
|
188
193
|
tilt (>= 1.3, < 3)
|
189
|
-
thor (1.
|
190
|
-
tilt (2.0
|
191
|
-
tzinfo (2.0.
|
194
|
+
thor (1.3.1)
|
195
|
+
tilt (2.3.0)
|
196
|
+
tzinfo (2.0.6)
|
192
197
|
concurrent-ruby (~> 1.0)
|
193
|
-
unf (0.1.4)
|
194
|
-
unf_ext
|
195
|
-
unf_ext (0.0.8)
|
196
198
|
unicode-display_width (1.8.0)
|
197
|
-
|
199
|
+
webrick (1.8.1)
|
200
|
+
websocket-driver (0.7.6)
|
198
201
|
websocket-extensions (>= 0.1.0)
|
199
202
|
websocket-extensions (0.1.5)
|
200
203
|
xpath (3.2.0)
|
201
204
|
nokogiri (~> 1.8)
|
202
|
-
yard (0.9.
|
203
|
-
zeitwerk (2.
|
205
|
+
yard (0.9.34)
|
206
|
+
zeitwerk (2.6.15)
|
207
|
+
zlib (2.1.1)
|
204
208
|
|
205
209
|
PLATFORMS
|
206
210
|
ruby
|
@@ -211,11 +215,12 @@ DEPENDENCIES
|
|
211
215
|
faker (~> 1.7)
|
212
216
|
pry (~> 0.10)
|
213
217
|
rake (~> 10.4)
|
218
|
+
rerun
|
214
219
|
rspec (~> 3.4)
|
215
220
|
rubocop (~> 0.49)
|
216
221
|
sinatra (~> 1.4)
|
217
222
|
wayfarer!
|
218
|
-
yard
|
223
|
+
yard
|
219
224
|
|
220
225
|
BUNDLED WITH
|
221
226
|
2.1.4
|
data/Rakefile
CHANGED
@@ -1,62 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "irb"
|
4
|
-
require "
|
5
|
-
require "
|
4
|
+
require "open-uri"
|
5
|
+
require "zip"
|
6
6
|
|
7
|
-
|
7
|
+
require "bundler/gem_tasks"
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
desc "Start an IRB session"
|
12
|
-
task :console do
|
13
|
-
require_relative "lib/wayfarer"
|
14
|
-
ARGV.clear
|
15
|
-
IRB.start
|
16
|
-
end
|
17
|
-
|
18
|
-
desc "Build gem"
|
19
|
-
task build: %i[clean] do
|
20
|
-
sh "gem build wayfarer.gemspec --verbose"
|
21
|
-
sh "mkdir #{BUILD_DIR}"
|
22
|
-
sh "mv wayfarer-*.gem #{BUILD_DIR}"
|
23
|
-
end
|
24
|
-
|
25
|
-
desc "Remove build directory"
|
26
|
-
task :clean do
|
27
|
-
sh "rm -rf #{BUILD_DIR}"
|
28
|
-
end
|
29
|
-
|
30
|
-
desc "Install gem"
|
31
|
-
task install: %i[build] do
|
32
|
-
sh "gem install #{BUILD_DIR}/*.gem"
|
33
|
-
end
|
34
|
-
|
35
|
-
namespace :test do
|
36
|
-
desc "Run only isolated tests"
|
37
|
-
RSpec::Core::RakeTask.new :isolated do |task|
|
38
|
-
task.rspec_opts = ["--tag ~selenium --tag ~ferrum --tag ~cli"]
|
39
|
-
end
|
9
|
+
Dir.glob("rake/*.rake").each { |file| load(file) }
|
40
10
|
|
41
|
-
|
42
|
-
RSpec::Core::RakeTask.new :selenium do |task|
|
43
|
-
task.rspec_opts = ["--tag selenium"]
|
44
|
-
end
|
45
|
-
|
46
|
-
desc "Run only Ferrum tests"
|
47
|
-
RSpec::Core::RakeTask.new :ferrum do |task|
|
48
|
-
task.rspec_opts = ["--tag ferrum"]
|
49
|
-
end
|
50
|
-
|
51
|
-
desc "Run only CLI tests"
|
52
|
-
RSpec::Core::RakeTask.new :cli do |task|
|
53
|
-
task.rspec_opts = ["--tag cli"]
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
desc "Run all tests"
|
58
|
-
RSpec::Core::RakeTask.new(:test)
|
59
|
-
|
60
|
-
RuboCop::RakeTask.new do |task|
|
61
|
-
task.formatters = %w[simple]
|
62
|
-
end
|
11
|
+
task default: :build
|
data/bin/wayfarer
CHANGED
data/docker-compose.yml
CHANGED
@@ -2,12 +2,19 @@ version: "3"
|
|
2
2
|
services:
|
3
3
|
wayfarer:
|
4
4
|
build: .
|
5
|
+
tty: true
|
5
6
|
volumes:
|
6
|
-
|
7
|
+
- "./:/opt/app"
|
7
8
|
ports:
|
8
|
-
|
9
|
+
- "9876:9876"
|
9
10
|
environment:
|
10
|
-
|
11
|
+
- CI=true
|
12
|
+
hostname: test
|
13
|
+
depends_on:
|
14
|
+
- redis
|
15
|
+
- chrome
|
16
|
+
- firefox
|
17
|
+
- docs
|
11
18
|
|
12
19
|
redis:
|
13
20
|
image: redis
|
@@ -15,18 +22,22 @@ services:
|
|
15
22
|
chrome:
|
16
23
|
image: browserless/chrome
|
17
24
|
ports:
|
18
|
-
|
25
|
+
- "3000:3000"
|
19
26
|
|
20
27
|
firefox:
|
21
28
|
image: selenium/standalone-firefox:4.0.0-rc-2-prerelease-20210923
|
22
29
|
ports:
|
23
|
-
|
30
|
+
- "4444:4444"
|
24
31
|
volumes:
|
25
|
-
|
32
|
+
- "/dev/shm:/dev/shm"
|
26
33
|
|
27
34
|
docs:
|
28
|
-
image: squidfunk/mkdocs-material:
|
35
|
+
image: squidfunk/mkdocs-material:9.5.9
|
29
36
|
volumes:
|
30
|
-
|
37
|
+
- "./:/docs"
|
31
38
|
ports:
|
32
|
-
|
39
|
+
- "8000:8000"
|
40
|
+
|
41
|
+
networks:
|
42
|
+
default:
|
43
|
+
driver: bridge
|
@@ -6,10 +6,10 @@ iframe, clicked, and makes the live page behind the screen accessible to
|
|
6
6
|
`#index`:
|
7
7
|
|
8
8
|
```ruby
|
9
|
-
Wayfarer.config
|
9
|
+
Wayfarer.config[:network][:agent] = :ferrum
|
10
10
|
|
11
11
|
class DummyJob < Wayfarer::Base
|
12
|
-
route
|
12
|
+
route.to :index, host: "example.com"
|
13
13
|
|
14
14
|
before_action if: :consent_required? do
|
15
15
|
sleep(5) # If the consent form has a loading animation
|
@@ -6,7 +6,7 @@ Executing JavaScript requires automating a browser.
|
|
6
6
|
|
7
7
|
```ruby
|
8
8
|
class DummyJob < Wayfarer::Base
|
9
|
-
route
|
9
|
+
route.to :index
|
10
10
|
|
11
11
|
def index
|
12
12
|
browser.evaluate("[window.scrollX, window.scrollY]")
|
@@ -18,7 +18,7 @@ Executing JavaScript requires automating a browser.
|
|
18
18
|
|
19
19
|
```ruby
|
20
20
|
class DummyJob < Wayfarer::Base
|
21
|
-
route
|
21
|
+
route.to :index
|
22
22
|
|
23
23
|
def index
|
24
24
|
# Mind the explicit return
|
@@ -31,7 +31,7 @@ Executing JavaScript requires automating a browser.
|
|
31
31
|
|
32
32
|
```ruby
|
33
33
|
class DummyJob < Wayfarer::Base
|
34
|
-
route
|
34
|
+
route.to :index
|
35
35
|
|
36
36
|
def index
|
37
37
|
# Capybara does not return value of JavaScript execution
|