wayfarer 0.4.7 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.env +17 -0
- data/.github/workflows/lint.yaml +8 -6
- data/.github/workflows/release.yaml +4 -3
- data/.github/workflows/tests.yaml +5 -14
- data/.gitignore +2 -2
- data/.rubocop.yml +31 -0
- data/.vale.ini +6 -3
- data/Dockerfile +3 -2
- data/Gemfile +21 -0
- data/Gemfile.lock +233 -128
- data/Rakefile +7 -0
- data/docker-compose.yml +13 -14
- data/docs/guides/callbacks.md +3 -1
- data/docs/guides/configuration.md +10 -35
- data/docs/guides/development.md +67 -0
- data/docs/guides/handlers.md +7 -7
- data/docs/guides/jobs.md +54 -11
- data/docs/guides/networking/custom_adapters.md +31 -10
- data/docs/guides/pages.md +24 -22
- data/docs/guides/routing.md +116 -34
- data/docs/guides/tasks.md +30 -10
- data/docs/guides/tutorial.md +23 -17
- data/docs/guides/user_agents.md +11 -9
- data/lib/wayfarer/base.rb +9 -8
- data/lib/wayfarer/batch_completion.rb +18 -14
- data/lib/wayfarer/callbacks.rb +14 -14
- data/lib/wayfarer/cli/route_printer.rb +78 -96
- data/lib/wayfarer/cli.rb +12 -30
- data/lib/wayfarer/gc.rb +6 -1
- data/lib/wayfarer/kv.rb +28 -0
- data/lib/wayfarer/middleware/chain.rb +7 -1
- data/lib/wayfarer/middleware/content_type.rb +20 -15
- data/lib/wayfarer/middleware/dedup.rb +9 -3
- data/lib/wayfarer/middleware/dispatch.rb +7 -2
- data/lib/wayfarer/middleware/normalize.rb +4 -12
- data/lib/wayfarer/middleware/router.rb +1 -1
- data/lib/wayfarer/middleware/uri_parser.rb +4 -3
- data/lib/wayfarer/networking/context.rb +12 -1
- data/lib/wayfarer/networking/ferrum.rb +1 -4
- data/lib/wayfarer/networking/follow.rb +2 -1
- data/lib/wayfarer/networking/pool.rb +12 -7
- data/lib/wayfarer/networking/selenium.rb +15 -7
- data/lib/wayfarer/page.rb +0 -2
- data/lib/wayfarer/parsing/xml.rb +1 -1
- data/lib/wayfarer/parsing.rb +2 -5
- data/lib/wayfarer/redis/barrier.rb +15 -2
- data/lib/wayfarer/redis/counter.rb +1 -2
- data/lib/wayfarer/routing/dsl.rb +166 -31
- data/lib/wayfarer/routing/hash_stack.rb +33 -0
- data/lib/wayfarer/routing/matchers/custom.rb +8 -5
- data/lib/wayfarer/routing/matchers/{suffix.rb → empty_params.rb} +2 -6
- data/lib/wayfarer/routing/matchers/host.rb +15 -9
- data/lib/wayfarer/routing/matchers/path.rb +11 -33
- data/lib/wayfarer/routing/matchers/query.rb +41 -17
- data/lib/wayfarer/routing/matchers/result.rb +12 -0
- data/lib/wayfarer/routing/matchers/scheme.rb +13 -5
- data/lib/wayfarer/routing/matchers/url.rb +13 -5
- data/lib/wayfarer/routing/path_consumer.rb +130 -0
- data/lib/wayfarer/routing/path_finder.rb +151 -23
- data/lib/wayfarer/routing/result.rb +1 -1
- data/lib/wayfarer/routing/root_route.rb +14 -2
- data/lib/wayfarer/routing/route.rb +71 -14
- data/lib/wayfarer/routing/serializable.rb +28 -0
- data/lib/wayfarer/routing/sub_route.rb +53 -0
- data/lib/wayfarer/routing/target_route.rb +17 -1
- data/lib/wayfarer/stringify.rb +1 -2
- data/lib/wayfarer/task.rb +3 -5
- data/lib/wayfarer/uri/normalization.rb +120 -0
- data/lib/wayfarer.rb +50 -10
- data/mise.toml +2 -0
- data/mkdocs.yml +8 -17
- data/rake/lint.rake +0 -96
- data/rake/release.rake +5 -11
- data/rake/tests.rake +8 -4
- data/requirements.txt +1 -1
- data/spec/factories/job.rb +8 -0
- data/spec/factories/middleware.rb +2 -2
- data/spec/factories/path_finder.rb +11 -0
- data/spec/factories/redis.rb +19 -0
- data/spec/factories/task.rb +39 -1
- data/spec/spec_helpers.rb +50 -57
- data/spec/support/active_job_helpers.rb +8 -0
- data/spec/support/integration_helpers.rb +21 -0
- data/spec/support/redis_helpers.rb +9 -0
- data/spec/support/test_app.rb +64 -43
- data/spec/{base_spec.rb → wayfarer/base_spec.rb} +32 -36
- data/spec/wayfarer/batch_completion_spec.rb +142 -0
- data/spec/wayfarer/cli/job_spec.rb +88 -0
- data/spec/wayfarer/cli/routing_spec.rb +322 -0
- data/spec/{cli → wayfarer/cli}/version_spec.rb +1 -1
- data/spec/wayfarer/gc_spec.rb +29 -0
- data/spec/{handler_spec.rb → wayfarer/handler_spec.rb} +1 -3
- data/spec/{integration → wayfarer/integration}/callbacks_spec.rb +9 -6
- data/spec/wayfarer/integration/content_type_spec.rb +37 -0
- data/spec/wayfarer/integration/custom_routing_spec.rb +51 -0
- data/spec/{integration → wayfarer/integration}/gc_spec.rb +9 -13
- data/spec/{integration → wayfarer/integration}/handler_spec.rb +9 -10
- data/spec/{integration → wayfarer/integration}/page_spec.rb +8 -6
- data/spec/{integration → wayfarer/integration}/params_spec.rb +4 -4
- data/spec/{integration → wayfarer/integration}/parsing_spec.rb +7 -33
- data/spec/wayfarer/integration/retry_spec.rb +112 -0
- data/spec/{integration → wayfarer/integration}/stage_spec.rb +5 -5
- data/spec/{middleware → wayfarer/middleware}/batch_completion_spec.rb +4 -5
- data/spec/{middleware → wayfarer/middleware}/chain_spec.rb +20 -15
- data/spec/{middleware → wayfarer/middleware}/content_type_spec.rb +18 -21
- data/spec/{middleware → wayfarer/middleware}/controller_spec.rb +22 -20
- data/spec/wayfarer/middleware/dedup_spec.rb +66 -0
- data/spec/wayfarer/middleware/normalize_spec.rb +32 -0
- data/spec/{middleware → wayfarer/middleware}/router_spec.rb +18 -20
- data/spec/{middleware → wayfarer/middleware}/stage_spec.rb +11 -10
- data/spec/wayfarer/middleware/uri_parser_spec.rb +63 -0
- data/spec/{middleware → wayfarer/middleware}/user_agent_spec.rb +34 -32
- data/spec/wayfarer/networking/capybara_spec.rb +13 -0
- data/spec/{networking → wayfarer/networking}/context_spec.rb +46 -38
- data/spec/wayfarer/networking/ferrum_spec.rb +13 -0
- data/spec/{networking → wayfarer/networking}/follow_spec.rb +9 -4
- data/spec/wayfarer/networking/http_spec.rb +12 -0
- data/spec/{networking → wayfarer/networking}/pool_spec.rb +11 -9
- data/spec/wayfarer/networking/selenium_spec.rb +12 -0
- data/spec/{networking → wayfarer/networking}/strategy.rb +33 -54
- data/spec/{page_spec.rb → wayfarer/page_spec.rb} +3 -3
- data/spec/{parsing → wayfarer/parsing}/json_spec.rb +1 -1
- data/spec/{parsing/xml_spec.rb → wayfarer/parsing/xml_parse_spec.rb} +4 -3
- data/spec/{redis → wayfarer/redis}/barrier_spec.rb +5 -4
- data/spec/wayfarer/redis/counter_spec.rb +34 -0
- data/spec/{redis → wayfarer/redis}/pool_spec.rb +3 -2
- data/spec/{routing → wayfarer/routing}/dsl_spec.rb +12 -22
- data/spec/wayfarer/routing/hash_stack_spec.rb +63 -0
- data/spec/wayfarer/routing/integration_spec.rb +101 -0
- data/spec/wayfarer/routing/matchers/custom_spec.rb +39 -0
- data/spec/wayfarer/routing/matchers/host_spec.rb +56 -0
- data/spec/wayfarer/routing/matchers/matcher.rb +17 -0
- data/spec/wayfarer/routing/matchers/path_spec.rb +43 -0
- data/spec/wayfarer/routing/matchers/query_spec.rb +123 -0
- data/spec/wayfarer/routing/matchers/scheme_spec.rb +45 -0
- data/spec/wayfarer/routing/matchers/url_spec.rb +33 -0
- data/spec/wayfarer/routing/path_consumer_spec.rb +123 -0
- data/spec/wayfarer/routing/path_finder_spec.rb +409 -0
- data/spec/wayfarer/routing/root_route_spec.rb +51 -0
- data/spec/wayfarer/routing/route_spec.rb +74 -0
- data/spec/wayfarer/routing/sub_route_spec.rb +103 -0
- data/spec/wayfarer/uri/normalization_spec.rb +98 -0
- data/spec/wayfarer_spec.rb +2 -2
- data/wayfarer.gemspec +17 -28
- metadata +768 -246
- data/.rbenv-gemsets +0 -1
- data/.ruby-version +0 -1
- data/RELEASING.md +0 -17
- data/docs/cookbook/user_agent.md +0 -7
- data/docs/design.md +0 -36
- data/docs/guides/jobs/error_handling.md +0 -40
- data/docs/reference/configuration.md +0 -36
- data/spec/batch_completion_spec.rb +0 -104
- data/spec/cli/job_spec.rb +0 -74
- data/spec/cli/routing_spec.rb +0 -101
- data/spec/fixtures/dummy_job.rb +0 -9
- data/spec/gc_spec.rb +0 -17
- data/spec/integration/content_type_spec.rb +0 -145
- data/spec/integration/routing_spec.rb +0 -18
- data/spec/middleware/dedup_spec.rb +0 -71
- data/spec/middleware/dispatch_spec.rb +0 -59
- data/spec/middleware/normalize_spec.rb +0 -60
- data/spec/middleware/uri_parser_spec.rb +0 -53
- data/spec/networking/capybara_spec.rb +0 -12
- data/spec/networking/ferrum_spec.rb +0 -12
- data/spec/networking/http_spec.rb +0 -12
- data/spec/networking/selenium_spec.rb +0 -12
- data/spec/redis/counter_spec.rb +0 -44
- data/spec/routing/integration_spec.rb +0 -110
- data/spec/routing/matchers/custom_spec.rb +0 -31
- data/spec/routing/matchers/host_spec.rb +0 -49
- data/spec/routing/matchers/path_spec.rb +0 -43
- data/spec/routing/matchers/query_spec.rb +0 -137
- data/spec/routing/matchers/scheme_spec.rb +0 -25
- data/spec/routing/matchers/suffix_spec.rb +0 -41
- data/spec/routing/matchers/uri_spec.rb +0 -27
- data/spec/routing/path_finder_spec.rb +0 -33
- data/spec/routing/root_route_spec.rb +0 -29
- data/spec/routing/route_spec.rb +0 -43
- data/docs/{reference → guides}/cli.md +0 -0
- data/spec/{stringify_spec.rb → wayfarer/stringify_spec.rb} +2 -2
- /data/spec/{task_spec.rb → wayfarer/task_spec.rb} +0 -0
@@ -3,62 +3,69 @@
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
5
|
describe Wayfarer::Networking::Context do
|
6
|
-
|
7
|
-
let(:strategy) { spy }
|
8
|
-
let(:renewing_error) { Class.new(StandardError) }
|
9
|
-
subject(:context) { Wayfarer::Networking::Context.new(strategy) }
|
6
|
+
subject(:context) { described_class.new(strategy) }
|
10
7
|
|
11
|
-
|
12
|
-
|
13
|
-
|
8
|
+
let(:instance) { Object.new }
|
9
|
+
let(:strategy) do
|
10
|
+
double( # rubocop:disable RSpec/VerifiedDoubles
|
11
|
+
create: instance,
|
12
|
+
destroy: nil,
|
13
|
+
renew_on: [renewing_error],
|
14
|
+
fetch: nil,
|
15
|
+
live: nil
|
16
|
+
)
|
14
17
|
end
|
15
18
|
|
16
|
-
|
17
|
-
it "assigns the strategy" do
|
18
|
-
expect(context.strategy).to be(strategy)
|
19
|
-
end
|
19
|
+
let(:renewing_error) { Class.new(StandardError) }
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
end
|
21
|
+
before do
|
22
|
+
allow(strategy).to receive(:destroy)
|
24
23
|
end
|
25
24
|
|
26
25
|
describe "#fetch" do
|
27
26
|
let(:url) { Object.new }
|
28
27
|
|
29
|
-
|
30
|
-
expect(strategy).to receive(:fetch).with(instance, url)
|
28
|
+
specify do
|
31
29
|
context.fetch(url)
|
30
|
+
|
31
|
+
expect(strategy).to have_received(:fetch).with(instance, url)
|
32
32
|
end
|
33
33
|
|
34
|
-
context "with renewing exception
|
34
|
+
context "with renewing exception" do
|
35
35
|
before do
|
36
36
|
allow(strategy).to receive(:fetch).and_raise(renewing_error)
|
37
37
|
end
|
38
38
|
|
39
39
|
it "renews and reraises" do
|
40
|
-
expect(context).to receive(:renew)
|
41
|
-
|
42
40
|
expect {
|
43
41
|
context.fetch(url)
|
44
42
|
}.to raise_error(renewing_error)
|
43
|
+
|
44
|
+
expect(strategy).to have_received(:destroy).with(instance)
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
|
-
context "with
|
48
|
+
context "with renewing exception raised" do
|
49
49
|
let(:other_error) { Class.new(StandardError) }
|
50
50
|
|
51
|
-
|
51
|
+
around do |example|
|
52
|
+
original_renew_on = Wayfarer.config[:network][:renew_on]
|
52
53
|
Wayfarer.config[:network][:renew_on] = [other_error]
|
54
|
+
example.run
|
55
|
+
Wayfarer.config[:network][:renew_on] = original_renew_on
|
56
|
+
end
|
57
|
+
|
58
|
+
before do
|
59
|
+
allow(strategy).to receive(:renew_on).and_return([renewing_error, other_error])
|
53
60
|
allow(strategy).to receive(:fetch).and_raise(other_error)
|
54
61
|
end
|
55
62
|
|
56
63
|
it "renews and reraises" do
|
57
|
-
expect(context).to receive(:renew)
|
58
|
-
|
59
64
|
expect {
|
60
65
|
context.fetch(url)
|
61
66
|
}.to raise_error(other_error)
|
67
|
+
|
68
|
+
expect(strategy).to have_received(:destroy).with(instance)
|
62
69
|
end
|
63
70
|
end
|
64
71
|
|
@@ -67,12 +74,12 @@ describe Wayfarer::Networking::Context do
|
|
67
74
|
allow(strategy).to receive(:fetch).and_raise(StandardError)
|
68
75
|
end
|
69
76
|
|
70
|
-
it "reraises" do
|
71
|
-
expect(context).not_to receive(:renew)
|
72
|
-
|
77
|
+
it "reraises without renewing" do
|
73
78
|
expect {
|
74
79
|
context.fetch(url)
|
75
80
|
}.to raise_error(StandardError)
|
81
|
+
|
82
|
+
expect(strategy).not_to have_received(:destroy)
|
76
83
|
end
|
77
84
|
end
|
78
85
|
end
|
@@ -81,28 +88,29 @@ describe Wayfarer::Networking::Context do
|
|
81
88
|
let(:page) { Object.new }
|
82
89
|
|
83
90
|
it "calls live" do
|
84
|
-
expect(strategy).to receive(:live).with(instance)
|
85
91
|
context.live
|
92
|
+
|
93
|
+
expect(strategy).to have_received(:live).with(instance)
|
86
94
|
end
|
87
95
|
|
88
|
-
context "with renewing exception
|
96
|
+
context "with renewing exception" do
|
89
97
|
before do
|
90
98
|
allow(strategy).to receive(:live).and_raise(renewing_error)
|
91
99
|
end
|
92
100
|
|
93
101
|
it "renews and reraises" do
|
94
|
-
expect(context).to receive(:renew)
|
95
|
-
|
96
102
|
expect {
|
97
103
|
context.live
|
98
104
|
}.to raise_error(renewing_error)
|
105
|
+
|
106
|
+
expect(strategy).to have_received(:destroy).with(instance)
|
99
107
|
end
|
100
108
|
|
101
109
|
context "when renewing raises" do
|
102
110
|
let(:other_error) { Class.new(StandardError) }
|
103
111
|
|
104
112
|
before do
|
105
|
-
allow(
|
113
|
+
allow(strategy).to receive(:destroy).and_raise(other_error)
|
106
114
|
end
|
107
115
|
|
108
116
|
it "reraises the original exception" do
|
@@ -118,27 +126,27 @@ describe Wayfarer::Networking::Context do
|
|
118
126
|
allow(strategy).to receive(:live).and_raise(StandardError)
|
119
127
|
end
|
120
128
|
|
121
|
-
it "reraises" do
|
122
|
-
expect(context).not_to receive(:renew)
|
123
|
-
|
129
|
+
it "reraises without renewing" do
|
124
130
|
expect {
|
125
131
|
context.live(page)
|
126
132
|
}.to raise_error(StandardError)
|
133
|
+
|
134
|
+
expect(strategy).not_to have_received(:destroy)
|
127
135
|
end
|
128
136
|
end
|
129
137
|
end
|
130
138
|
|
131
139
|
describe "#renew" do
|
132
|
-
|
133
|
-
expect(strategy).to receive(:destroy).with(instance)
|
140
|
+
specify do
|
134
141
|
context.renew
|
142
|
+
expect(strategy).to have_received(:destroy).with(instance)
|
135
143
|
end
|
136
144
|
end
|
137
145
|
|
138
146
|
describe "#instance" do
|
139
|
-
|
140
|
-
expect(strategy).to receive(:create).and_return(instance)
|
147
|
+
specify do
|
141
148
|
expect(context.instance).to be(instance)
|
149
|
+
expect(strategy).to have_received(:create)
|
142
150
|
end
|
143
151
|
end
|
144
152
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
require_relative "strategy"
|
5
|
+
|
6
|
+
describe Wayfarer::Networking::Ferrum, :ferrum do
|
7
|
+
it_behaves_like "Network strategy", strategy: described_class,
|
8
|
+
browser: true,
|
9
|
+
request_headers: true,
|
10
|
+
response_headers: true,
|
11
|
+
status_code: true,
|
12
|
+
raises_on_error_response: true
|
13
|
+
end
|
@@ -3,13 +3,14 @@
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
5
|
describe Wayfarer::Networking::Follow do
|
6
|
+
subject(:outer) { described_class.new(inner) }
|
7
|
+
|
6
8
|
let(:page) { build(:page) }
|
7
9
|
let(:url) { test_app_path("/foo") }
|
8
10
|
let(:redirect_url) { test_app_path("/redirect") }
|
9
11
|
let(:success) { Wayfarer::Networking::Result::Success.new(page) }
|
10
12
|
let(:redirect) { Wayfarer::Networking::Result::Redirect.new(redirect_url) }
|
11
13
|
let(:inner) { spy }
|
12
|
-
subject(:outer) { described_class.new(inner) }
|
13
14
|
|
14
15
|
describe "#fetch" do
|
15
16
|
context "when context returns Success" do
|
@@ -21,10 +22,14 @@ describe Wayfarer::Networking::Follow do
|
|
21
22
|
end
|
22
23
|
|
23
24
|
context "when context returns Redirect" do
|
24
|
-
it "follows
|
25
|
-
|
26
|
-
|
25
|
+
it "follows redirect URL" do
|
26
|
+
allow(inner).to receive(:fetch).with(url).and_return(redirect)
|
27
|
+
allow(inner).to receive(:fetch).with(redirect_url).and_return(success)
|
28
|
+
|
27
29
|
outer.fetch(url, follow: 2)
|
30
|
+
|
31
|
+
expect(inner).to have_received(:fetch).with(url).ordered
|
32
|
+
expect(inner).to have_received(:fetch).with(redirect_url).ordered
|
28
33
|
end
|
29
34
|
end
|
30
35
|
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
require_relative "strategy"
|
5
|
+
|
6
|
+
describe Wayfarer::Networking::HTTP do
|
7
|
+
it_behaves_like "Network strategy", strategy: described_class,
|
8
|
+
browser: false,
|
9
|
+
request_headers: true,
|
10
|
+
response_headers: true,
|
11
|
+
status_code: true
|
12
|
+
end
|
@@ -3,12 +3,12 @@
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
5
|
describe Wayfarer::Networking::Pool do
|
6
|
-
subject(:pool) {
|
6
|
+
subject(:pool) { described_class.send(:new) }
|
7
7
|
|
8
8
|
after { pool.free }
|
9
9
|
|
10
10
|
describe "#with" do
|
11
|
-
context "
|
11
|
+
context "with default" do
|
12
12
|
it "yields HTTP" do
|
13
13
|
pool.with do |context|
|
14
14
|
expect(context.strategy).to be_a(Wayfarer::Networking::HTTP)
|
@@ -16,7 +16,7 @@ describe Wayfarer::Networking::Pool do
|
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
|
-
context "when using Ferrum", ferrum
|
19
|
+
context "when using Ferrum", :ferrum do
|
20
20
|
before { Wayfarer.config[:network][:agent] = :ferrum }
|
21
21
|
|
22
22
|
it "yields Ferrum" do
|
@@ -26,7 +26,7 @@ describe Wayfarer::Networking::Pool do
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
-
context "when using Selenium", selenium
|
29
|
+
context "when using Selenium", :selenium do
|
30
30
|
before { Wayfarer.config[:network][:agent] = :selenium }
|
31
31
|
|
32
32
|
it "yields Selenium" do
|
@@ -36,7 +36,7 @@ describe Wayfarer::Networking::Pool do
|
|
36
36
|
end
|
37
37
|
end
|
38
38
|
|
39
|
-
context "when using Capybara", ferrum
|
39
|
+
context "when using Capybara", :ferrum do
|
40
40
|
before do
|
41
41
|
Wayfarer.config[:network][:agent] = :capybara
|
42
42
|
Wayfarer.config[:capybara][:driver] = :cuprite
|
@@ -51,17 +51,19 @@ describe Wayfarer::Networking::Pool do
|
|
51
51
|
end
|
52
52
|
|
53
53
|
describe "#free" do
|
54
|
-
let(:strategy) { double(create:
|
54
|
+
let(:strategy) { double(create: nil, destroy: nil) } # rubocop:disable RSpec/VerifiedDoubles
|
55
55
|
|
56
56
|
before do
|
57
|
-
pool.class.registry[:foobar] = double(new: strategy)
|
57
|
+
pool.class.registry[:foobar] = double(new: strategy) # rubocop:disable RSpec/VerifiedDoubles
|
58
|
+
pool.class.finalizer = ->(pool) { pool.reload(&:renew) }
|
59
|
+
|
58
60
|
Wayfarer.config[:network][:agent] = :foobar
|
59
61
|
end
|
60
62
|
|
61
63
|
it "destroys the strategy" do
|
62
|
-
|
63
|
-
pool.with {}
|
64
|
+
pool.with {} # rubocop:disable Lint/EmptyBlock
|
64
65
|
pool.free
|
66
|
+
expect(strategy).to have_received(:destroy)
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
require_relative "strategy"
|
5
|
+
|
6
|
+
describe Wayfarer::Networking::Selenium, :selenium do
|
7
|
+
it_behaves_like "Network strategy", strategy: described_class,
|
8
|
+
browser: true,
|
9
|
+
request_headers: false,
|
10
|
+
response_headers: false,
|
11
|
+
status_code: false
|
12
|
+
end
|
@@ -5,13 +5,15 @@ require "spec_helpers"
|
|
5
5
|
RSpec.shared_examples "Network strategy" do |options|
|
6
6
|
attr_accessor :context
|
7
7
|
|
8
|
-
|
8
|
+
# rubocop:disable RSpec/BeforeAfterAll
|
9
|
+
before :all do
|
9
10
|
self.context = Wayfarer::Networking::Context.new(options[:strategy].new)
|
10
11
|
end
|
11
12
|
|
12
|
-
after
|
13
|
+
after :all do
|
13
14
|
context.renew
|
14
15
|
end
|
16
|
+
# rubocop:enable RSpec/BeforeAfterAll
|
15
17
|
|
16
18
|
describe "#renew_on" do
|
17
19
|
it "returns an Array" do
|
@@ -21,17 +23,21 @@ RSpec.shared_examples "Network strategy" do |options|
|
|
21
23
|
|
22
24
|
describe "Context" do
|
23
25
|
describe "#fetch" do
|
24
|
-
it "returns Success" do
|
25
|
-
url = test_app_path("/
|
26
|
+
it "returns Success", :aggregate_failures do
|
27
|
+
url = test_app_path("/status/200")
|
26
28
|
result = context.fetch(url)
|
27
29
|
expect(result).to be_a(Wayfarer::Networking::Result::Success)
|
28
30
|
expect(result.page).to be_a(Wayfarer::Page)
|
29
31
|
end
|
30
32
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
33
|
+
context "with 4xx response code" do
|
34
|
+
skip if options[:raises_on_error_response]
|
35
|
+
|
36
|
+
it "sets the URL" do
|
37
|
+
url = test_app_path("hello_world")
|
38
|
+
page = context.fetch(url).page
|
39
|
+
expect(page.url).to eq test_app_path("hello_world")
|
40
|
+
end
|
35
41
|
end
|
36
42
|
|
37
43
|
it "sets the response body" do
|
@@ -40,23 +46,26 @@ RSpec.shared_examples "Network strategy" do |options|
|
|
40
46
|
expect(page.body).to match(/Hello world!/)
|
41
47
|
end
|
42
48
|
|
43
|
-
|
44
|
-
|
49
|
+
context "with error responses" do
|
50
|
+
it "sets the status code" do
|
51
|
+
skip unless options[:status_code]
|
52
|
+
skip if options[:raises_on_error_response]
|
45
53
|
|
46
|
-
|
47
|
-
|
48
|
-
|
54
|
+
url = test_app_path("status_code/404")
|
55
|
+
page = context.fetch(url).page
|
56
|
+
expect(page.status_code).to be(404)
|
57
|
+
end
|
49
58
|
end
|
50
59
|
|
51
60
|
it "sets response headers" do
|
52
61
|
skip unless options[:response_headers]
|
53
62
|
|
54
|
-
url = test_app_path("
|
63
|
+
url = test_app_path("hello_world")
|
55
64
|
page = context.fetch(url).page
|
56
65
|
expect(page.headers["hello"]).to eq("world")
|
57
66
|
end
|
58
67
|
|
59
|
-
context "Selenium" do
|
68
|
+
context "with Selenium" do
|
60
69
|
before { skip unless context.strategy.is_a?(Wayfarer::Networking::Selenium) }
|
61
70
|
|
62
71
|
it "sets a mock status code" do
|
@@ -74,21 +83,6 @@ RSpec.shared_examples "Network strategy" do |options|
|
|
74
83
|
end
|
75
84
|
end
|
76
85
|
|
77
|
-
describe "HTTP headers" do
|
78
|
-
before do
|
79
|
-
Wayfarer.config[:network][:http_headers] = { "User-Agent" => "Foobar" }
|
80
|
-
context.renew
|
81
|
-
end
|
82
|
-
|
83
|
-
it "uses configured HTTP headers" do
|
84
|
-
skip unless options[:request_headers]
|
85
|
-
|
86
|
-
url = test_app_path("/headers/HTTP_USER_AGENT")
|
87
|
-
page = context.fetch(url).page
|
88
|
-
expect(page.body).to match(/Foobar/)
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
86
|
context "when encountering a redirect" do
|
93
87
|
it "returns a Redirect" do
|
94
88
|
skip if options[:browser]
|
@@ -103,29 +97,29 @@ RSpec.shared_examples "Network strategy" do |options|
|
|
103
97
|
|
104
98
|
describe "#live" do
|
105
99
|
context "when automating a browser" do
|
106
|
-
|
100
|
+
subject(:page) { context.live.page }
|
101
|
+
|
102
|
+
let(:url) { test_app_path("hello_world") }
|
107
103
|
|
108
104
|
before do
|
109
105
|
skip unless options[:browser]
|
110
106
|
|
111
|
-
context.fetch(
|
107
|
+
context.fetch(url)
|
112
108
|
context.strategy.navigate(context.instance, url)
|
113
109
|
end
|
114
110
|
|
115
|
-
subject(:page) { context.live.page }
|
116
|
-
|
117
111
|
it "sets the URL" do
|
118
112
|
expect(page.url).to eq(url)
|
119
113
|
end
|
120
114
|
|
121
115
|
it "sets the response body" do
|
122
|
-
expect(page.body).to match(/
|
116
|
+
expect(page.body).to match(/Hello world/)
|
123
117
|
end
|
124
118
|
|
125
119
|
it "sets response headers" do
|
126
120
|
skip unless options[:response_headers]
|
127
121
|
|
128
|
-
expect(page.headers["content-length"]).to eq("
|
122
|
+
expect(page.headers["content-length"]).to eq("12")
|
129
123
|
end
|
130
124
|
|
131
125
|
it "sets the status code" do
|
@@ -139,31 +133,16 @@ RSpec.shared_examples "Network strategy" do |options|
|
|
139
133
|
it "returns nil" do
|
140
134
|
skip if options[:browser]
|
141
135
|
|
142
|
-
expect(context.live).to
|
136
|
+
expect(context.live).to be_nil
|
143
137
|
end
|
144
138
|
end
|
145
139
|
end
|
146
140
|
|
147
141
|
describe "#renew" do
|
148
|
-
it "replaces the instance"
|
142
|
+
it "replaces the instance" do
|
149
143
|
expect {
|
150
144
|
context.renew
|
151
|
-
}.to(change
|
152
|
-
end
|
153
|
-
end
|
154
|
-
end
|
155
|
-
|
156
|
-
describe "Selenium" do
|
157
|
-
before { skip unless context.strategy.is_a?(Wayfarer::Networking::Selenium) }
|
158
|
-
|
159
|
-
describe "Client timeout" do
|
160
|
-
before do
|
161
|
-
Wayfarer.config[:selenium][:client_timeout] = 42
|
162
|
-
context.renew
|
163
|
-
end
|
164
|
-
|
165
|
-
it "sets the client timeout" do
|
166
|
-
expect(context.instance.send(:bridge).send(:http).read_timeout).to be(42)
|
145
|
+
}.to(change(context, :instance))
|
167
146
|
end
|
168
147
|
end
|
169
148
|
end
|
@@ -4,10 +4,10 @@ require "spec_helpers"
|
|
4
4
|
|
5
5
|
describe Wayfarer::Page do
|
6
6
|
describe "#mime_type" do
|
7
|
-
let(:headers) { { "Content-Type" => "text/html; charset=UTF-8" } }
|
8
|
-
|
9
7
|
subject { build(:page, headers: headers).mime_type }
|
10
8
|
|
9
|
+
let(:headers) { { "Content-Type" => "text/html; charset=UTF-8" } }
|
10
|
+
|
11
11
|
it { is_expected.to eq("text/html") }
|
12
12
|
end
|
13
13
|
|
@@ -15,7 +15,7 @@ describe Wayfarer::Page do
|
|
15
15
|
context "when Content-Type is absent" do
|
16
16
|
subject { build(:page, :html, headers: {}).doc }
|
17
17
|
|
18
|
-
it { is_expected.to
|
18
|
+
it { is_expected.to be_nil }
|
19
19
|
end
|
20
20
|
|
21
21
|
context "when Content-Type is HTML" do
|
@@ -3,10 +3,11 @@
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
5
|
describe Wayfarer::Parsing::XML, ".parse" do
|
6
|
+
subject(:doc) { described_class.parse(xml, variant) }
|
7
|
+
|
6
8
|
let(:xml) { "<span>Foobar</span>" }
|
7
|
-
subject(:doc) { Wayfarer::Parsing::XML.parse(xml, variant) }
|
8
9
|
|
9
|
-
context "HTML" do
|
10
|
+
context "with HTML" do
|
10
11
|
let(:variant) { :html }
|
11
12
|
|
12
13
|
specify do
|
@@ -14,7 +15,7 @@ describe Wayfarer::Parsing::XML, ".parse" do
|
|
14
15
|
end
|
15
16
|
end
|
16
17
|
|
17
|
-
context "XML" do
|
18
|
+
context "with XML" do
|
18
19
|
let(:variant) { :xml }
|
19
20
|
|
20
21
|
specify do
|
@@ -2,9 +2,10 @@
|
|
2
2
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
|
-
describe Wayfarer::Redis::Barrier, redis
|
5
|
+
describe Wayfarer::Redis::Barrier, :redis do
|
6
|
+
subject(:barrier) { build(:barrier, task: task) }
|
7
|
+
|
6
8
|
let(:task) { build(:task, :redis_pool) }
|
7
|
-
subject(:barrier) { Wayfarer::Redis::Barrier.new(task) }
|
8
9
|
|
9
10
|
describe "#redis_key" do
|
10
11
|
it "returns the expected Redis key" do
|
@@ -24,13 +25,13 @@ describe Wayfarer::Redis::Barrier, redis: true do
|
|
24
25
|
context "with seen URL" do
|
25
26
|
before { barrier.check!(task.url) }
|
26
27
|
|
27
|
-
|
28
|
+
specify do
|
28
29
|
expect(barrier.check!(task.url)).to be(true)
|
29
30
|
end
|
30
31
|
end
|
31
32
|
|
32
33
|
context "with unseen URL" do
|
33
|
-
|
34
|
+
specify do
|
34
35
|
expect(barrier.check!(task.url)).to be(false)
|
35
36
|
end
|
36
37
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::Redis::Counter, :redis do
|
6
|
+
subject(:counter) { build(:counter, task: task) }
|
7
|
+
|
8
|
+
let(:task) { build(:task, :redis_pool) }
|
9
|
+
|
10
|
+
describe "#redis_key" do
|
11
|
+
specify do
|
12
|
+
expect(counter.redis_key).to eq("wayfarer-counter-batch")
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
describe "#increment" do
|
17
|
+
specify do
|
18
|
+
expect { counter.increment }.to change(counter, :value).by(1)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe "#decrement" do
|
23
|
+
specify do
|
24
|
+
expect { counter.decrement }.to change(counter, :value).by(-1)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
describe "#reset!" do
|
29
|
+
it "resets" do
|
30
|
+
3.times { counter.increment }
|
31
|
+
expect { counter.reset! }.to change(counter, :value).to(0)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -2,9 +2,10 @@
|
|
2
2
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
|
-
describe Wayfarer::Redis::Pool, redis
|
5
|
+
describe Wayfarer::Redis::Pool, :redis do
|
6
|
+
subject(:pool) { described_class.send(:new) }
|
7
|
+
|
6
8
|
let(:conn) { spy }
|
7
|
-
subject(:pool) { Wayfarer::Redis::Pool.send(:new) }
|
8
9
|
|
9
10
|
describe "::with" do
|
10
11
|
before do
|