wayfarer 0.4.6 → 0.4.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/lint.yaml +25 -0
- data/.github/workflows/release.yaml +29 -0
- data/.github/workflows/tests.yaml +30 -0
- data/.gitignore +4 -0
- data/.rubocop.yml +5 -0
- data/.vale.ini +5 -0
- data/.yardopts +1 -3
- data/Dockerfile +5 -4
- data/Gemfile +3 -0
- data/Gemfile.lock +107 -102
- data/Rakefile +5 -56
- data/bin/wayfarer +1 -1
- data/docker-compose.yml +20 -9
- data/docs/cookbook/consent_screen.md +2 -2
- data/docs/cookbook/executing_javascript.md +3 -3
- data/docs/cookbook/navigation.md +12 -12
- data/docs/cookbook/querying_html.md +3 -3
- data/docs/cookbook/screenshots.md +2 -2
- data/docs/cookbook/user_agent.md +1 -1
- data/docs/design.md +36 -0
- data/docs/guides/callbacks.md +24 -126
- data/docs/guides/configuration.md +8 -8
- data/docs/guides/handlers.md +60 -0
- data/docs/guides/index.md +1 -0
- data/docs/guides/jobs/error_handling.md +40 -0
- data/docs/guides/jobs.md +99 -31
- data/docs/guides/navigation.md +1 -1
- data/docs/guides/networking/capybara.md +13 -22
- data/docs/guides/networking/custom_adapters.md +82 -41
- data/docs/guides/networking/ferrum.md +4 -4
- data/docs/guides/networking/http.md +9 -13
- data/docs/guides/networking/selenium.md +10 -11
- data/docs/guides/pages.md +76 -10
- data/docs/guides/redis.md +10 -0
- data/docs/guides/routing.md +74 -0
- data/docs/guides/tasks.md +33 -9
- data/docs/guides/tutorial.md +60 -0
- data/docs/guides/user_agents.md +113 -0
- data/docs/index.md +17 -40
- data/docs/reference/cli.md +35 -25
- data/docs/reference/configuration.md +36 -0
- data/lib/wayfarer/base.rb +124 -46
- data/lib/wayfarer/batch_completion.rb +56 -0
- data/lib/wayfarer/callbacks.rb +22 -48
- data/lib/wayfarer/cli/route_printer.rb +71 -57
- data/lib/wayfarer/cli.rb +121 -0
- data/lib/wayfarer/gc.rb +13 -6
- data/lib/wayfarer/handler.rb +15 -7
- data/lib/wayfarer/logging.rb +38 -0
- data/lib/wayfarer/middleware/base.rb +2 -0
- data/lib/wayfarer/middleware/batch_completion.rb +19 -0
- data/lib/wayfarer/middleware/content_type.rb +54 -0
- data/lib/wayfarer/middleware/controller.rb +19 -15
- data/lib/wayfarer/middleware/dedup.rb +16 -13
- data/lib/wayfarer/middleware/dispatch.rb +12 -4
- data/lib/wayfarer/middleware/normalize.rb +12 -11
- data/lib/wayfarer/middleware/redis.rb +15 -0
- data/lib/wayfarer/middleware/router.rb +33 -35
- data/lib/wayfarer/middleware/stage.rb +5 -5
- data/lib/wayfarer/middleware/uri_parser.rb +30 -0
- data/lib/wayfarer/middleware/user_agent.rb +49 -0
- data/lib/wayfarer/networking/capybara.rb +1 -1
- data/lib/wayfarer/networking/context.rb +2 -2
- data/lib/wayfarer/networking/ferrum.rb +2 -2
- data/lib/wayfarer/networking/follow.rb +12 -6
- data/lib/wayfarer/networking/http.rb +1 -1
- data/lib/wayfarer/networking/pool.rb +17 -12
- data/lib/wayfarer/networking/selenium.rb +3 -3
- data/lib/wayfarer/networking/strategy.rb +2 -2
- data/lib/wayfarer/page.rb +36 -14
- data/lib/wayfarer/parsing/xml.rb +6 -6
- data/lib/wayfarer/parsing.rb +24 -0
- data/lib/wayfarer/redis/barrier.rb +13 -21
- data/lib/wayfarer/redis/counter.rb +19 -9
- data/lib/wayfarer/redis/pool.rb +1 -1
- data/lib/wayfarer/redis/resettable.rb +19 -0
- data/lib/wayfarer/routing/dsl.rb +1 -0
- data/lib/wayfarer/routing/matchers/path.rb +4 -2
- data/lib/wayfarer/routing/root_route.rb +5 -1
- data/lib/wayfarer/routing/route.rb +4 -14
- data/lib/wayfarer/stringify.rb +22 -30
- data/lib/wayfarer/task.rb +12 -18
- data/lib/wayfarer.rb +28 -1
- data/mkdocs.yml +52 -7
- data/rake/docs.rake +26 -0
- data/rake/lint.rake +105 -0
- data/rake/release.rake +29 -0
- data/rake/tests.rake +28 -0
- data/requirements.txt +1 -1
- data/spec/base_spec.rb +140 -160
- data/spec/batch_completion_spec.rb +104 -0
- data/spec/cli/job_spec.rb +19 -23
- data/spec/cli/routing_spec.rb +101 -0
- data/spec/cli/version_spec.rb +1 -1
- data/spec/factories/task.rb +7 -1
- data/spec/fixtures/dummy_job.rb +5 -3
- data/spec/gc_spec.rb +8 -50
- data/spec/handler_spec.rb +1 -1
- data/spec/integration/callbacks_spec.rb +157 -45
- data/spec/integration/content_type_spec.rb +145 -0
- data/spec/integration/gc_spec.rb +44 -0
- data/spec/integration/handler_spec.rb +66 -0
- data/spec/integration/page_spec.rb +44 -29
- data/spec/integration/params_spec.rb +33 -25
- data/spec/integration/parsing_spec.rb +125 -0
- data/spec/integration/routing_spec.rb +18 -0
- data/spec/integration/stage_spec.rb +27 -20
- data/spec/middleware/batch_completion_spec.rb +34 -0
- data/spec/middleware/chain_spec.rb +8 -8
- data/spec/middleware/content_type_spec.rb +86 -0
- data/spec/middleware/controller_spec.rb +5 -5
- data/spec/middleware/dedup_spec.rb +38 -55
- data/spec/middleware/dispatch_spec.rb +23 -7
- data/spec/middleware/normalize_spec.rb +44 -13
- data/spec/middleware/router_spec.rb +29 -30
- data/spec/middleware/stage_spec.rb +8 -8
- data/spec/middleware/uri_parser_spec.rb +53 -0
- data/spec/middleware/{fetch_spec.rb → user_agent_spec.rb} +28 -27
- data/spec/networking/context_spec.rb +1 -1
- data/spec/networking/follow_spec.rb +2 -2
- data/spec/networking/pool_spec.rb +5 -5
- data/spec/networking/strategy.rb +2 -2
- data/spec/page_spec.rb +42 -20
- data/spec/parsing/xml_spec.rb +11 -12
- data/spec/redis/barrier_spec.rb +8 -48
- data/spec/redis/counter_spec.rb +13 -1
- data/spec/redis/pool_spec.rb +1 -1
- data/spec/spec_helpers.rb +27 -16
- data/spec/support/test_app.rb +8 -0
- data/spec/task_spec.rb +3 -24
- data/spec/wayfarer_spec.rb +1 -1
- data/wayfarer.gemspec +4 -3
- metadata +61 -51
- data/.github/workflows/ci.yaml +0 -32
- data/docs/guides/error_handling.md +0 -53
- data/docs/guides/networking.md +0 -94
- data/docs/guides/performance.md +0 -130
- data/docs/guides/reliability.md +0 -41
- data/docs/guides/routing/steering.md +0 -30
- data/docs/reference/api/base.md +0 -48
- data/docs/reference/configuration_keys.md +0 -43
- data/docs/reference/environment_variables.md +0 -83
- data/lib/wayfarer/cli/base.rb +0 -45
- data/lib/wayfarer/cli/generate.rb +0 -17
- data/lib/wayfarer/cli/job.rb +0 -56
- data/lib/wayfarer/cli/route.rb +0 -29
- data/lib/wayfarer/cli/runner.rb +0 -34
- data/lib/wayfarer/cli/templates/Gemfile.tt +0 -5
- data/lib/wayfarer/cli/templates/job.rb.tt +0 -10
- data/lib/wayfarer/config/capybara.rb +0 -10
- data/lib/wayfarer/config/ferrum.rb +0 -11
- data/lib/wayfarer/config/networking.rb +0 -29
- data/lib/wayfarer/config/redis.rb +0 -14
- data/lib/wayfarer/config/root.rb +0 -11
- data/lib/wayfarer/config/selenium.rb +0 -21
- data/lib/wayfarer/config/strconv.rb +0 -45
- data/lib/wayfarer/config/struct.rb +0 -72
- data/lib/wayfarer/middleware/fetch.rb +0 -56
- data/lib/wayfarer/redis/connection.rb +0 -13
- data/lib/wayfarer/redis/version.rb +0 -19
- data/lib/wayfarer/routing/router.rb +0 -28
- data/spec/callbacks_spec.rb +0 -102
- data/spec/cli/generate_spec.rb +0 -39
- data/spec/config/capybara_spec.rb +0 -18
- data/spec/config/ferrum_spec.rb +0 -24
- data/spec/config/networking_spec.rb +0 -73
- data/spec/config/redis_spec.rb +0 -32
- data/spec/config/root_spec.rb +0 -31
- data/spec/config/selenium_spec.rb +0 -56
- data/spec/config/strconv_spec.rb +0 -58
- data/spec/config/struct_spec.rb +0 -66
- data/spec/integration/steering_spec.rb +0 -57
- data/spec/redis/version_spec.rb +0 -13
- data/spec/routing/router_spec.rb +0 -24
data/spec/base_spec.rb
CHANGED
@@ -3,222 +3,202 @@
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
5
|
describe Wayfarer::Base, redis: true do
|
6
|
-
|
6
|
+
let(:task) { build(:task, :redis_pool) }
|
7
7
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
let(:klass) { Class.new(Wayfarer::Base) }
|
12
|
-
|
13
|
-
before { stub_const("DummyJob", klass) }
|
8
|
+
before do
|
9
|
+
stub_const("DummyJob", Class.new(ActiveJob::Base).include(Wayfarer::Base))
|
10
|
+
end
|
14
11
|
|
15
12
|
describe "::crawl" do
|
16
13
|
it "enqueues a task" do
|
17
14
|
expect(DummyJob).to receive(:perform_later).with(task)
|
18
|
-
DummyJob.crawl(url, batch: batch)
|
15
|
+
DummyJob.crawl(task.url, batch: task.batch)
|
19
16
|
end
|
20
17
|
|
21
18
|
it "returns a task" do
|
22
|
-
expect(DummyJob.crawl(url)).to be_a(Wayfarer::Task)
|
19
|
+
expect(DummyJob.crawl(task.url)).to be_a(Wayfarer::Task)
|
23
20
|
end
|
24
21
|
end
|
25
22
|
|
26
|
-
describe "
|
27
|
-
|
23
|
+
describe "#perform" do
|
24
|
+
subject(:counter) { Wayfarer::Redis::Counter.new(task) }
|
25
|
+
let!(:initial) { counter.increment }
|
28
26
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
DummyJob.crawl(url, batch: batch)
|
33
|
-
}.to change { counter.value }.by(1)
|
27
|
+
before do
|
28
|
+
DummyJob.class_eval do
|
29
|
+
route.to :index
|
34
30
|
end
|
35
31
|
end
|
36
32
|
|
37
|
-
describe "
|
38
|
-
|
39
|
-
DummyJob.
|
40
|
-
|
41
|
-
|
33
|
+
describe "succeeding job" do
|
34
|
+
before do
|
35
|
+
DummyJob.class_eval do
|
36
|
+
def index; end
|
37
|
+
end
|
42
38
|
end
|
43
39
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
perform_enqueued_jobs
|
48
|
-
redis do |conn|
|
49
|
-
expect(conn.exists?(task.barrier.redis_key)).to be(false)
|
50
|
-
end
|
51
|
-
end
|
40
|
+
specify do
|
41
|
+
expect_any_instance_of(Wayfarer::Redis::Counter)
|
42
|
+
.to receive(:increment).at_least(:once).and_call_original
|
52
43
|
|
53
|
-
|
54
|
-
DummyJob.
|
44
|
+
expect {
|
45
|
+
DummyJob.perform_later(task)
|
55
46
|
perform_enqueued_jobs
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
end
|
60
|
-
|
61
|
-
it "runs after batch callbacks" do
|
62
|
-
expect { |spy|
|
63
|
-
klass.after_batch(&spy)
|
64
|
-
DummyJob.crawl(url, batch: batch)
|
65
|
-
perform_enqueued_jobs
|
66
|
-
}.to yield_control
|
67
|
-
end
|
47
|
+
assert_performed_jobs 1
|
48
|
+
expect(enqueued_jobs).to be_empty
|
49
|
+
}.not_to change { counter.value }.from(initial)
|
68
50
|
end
|
69
51
|
end
|
70
|
-
end
|
71
|
-
|
72
|
-
describe "Unhandled exceptions" do
|
73
|
-
let(:klass) { Class.new(Wayfarer::Base) }
|
74
|
-
|
75
|
-
before do
|
76
|
-
allow_any_instance_of(DummyJob).to receive(:perform).and_raise(RuntimeError.new)
|
77
|
-
end
|
78
52
|
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
rescue StandardError
|
86
|
-
nil
|
53
|
+
describe "failing job" do
|
54
|
+
before do
|
55
|
+
DummyJob.class_eval do
|
56
|
+
def index
|
57
|
+
raise
|
58
|
+
end
|
87
59
|
end
|
88
|
-
|
89
|
-
end
|
60
|
+
end
|
90
61
|
|
91
|
-
|
92
|
-
|
62
|
+
specify do
|
63
|
+
expect_any_instance_of(Wayfarer::Redis::Counter)
|
64
|
+
.to receive(:increment).at_least(:once).and_call_original
|
93
65
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
66
|
+
expect {
|
67
|
+
DummyJob.perform_later(task)
|
68
|
+
expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
|
69
|
+
assert_performed_jobs 1
|
70
|
+
expect(enqueued_jobs).to be_empty
|
71
|
+
}.not_to change { counter.value }.from(initial)
|
99
72
|
end
|
100
|
-
|
101
|
-
expect(task.counter.value).to be(3)
|
102
73
|
end
|
103
|
-
end
|
104
74
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
75
|
+
describe "failing job with retries" do
|
76
|
+
before do
|
77
|
+
DummyJob.class_eval do
|
78
|
+
retry_on StandardError, attempts: 3
|
79
|
+
|
80
|
+
def index
|
81
|
+
raise
|
82
|
+
end
|
110
83
|
end
|
111
84
|
end
|
112
|
-
end
|
113
85
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
task.metadata.job = job
|
86
|
+
specify do
|
87
|
+
expect_any_instance_of(Wayfarer::Redis::Counter)
|
88
|
+
.to receive(:increment).at_least(:once).and_call_original
|
118
89
|
|
119
|
-
|
90
|
+
expect {
|
91
|
+
DummyJob.perform_later(task)
|
92
|
+
2.times { perform_enqueued_jobs }
|
93
|
+
expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
|
94
|
+
assert_performed_jobs 3
|
95
|
+
expect(enqueued_jobs).to be_empty
|
96
|
+
}.not_to change { counter.value }.from(initial)
|
120
97
|
end
|
121
|
-
|
122
|
-
stub_const("Spy", spy)
|
123
98
|
end
|
124
99
|
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
100
|
+
describe "discarded after retries" do
|
101
|
+
before do
|
102
|
+
DummyJob.class_eval do
|
103
|
+
ErrorA = Class.new(StandardError)
|
104
|
+
ErrorB = Class.new(StandardError)
|
129
105
|
|
130
|
-
|
106
|
+
retry_on ErrorA, attempts: 5
|
131
107
|
|
132
|
-
|
133
|
-
perform_enqueued_jobs
|
134
|
-
}.to change { enqueued_jobs.last["executions"] }.by(1)
|
108
|
+
discard_on ErrorB
|
135
109
|
|
136
|
-
|
137
|
-
|
138
|
-
}.to change { enqueued_jobs.last["executions"] }.by(1)
|
110
|
+
def index
|
111
|
+
raise ErrorB if executions == 5
|
139
112
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
it "marks the URL seen" do
|
146
|
-
task.counter.increment # otherwise barrier gets reset
|
147
|
-
DummyJob.crawl(url, batch: batch)
|
148
|
-
3.times { perform_enqueued_jobs }
|
149
|
-
expect(task.barrier.seen?(task.url)).to be(true)
|
150
|
-
end
|
151
|
-
|
152
|
-
it "decrements the counter" do
|
153
|
-
3.times { task.counter.increment }
|
113
|
+
raise ErrorA
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
154
117
|
|
155
|
-
|
156
|
-
|
118
|
+
specify do
|
119
|
+
expect_any_instance_of(Wayfarer::Redis::Counter)
|
120
|
+
.to receive(:increment).at_least(:once).and_call_original
|
157
121
|
|
158
|
-
|
122
|
+
expect {
|
123
|
+
DummyJob.perform_later(task)
|
124
|
+
4.times { perform_enqueued_jobs }
|
125
|
+
perform_enqueued_jobs
|
126
|
+
assert_performed_jobs 5
|
127
|
+
expect(enqueued_jobs).to be_empty
|
128
|
+
}.not_to change { counter.value }.from(initial)
|
129
|
+
end
|
159
130
|
end
|
160
131
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
3.times { perform_enqueued_jobs }
|
166
|
-
}.to yield_control
|
167
|
-
end
|
168
|
-
end
|
132
|
+
describe "discarded job" do
|
133
|
+
before do
|
134
|
+
DummyJob.class_eval do
|
135
|
+
discard_on RuntimeError
|
169
136
|
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
discard_on RuntimeError do |job, error|
|
174
|
-
Spy.call(job, error)
|
137
|
+
def index
|
138
|
+
raise "boom"
|
139
|
+
end
|
175
140
|
end
|
176
141
|
end
|
177
|
-
end
|
178
142
|
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
task.metadata.job = job
|
143
|
+
specify do
|
144
|
+
expect_any_instance_of(Wayfarer::Redis::Counter)
|
145
|
+
.to receive(:increment).at_least(:once).and_call_original
|
183
146
|
|
184
|
-
|
147
|
+
expect {
|
148
|
+
DummyJob.perform_later(task)
|
149
|
+
perform_enqueued_jobs
|
150
|
+
assert_performed_jobs 1
|
151
|
+
expect(enqueued_jobs).to be_empty
|
152
|
+
}.not_to change { counter.value }.from(initial)
|
185
153
|
end
|
186
|
-
|
187
|
-
stub_const("Spy", spy)
|
188
154
|
end
|
189
155
|
|
190
|
-
|
191
|
-
|
192
|
-
.with(kind_of(DummyJob),
|
193
|
-
kind_of(RuntimeError))
|
156
|
+
describe "after_batch callback" do
|
157
|
+
let(:initial) { counter.value }
|
194
158
|
|
195
|
-
|
159
|
+
describe "when after_perform callback fails" do
|
160
|
+
before do
|
161
|
+
DummyJob.class_eval do
|
162
|
+
after_perform { raise }
|
196
163
|
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
end
|
164
|
+
def index; end
|
165
|
+
end
|
166
|
+
end
|
201
167
|
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
perform_enqueued_jobs
|
206
|
-
expect(task.barrier.seen?(task.url)).to be(true)
|
207
|
-
end
|
168
|
+
specify do
|
169
|
+
expect_any_instance_of(Wayfarer::Redis::Counter)
|
170
|
+
.to receive(:increment).at_least(:once).and_call_original
|
208
171
|
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
172
|
+
expect {
|
173
|
+
DummyJob.perform_later(task)
|
174
|
+
expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
|
175
|
+
assert_performed_jobs 1
|
176
|
+
expect(enqueued_jobs).to be_empty
|
177
|
+
}.not_to change { counter.value }.from(initial)
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
describe "when after_batch callback fails" do
|
182
|
+
before do
|
183
|
+
DummyJob.class_eval do
|
184
|
+
after_batch { raise }
|
185
|
+
|
186
|
+
def index; end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
specify do
|
191
|
+
expect_any_instance_of(Wayfarer::Redis::Counter)
|
192
|
+
.to receive(:increment).at_least(:once).and_call_original
|
215
193
|
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
194
|
+
expect {
|
195
|
+
DummyJob.perform_later(task)
|
196
|
+
expect { perform_enqueued_jobs }.to raise_error(RuntimeError)
|
197
|
+
assert_performed_jobs 1
|
198
|
+
expect(enqueued_jobs).to be_empty
|
199
|
+
}.not_to change { counter.value }.from(initial)
|
200
|
+
end
|
201
|
+
end
|
222
202
|
end
|
223
203
|
end
|
224
204
|
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::BatchCompletion, redis: true do
|
6
|
+
let(:task) { build(:task, :redis_pool) }
|
7
|
+
let(:job) { double(arguments: [task]) }
|
8
|
+
|
9
|
+
describe "::call" do
|
10
|
+
let(:name) { "foo" }
|
11
|
+
|
12
|
+
subject { described_class.call(name, nil, nil, nil, { job: job }) }
|
13
|
+
|
14
|
+
context "with Wayfarer job" do
|
15
|
+
before { job.extend(Wayfarer::Base) }
|
16
|
+
|
17
|
+
specify do
|
18
|
+
expect(Wayfarer::BatchCompletion)
|
19
|
+
.to receive(:handle).with(name, job, task, instance_of(Wayfarer::Redis::Counter))
|
20
|
+
|
21
|
+
subject
|
22
|
+
end
|
23
|
+
|
24
|
+
it "does not reassign Redis pool" do
|
25
|
+
expect { subject }.not_to change { task[:redis_pool] }.from(Wayfarer::Redis::Pool.instance)
|
26
|
+
end
|
27
|
+
|
28
|
+
context "without Redis pool" do
|
29
|
+
before do
|
30
|
+
task[:redis_pool] = nil
|
31
|
+
end
|
32
|
+
|
33
|
+
it "assigns Redis pool" do
|
34
|
+
expect { subject }.to change { task[:redis_pool] }.from(nil).to(Wayfarer::Redis::Pool.instance)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
context "with other job" do
|
40
|
+
specify do
|
41
|
+
expect(Wayfarer::BatchCompletion).not_to receive(:handle)
|
42
|
+
|
43
|
+
subject
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
describe "::handle" do
|
49
|
+
let(:job) do
|
50
|
+
double(arguments: [task],
|
51
|
+
executions: executions,
|
52
|
+
exception_executions: exception_executions)
|
53
|
+
end
|
54
|
+
|
55
|
+
let(:initial_exception_executions) { {} }
|
56
|
+
let(:exception_executions) { initial_exception_executions }
|
57
|
+
let(:executions) { 0 }
|
58
|
+
let(:counter) { Wayfarer::Redis::Counter.new(task) }
|
59
|
+
|
60
|
+
subject(:handle) { described_class.handle(event, job, task, counter) }
|
61
|
+
|
62
|
+
before { task[:initial_exception_executions] = initial_exception_executions }
|
63
|
+
|
64
|
+
context "enqueue.active_job" do
|
65
|
+
let(:event) { "enqueue.active_job" }
|
66
|
+
|
67
|
+
specify do
|
68
|
+
expect { handle }.to change { counter.value }.by(1)
|
69
|
+
end
|
70
|
+
|
71
|
+
context "with retry" do
|
72
|
+
let(:executions) { 1 }
|
73
|
+
|
74
|
+
specify do
|
75
|
+
expect { handle }.not_to(change { counter.value })
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
context "perform.active_job" do
|
81
|
+
let(:event) { "perform.active_job" }
|
82
|
+
|
83
|
+
specify do
|
84
|
+
expect { handle }.to change { counter.value }.by(-1)
|
85
|
+
end
|
86
|
+
|
87
|
+
context "with exception occurred" do
|
88
|
+
let(:exception_executions) { { "[RuntimeError]" => 1 } }
|
89
|
+
|
90
|
+
specify do
|
91
|
+
expect { handle }.not_to(change { counter.value })
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
context "retry_stopped.active_job" do
|
97
|
+
let(:event) { "retry_stopped.active_job" }
|
98
|
+
|
99
|
+
specify do
|
100
|
+
expect { handle }.to change { counter.value }.by(-1)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
data/spec/cli/job_spec.rb
CHANGED
@@ -2,76 +2,72 @@
|
|
2
2
|
|
3
3
|
require "spec_helpers"
|
4
4
|
|
5
|
-
describe Wayfarer::CLI
|
5
|
+
describe Wayfarer::CLI, cli: true, redis: true do
|
6
6
|
include Wayfarer::Redis
|
7
7
|
|
8
8
|
let(:url) { test_app_path("/hello_world") }
|
9
9
|
let(:batch) { "my-batch" }
|
10
|
-
subject(:cli) { Wayfarer::CLI
|
10
|
+
subject(:cli) { Wayfarer::CLI }
|
11
11
|
|
12
12
|
before do
|
13
|
-
write_file "
|
14
|
-
class DummyJob <
|
13
|
+
write_file "dummy_job.rb", <<~FILE
|
14
|
+
class DummyJob < ActiveJob::Base
|
15
|
+
include Wayfarer::Base
|
15
16
|
end
|
16
17
|
FILE
|
17
18
|
end
|
18
19
|
|
19
|
-
before { Wayfarer::CLI
|
20
|
+
before { Wayfarer::CLI.new.send(:load_environment, "dummy_job.rb") }
|
20
21
|
|
21
|
-
describe "
|
22
|
+
describe "perform" do
|
22
23
|
it "performs the worker" do
|
23
24
|
expect_any_instance_of(DummyJob).to receive(:perform).with(kind_of(Wayfarer::Task)) do |job|
|
24
25
|
task = job.arguments.first
|
25
|
-
task
|
26
|
+
task[:job] = job
|
26
27
|
end
|
27
28
|
|
28
|
-
cli.start(["
|
29
|
-
end
|
30
|
-
|
31
|
-
it "collects garbage" do
|
32
|
-
expect_any_instance_of(Wayfarer::GC).to receive(:run).exactly(:once)
|
33
|
-
cli.start(["job", "perform", "DummyJob", url])
|
29
|
+
cli.start(["perform", "-r", "dummy_job.rb", "DummyJob", url])
|
34
30
|
end
|
35
31
|
|
36
32
|
context "using MockRedis" do
|
37
33
|
it "performs the worker using MockRedis" do
|
38
|
-
cli.start(["
|
39
|
-
expect(Wayfarer.config
|
34
|
+
cli.start(["perform", "--mock-redis", "DummyJob", url])
|
35
|
+
expect(Wayfarer.config[:redis][:factory].call(nil)).to be_a(MockRedis)
|
40
36
|
end
|
41
37
|
end
|
42
38
|
end
|
43
39
|
|
44
|
-
describe "
|
40
|
+
describe "enqueue" do
|
45
41
|
it "enqueues the job" do
|
46
42
|
expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
|
47
|
-
cli.start(["
|
43
|
+
cli.start(["enqueue", "DummyJob", url])
|
48
44
|
end
|
49
45
|
|
50
46
|
context "with batch provided" do
|
51
47
|
it "enqueues the job" do
|
52
48
|
expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: batch)
|
53
|
-
cli.start(["
|
49
|
+
cli.start(["enqueue", "--batch", batch, "DummyJob", url])
|
54
50
|
end
|
55
51
|
end
|
56
52
|
end
|
57
53
|
|
58
|
-
describe "
|
54
|
+
describe "execute" do
|
59
55
|
it "executes the job" do
|
60
56
|
expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: kind_of(String))
|
61
|
-
cli.start(["
|
57
|
+
cli.start(["execute", "DummyJob", url])
|
62
58
|
end
|
63
59
|
|
64
60
|
context "with batch provided" do
|
65
61
|
it "enqueues the job" do
|
66
62
|
expect(DummyJob).to receive(:crawl).with(Addressable::URI.parse(url), batch: batch)
|
67
|
-
cli.start(["
|
63
|
+
cli.start(["execute", "--batch", batch, "DummyJob", url])
|
68
64
|
end
|
69
65
|
end
|
70
66
|
|
71
67
|
context "using MockRedis" do
|
72
68
|
it "performs the worker using MockRedis" do
|
73
|
-
cli.start(["
|
74
|
-
expect(Wayfarer.config
|
69
|
+
cli.start(["execute", "--mock-redis", "DummyJob", url])
|
70
|
+
expect(Wayfarer.config[:redis][:factory].call(nil)).to be_a(MockRedis)
|
75
71
|
end
|
76
72
|
end
|
77
73
|
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "spec_helpers"
|
4
|
+
|
5
|
+
describe Wayfarer::CLI, cli: true do
|
6
|
+
before { write_file("dummy_job.rb", contents) }
|
7
|
+
|
8
|
+
let(:contents) do
|
9
|
+
<<~RUBY
|
10
|
+
class DummyJob < ActiveJob::Base
|
11
|
+
include Wayfarer::Base
|
12
|
+
|
13
|
+
#{routes}
|
14
|
+
end
|
15
|
+
RUBY
|
16
|
+
end
|
17
|
+
|
18
|
+
describe "route" do
|
19
|
+
let(:routes) do
|
20
|
+
<<~RUBY
|
21
|
+
route.host #{test_app_host.inspect}, to: :index
|
22
|
+
RUBY
|
23
|
+
end
|
24
|
+
|
25
|
+
subject(:route) { described_class.start(["route", "-r", "dummy_job.rb", "DummyJob", url]) }
|
26
|
+
|
27
|
+
context "with matching URL" do
|
28
|
+
let(:url) { "http://#{test_app_host}" }
|
29
|
+
|
30
|
+
specify do
|
31
|
+
expect { route }.to output("Match => :index\n").to_stdout
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
context "with mismatching URL" do
|
36
|
+
let(:url) { "http://example.com" }
|
37
|
+
|
38
|
+
specify do
|
39
|
+
expect { route }.to output("Mismatch\n").to_stdout
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe "tree" do
|
45
|
+
subject(:tree) { described_class.start(["tree", "-r", "dummy_job.rb", "DummyJob", url]) }
|
46
|
+
|
47
|
+
context "without child routes" do
|
48
|
+
let(:url) { "http://#{test_app_host}" }
|
49
|
+
let(:routes) { "" }
|
50
|
+
|
51
|
+
specify do
|
52
|
+
expect { tree }.to output("Mismatch\n").to_stdout
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
context "symbol target" do
|
57
|
+
let(:url) { "http://#{test_app_host}" }
|
58
|
+
let(:routes) do
|
59
|
+
<<~RUBY
|
60
|
+
route.host #{test_app_host.inspect}, to: :index
|
61
|
+
RUBY
|
62
|
+
end
|
63
|
+
|
64
|
+
let(:expected_output) do
|
65
|
+
<<~OUTPUT
|
66
|
+
Match(:index)
|
67
|
+
└──Host("#{test_app_host}", match: true)
|
68
|
+
└──Target(match: true)
|
69
|
+
OUTPUT
|
70
|
+
end
|
71
|
+
|
72
|
+
specify do
|
73
|
+
expect { tree }.to output(expected_output).to_stdout
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
context "symbol target with params" do
|
78
|
+
let(:url) { "http://#{test_app_host}/barqux" }
|
79
|
+
let(:routes) do
|
80
|
+
<<~RUBY
|
81
|
+
route.host #{test_app_host.inspect}, to: :index do
|
82
|
+
path ":foobar"
|
83
|
+
end
|
84
|
+
RUBY
|
85
|
+
end
|
86
|
+
|
87
|
+
let(:expected_output) do
|
88
|
+
<<~OUTPUT
|
89
|
+
Match(:index, params: {:foobar=>"barqux"})
|
90
|
+
└──Host("#{test_app_host}", match: true)
|
91
|
+
└──Target(match: true)
|
92
|
+
└──Path("/:foobar", match: true, params: {:foobar=>"barqux"})
|
93
|
+
OUTPUT
|
94
|
+
end
|
95
|
+
|
96
|
+
specify do
|
97
|
+
expect { tree }.to output(expected_output).to_stdout
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|