delayed 1.2.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +19 -6
- data/Rakefile +5 -1
- data/app/models/delayed/job.rb +42 -29
- data/db/migrate/1_create_delayed_jobs.rb +0 -2
- data/db/migrate/3_add_index_to_delayed_jobs_name.rb +14 -6
- data/db/migrate/4_index_live_jobs.rb +33 -0
- data/db/migrate/5_index_failed_jobs.rb +24 -0
- data/db/migrate/6_set_postgres_fillfactor.rb +31 -0
- data/db/migrate/7_remove_legacy_index.rb +12 -0
- data/lib/delayed/backend/job_preparer.rb +19 -0
- data/lib/delayed/exceptions.rb +4 -1
- data/lib/delayed/helpers/migration.rb +116 -0
- data/lib/delayed/monitor.rb +21 -11
- data/lib/delayed/version.rb +1 -1
- data/lib/delayed/worker.rb +1 -1
- data/lib/delayed.rb +1 -0
- data/spec/delayed/__snapshots__/job_spec.rb.snap +271 -0
- data/spec/delayed/__snapshots__/monitor_spec.rb.snap +969 -0
- data/spec/delayed/job_spec.rb +189 -13
- data/spec/delayed/monitor_spec.rb +61 -18
- data/spec/helper.rb +129 -11
- data/spec/sample_jobs.rb +10 -0
- data/spec/worker_spec.rb +18 -0
- metadata +15 -3
data/spec/delayed/job_spec.rb
CHANGED
|
@@ -4,7 +4,7 @@ describe Delayed::Job do
|
|
|
4
4
|
let(:worker) { Delayed::Worker.new }
|
|
5
5
|
|
|
6
6
|
def create_job(opts = {})
|
|
7
|
-
described_class.
|
|
7
|
+
described_class.enqueue(SimpleJob.new, **opts)
|
|
8
8
|
end
|
|
9
9
|
|
|
10
10
|
before do
|
|
@@ -197,6 +197,103 @@ describe Delayed::Job do
|
|
|
197
197
|
end
|
|
198
198
|
end
|
|
199
199
|
|
|
200
|
+
describe '.claimable_by' do
|
|
201
|
+
let(:now) { '2025-11-10 17:20:13 UTC' }
|
|
202
|
+
let(:worker) do
|
|
203
|
+
instance_double(Delayed::Worker, name: "worker1", read_ahead: 1, max_claims: 1, min_priority: nil, max_priority: nil, queues: queues)
|
|
204
|
+
end
|
|
205
|
+
let(:queues) { [] }
|
|
206
|
+
let(:query) { QueryUnderTest.for(described_class.claimable_by(worker)) }
|
|
207
|
+
|
|
208
|
+
around { |example| Timecop.freeze(now) { example.run } }
|
|
209
|
+
|
|
210
|
+
it "generates the expected #{current_adapter} query" do
|
|
211
|
+
expect(query.formatted).to match_snapshot
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
it "generates the expected #{current_adapter} query plan" do
|
|
215
|
+
expect(query.explain).to match_snapshot
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
context 'when a single queue is specified' do
|
|
219
|
+
let(:queues) { %w(default) }
|
|
220
|
+
|
|
221
|
+
it "generates the expected #{current_adapter} query for one queue" do
|
|
222
|
+
expect(query.formatted).to match_snapshot
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
it "generates a #{current_adapter} query plan for one queue" do
|
|
226
|
+
expect(query.explain).to match_snapshot
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
context 'multiple queues are specified' do
|
|
231
|
+
let(:queues) { %w(default mailers tracking) }
|
|
232
|
+
|
|
233
|
+
it "generates the expected #{current_adapter} query for multiple queue" do
|
|
234
|
+
expect(query.formatted).to match_snapshot
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
it "generates a #{current_adapter} query plan for multiple queues" do
|
|
238
|
+
expect(query.explain).to match_snapshot
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
context 'when using the legacy index', :with_legacy_table_index do
|
|
243
|
+
it "[legacy index] generates the expected #{current_adapter} query plan" do
|
|
244
|
+
expect(query.explain).to match_snapshot
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
context 'when a single queue is specified' do
|
|
248
|
+
let(:queues) { %w(default) }
|
|
249
|
+
|
|
250
|
+
it "[legacy index] generates a #{current_adapter} query plan for one queue" do
|
|
251
|
+
expect(query.explain).to match_snapshot
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
context 'multiple queues are specified' do
|
|
256
|
+
let(:queues) { %w(default mailers tracking) }
|
|
257
|
+
|
|
258
|
+
it "[legacy index] generates a #{current_adapter} query plan for multiple queues" do
|
|
259
|
+
expect(query.explain).to match_snapshot
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
describe '.claimed_by' do
|
|
266
|
+
let(:now) { '2025-11-10 17:20:13 UTC' }
|
|
267
|
+
let(:worker) { instance_double(Delayed::Worker, name: "worker1") }
|
|
268
|
+
let(:query) { QueryUnderTest.for(described_class.claimed_by(worker).select(:locked_at, :locked_by)) }
|
|
269
|
+
|
|
270
|
+
around { |example| Timecop.freeze(now) { example.run } }
|
|
271
|
+
|
|
272
|
+
it "generates a well-scoped #{current_adapter} query" do
|
|
273
|
+
expect(query.formatted).to match_snapshot
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
it "generates an efficient #{current_adapter} query plan" do
|
|
277
|
+
expect(query.explain).to match_snapshot
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
context 'when using legacy index', :with_legacy_table_index do
|
|
281
|
+
it "[legacy index] generates an efficient #{current_adapter} query plan" do
|
|
282
|
+
expect(query.explain).to match_snapshot
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
describe '.lock_timeout' do
|
|
288
|
+
it 'changes relative to Worker.max_run_time' do
|
|
289
|
+
Delayed::Worker.max_run_time = 5.minutes
|
|
290
|
+
expect(described_class.lock_timeout).to eq(5.minutes + described_class::REENQUEUE_BUFFER)
|
|
291
|
+
|
|
292
|
+
Delayed::Worker.max_run_time = 10.minutes
|
|
293
|
+
expect(described_class.lock_timeout).to eq(10.minutes + described_class::REENQUEUE_BUFFER)
|
|
294
|
+
end
|
|
295
|
+
end
|
|
296
|
+
|
|
200
297
|
describe 'reserve' do
|
|
201
298
|
before do
|
|
202
299
|
Delayed::Worker.max_run_time = 2.minutes
|
|
@@ -204,6 +301,7 @@ describe Delayed::Job do
|
|
|
204
301
|
|
|
205
302
|
after do
|
|
206
303
|
Time.zone = nil
|
|
304
|
+
self.default_timezone = :utc
|
|
207
305
|
end
|
|
208
306
|
|
|
209
307
|
it 'does not reserve failed jobs' do
|
|
@@ -221,10 +319,82 @@ describe Delayed::Job do
|
|
|
221
319
|
expect(described_class.reserve(worker)).to eq([job])
|
|
222
320
|
end
|
|
223
321
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
322
|
+
context 'during DST change' do
|
|
323
|
+
before do
|
|
324
|
+
Time.zone = 'US/Eastern'
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
let(:dst_start) { Time.zone.parse('2024-11-03 01:00:00 EDT') }
|
|
328
|
+
|
|
329
|
+
it 'does not reserve future-scheduled jobs scheduled for during the "fall back" hour' do
|
|
330
|
+
allow(Delayed.logger).to receive(:warn)
|
|
331
|
+
job = create_job run_at: dst_start + 1.hour + 1.minute
|
|
332
|
+
expect(described_class.reserve(worker, dst_start + 59.minutes)).to eq []
|
|
333
|
+
expect(job.run_at).to eq(Time.zone.parse('2024-11-03 01:01:00 EST'))
|
|
334
|
+
expect(Delayed.logger).not_to have_received(:warn)
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
it 'does not reserve future-scheduled jobs scheduled for after the "fall back" hour' do
|
|
338
|
+
job = create_job run_at: dst_start + 2.hours + 1.minute
|
|
339
|
+
expect(described_class.reserve(worker, dst_start + 59.minutes)).to eq []
|
|
340
|
+
expect(job.run_at).to eq(Time.zone.parse('2024-11-03 02:01:00 EST'))
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
it 'reserves jobs scheduled for the past' do
|
|
344
|
+
job = create_job run_at: dst_start + 1.minute
|
|
345
|
+
expect(described_class.reserve(worker, dst_start + 59.minutes)).to eq([job])
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
it 'does not break when run_at is explicitly nil' do
|
|
349
|
+
job = create_job run_at: nil
|
|
350
|
+
expect(job.run_at).not_to be_nil
|
|
351
|
+
end
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
context 'when using :local non-UTC time for DB timestamps' do
|
|
355
|
+
before do
|
|
356
|
+
self.default_timezone = :local
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
it 'does not reserve jobs scheduled for the future' do
|
|
360
|
+
create_job run_at: 1.minute.from_now
|
|
361
|
+
expect(described_class.reserve(worker)).to eq []
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
it 'reserves jobs scheduled for the past' do
|
|
365
|
+
job = create_job run_at: 1.minute.ago
|
|
366
|
+
expect(described_class.reserve(worker)).to eq([job])
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
context 'during DST change' do
|
|
370
|
+
before do
|
|
371
|
+
Time.zone = 'US/Eastern'
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
let(:dst_start) { Time.zone.parse('2024-11-03 01:00:00 EDT') }
|
|
375
|
+
|
|
376
|
+
it 'does not reserve future-scheduled jobs scheduled for during the "fall back" hour' do
|
|
377
|
+
allow(Delayed.logger).to receive(:warn)
|
|
378
|
+
job = create_job run_at: dst_start + 1.hour + 1.minute
|
|
379
|
+
expect(described_class.reserve(worker, dst_start + 59.minutes)).to eq []
|
|
380
|
+
# see Delayed::Backend::JobPreparer#handle_dst for default_timezone = `:local` handling:
|
|
381
|
+
expect(job.run_at).to eq(Time.zone.parse('2024-11-03 02:00:00 EST'))
|
|
382
|
+
expect(Delayed.logger).to have_received(:warn).with(
|
|
383
|
+
"Adjusted run_at from 2024-11-03 01:01:00 -0500 to 2024-11-03 02:00:00 -0500 to account for fall back DST transition",
|
|
384
|
+
)
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
it 'does not reserve future-scheduled jobs scheduled for after the "fall back" hour' do
|
|
388
|
+
job = create_job run_at: dst_start + 2.hours + 1.minute
|
|
389
|
+
expect(described_class.reserve(worker, dst_start + 59.minutes)).to eq []
|
|
390
|
+
expect(job.run_at).to eq(Time.zone.parse('2024-11-03 02:01:00 EST'))
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
it 'reserves jobs scheduled for the past' do
|
|
394
|
+
job = create_job run_at: dst_start + 1.minute
|
|
395
|
+
expect(described_class.reserve(worker, dst_start + 59.minutes)).to eq([job])
|
|
396
|
+
end
|
|
397
|
+
end
|
|
228
398
|
end
|
|
229
399
|
|
|
230
400
|
it 'does not reserve jobs locked by other workers' do
|
|
@@ -412,18 +582,24 @@ describe Delayed::Job do
|
|
|
412
582
|
end
|
|
413
583
|
|
|
414
584
|
context 'clear_locks!' do
|
|
415
|
-
|
|
416
|
-
|
|
585
|
+
let(:worker) do
|
|
586
|
+
instance_double(Delayed::Worker, name: "worker1", read_ahead: 1, max_claims: 1, min_priority: nil, max_priority: nil, queues: [])
|
|
587
|
+
end
|
|
588
|
+
let(:different_worker) do
|
|
589
|
+
instance_double(Delayed::Worker, name: "worker2", read_ahead: 1, max_claims: 1, min_priority: nil, max_priority: nil, queues: [])
|
|
417
590
|
end
|
|
418
591
|
|
|
419
592
|
it 'clears locks for the given worker' do
|
|
420
|
-
|
|
421
|
-
|
|
593
|
+
job = create_job(locked_by: 'worker1', locked_at: described_class.db_time_now)
|
|
594
|
+
described_class.clear_locks!(worker)
|
|
595
|
+
expect(described_class.reserve(different_worker)).to eq([job])
|
|
422
596
|
end
|
|
423
597
|
|
|
424
598
|
it 'does not clear locks for other workers' do
|
|
425
|
-
|
|
426
|
-
|
|
599
|
+
job = create_job(locked_by: 'worker1', locked_at: described_class.db_time_now)
|
|
600
|
+
described_class.clear_locks!(different_worker)
|
|
601
|
+
expect(described_class.reserve(different_worker)).not_to include(job)
|
|
602
|
+
expect(described_class.reserve(worker)).to eq([job])
|
|
427
603
|
end
|
|
428
604
|
end
|
|
429
605
|
|
|
@@ -674,7 +850,7 @@ describe Delayed::Job do
|
|
|
674
850
|
worker.perform(job)
|
|
675
851
|
job.reload
|
|
676
852
|
|
|
677
|
-
expect((described_class.db_time_now + 99.minutes - job.run_at).abs).to
|
|
853
|
+
expect((described_class.db_time_now + 99.minutes - job.run_at).abs).to be_within(1).of(0.5)
|
|
678
854
|
end
|
|
679
855
|
|
|
680
856
|
it "does not fail when the triggered error doesn't have a message" do
|
|
@@ -839,7 +1015,7 @@ describe Delayed::Job do
|
|
|
839
1015
|
context "db_time_now" do
|
|
840
1016
|
after do
|
|
841
1017
|
Time.zone = nil
|
|
842
|
-
self.default_timezone = :
|
|
1018
|
+
self.default_timezone = :utc
|
|
843
1019
|
end
|
|
844
1020
|
|
|
845
1021
|
it "returns time in current time zone if set" do
|
|
@@ -94,7 +94,7 @@ RSpec.describe Delayed::Monitor do
|
|
|
94
94
|
end
|
|
95
95
|
|
|
96
96
|
context 'when there are jobs in the queue' do
|
|
97
|
-
let(:now) {
|
|
97
|
+
let(:now) { Delayed::Job.db_time_now.change(nsec: 0) }
|
|
98
98
|
let(:job_attributes) do
|
|
99
99
|
{
|
|
100
100
|
run_at: now,
|
|
@@ -103,11 +103,11 @@ RSpec.describe Delayed::Monitor do
|
|
|
103
103
|
attempts: 0,
|
|
104
104
|
}
|
|
105
105
|
end
|
|
106
|
-
let(:failed_attributes) { { run_at: now - 1.week,
|
|
107
|
-
let(:p0_attributes) { job_attributes.merge(priority: 1) }
|
|
108
|
-
let(:p10_attributes) { job_attributes.merge(priority: 13) }
|
|
109
|
-
let(:p20_attributes) { job_attributes.merge(priority: 23) }
|
|
110
|
-
let(:p30_attributes) { job_attributes.merge(priority: 999) }
|
|
106
|
+
let(:failed_attributes) { { run_at: now - 1.week, attempts: 1, failed_at: now - 1.day, locked_at: now - 1.day } }
|
|
107
|
+
let(:p0_attributes) { job_attributes.merge(priority: 1, attempts: 1) }
|
|
108
|
+
let(:p10_attributes) { job_attributes.merge(priority: 13, locked_at: now - 1.day) }
|
|
109
|
+
let(:p20_attributes) { job_attributes.merge(priority: 23, attempts: 1) }
|
|
110
|
+
let(:p30_attributes) { job_attributes.merge(priority: 999, locked_at: now - 1.day) }
|
|
111
111
|
let(:p0_payload) { default_payload.merge(priority: 'interactive') }
|
|
112
112
|
let(:p10_payload) { default_payload.merge(priority: 'user_visible') }
|
|
113
113
|
let(:p20_payload) { default_payload.merge(priority: 'eventual') }
|
|
@@ -139,8 +139,8 @@ RSpec.describe Delayed::Monitor do
|
|
|
139
139
|
.to emit_notification("delayed.monitor.run").with_payload(default_payload.except(:queue))
|
|
140
140
|
.and emit_notification("delayed.job.count").with_payload(p0_payload).with_value(4)
|
|
141
141
|
.and emit_notification("delayed.job.future_count").with_payload(p0_payload).with_value(1)
|
|
142
|
-
.and emit_notification("delayed.job.locked_count").with_payload(p0_payload).with_value(
|
|
143
|
-
.and emit_notification("delayed.job.erroring_count").with_payload(p0_payload).with_value(
|
|
142
|
+
.and emit_notification("delayed.job.locked_count").with_payload(p0_payload).with_value(1)
|
|
143
|
+
.and emit_notification("delayed.job.erroring_count").with_payload(p0_payload).with_value(3)
|
|
144
144
|
.and emit_notification("delayed.job.failed_count").with_payload(p0_payload).with_value(1)
|
|
145
145
|
.and emit_notification("delayed.job.working_count").with_payload(p0_payload).with_value(1)
|
|
146
146
|
.and emit_notification("delayed.job.workable_count").with_payload(p0_payload).with_value(1)
|
|
@@ -149,8 +149,8 @@ RSpec.describe Delayed::Monitor do
|
|
|
149
149
|
.and emit_notification("delayed.job.alert_age_percent").with_payload(p0_payload).with_value(30.0.seconds / 1.minute * 100)
|
|
150
150
|
.and emit_notification("delayed.job.count").with_payload(p10_payload).with_value(4)
|
|
151
151
|
.and emit_notification("delayed.job.future_count").with_payload(p10_payload).with_value(1)
|
|
152
|
-
.and emit_notification("delayed.job.locked_count").with_payload(p10_payload).with_value(
|
|
153
|
-
.and emit_notification("delayed.job.erroring_count").with_payload(p10_payload).with_value(
|
|
152
|
+
.and emit_notification("delayed.job.locked_count").with_payload(p10_payload).with_value(1)
|
|
153
|
+
.and emit_notification("delayed.job.erroring_count").with_payload(p10_payload).with_value(0)
|
|
154
154
|
.and emit_notification("delayed.job.failed_count").with_payload(p10_payload).with_value(1)
|
|
155
155
|
.and emit_notification("delayed.job.working_count").with_payload(p10_payload).with_value(1)
|
|
156
156
|
.and emit_notification("delayed.job.workable_count").with_payload(p10_payload).with_value(1)
|
|
@@ -159,8 +159,8 @@ RSpec.describe Delayed::Monitor do
|
|
|
159
159
|
.and emit_notification("delayed.job.alert_age_percent").with_payload(p10_payload).with_value(2.0.minutes / 3.minutes * 100)
|
|
160
160
|
.and emit_notification("delayed.job.count").with_payload(p20_payload).with_value(4)
|
|
161
161
|
.and emit_notification("delayed.job.future_count").with_payload(p20_payload).with_value(1)
|
|
162
|
-
.and emit_notification("delayed.job.locked_count").with_payload(p20_payload).with_value(
|
|
163
|
-
.and emit_notification("delayed.job.erroring_count").with_payload(p20_payload).with_value(
|
|
162
|
+
.and emit_notification("delayed.job.locked_count").with_payload(p20_payload).with_value(1)
|
|
163
|
+
.and emit_notification("delayed.job.erroring_count").with_payload(p20_payload).with_value(3)
|
|
164
164
|
.and emit_notification("delayed.job.failed_count").with_payload(p20_payload).with_value(1)
|
|
165
165
|
.and emit_notification("delayed.job.working_count").with_payload(p20_payload).with_value(1)
|
|
166
166
|
.and emit_notification("delayed.job.workable_count").with_payload(p20_payload).with_value(1)
|
|
@@ -169,8 +169,8 @@ RSpec.describe Delayed::Monitor do
|
|
|
169
169
|
.and emit_notification("delayed.job.alert_age_percent").with_payload(p20_payload).with_value(1.hour / 1.5.hours * 100)
|
|
170
170
|
.and emit_notification("delayed.job.count").with_payload(p30_payload).with_value(4)
|
|
171
171
|
.and emit_notification("delayed.job.future_count").with_payload(p30_payload).with_value(1)
|
|
172
|
-
.and emit_notification("delayed.job.locked_count").with_payload(p30_payload).with_value(
|
|
173
|
-
.and emit_notification("delayed.job.erroring_count").with_payload(p30_payload).with_value(
|
|
172
|
+
.and emit_notification("delayed.job.locked_count").with_payload(p30_payload).with_value(1)
|
|
173
|
+
.and emit_notification("delayed.job.erroring_count").with_payload(p30_payload).with_value(0)
|
|
174
174
|
.and emit_notification("delayed.job.failed_count").with_payload(p30_payload).with_value(1)
|
|
175
175
|
.and emit_notification("delayed.job.working_count").with_payload(p30_payload).with_value(1)
|
|
176
176
|
.and emit_notification("delayed.job.workable_count").with_payload(p30_payload).with_value(1)
|
|
@@ -196,8 +196,8 @@ RSpec.describe Delayed::Monitor do
|
|
|
196
196
|
.to emit_notification("delayed.monitor.run").with_payload(default_payload.except(:queue))
|
|
197
197
|
.and emit_notification("delayed.job.count").with_payload(p0_payload).with_value(8)
|
|
198
198
|
.and emit_notification("delayed.job.future_count").with_payload(p0_payload).with_value(2)
|
|
199
|
-
.and emit_notification("delayed.job.locked_count").with_payload(p0_payload).with_value(
|
|
200
|
-
.and emit_notification("delayed.job.erroring_count").with_payload(p0_payload).with_value(
|
|
199
|
+
.and emit_notification("delayed.job.locked_count").with_payload(p0_payload).with_value(2)
|
|
200
|
+
.and emit_notification("delayed.job.erroring_count").with_payload(p0_payload).with_value(3)
|
|
201
201
|
.and emit_notification("delayed.job.failed_count").with_payload(p0_payload).with_value(2)
|
|
202
202
|
.and emit_notification("delayed.job.working_count").with_payload(p0_payload).with_value(2)
|
|
203
203
|
.and emit_notification("delayed.job.workable_count").with_payload(p0_payload).with_value(2)
|
|
@@ -206,8 +206,8 @@ RSpec.describe Delayed::Monitor do
|
|
|
206
206
|
.and emit_notification("delayed.job.alert_age_percent").with_payload(p0_payload).with_value(0)
|
|
207
207
|
.and emit_notification("delayed.job.count").with_payload(p20_payload).with_value(8)
|
|
208
208
|
.and emit_notification("delayed.job.future_count").with_payload(p20_payload).with_value(2)
|
|
209
|
-
.and emit_notification("delayed.job.locked_count").with_payload(p20_payload).with_value(
|
|
210
|
-
.and emit_notification("delayed.job.erroring_count").with_payload(p20_payload).with_value(
|
|
209
|
+
.and emit_notification("delayed.job.locked_count").with_payload(p20_payload).with_value(2)
|
|
210
|
+
.and emit_notification("delayed.job.erroring_count").with_payload(p20_payload).with_value(3)
|
|
211
211
|
.and emit_notification("delayed.job.failed_count").with_payload(p20_payload).with_value(2)
|
|
212
212
|
.and emit_notification("delayed.job.working_count").with_payload(p20_payload).with_value(2)
|
|
213
213
|
.and emit_notification("delayed.job.workable_count").with_payload(p20_payload).with_value(2)
|
|
@@ -273,4 +273,47 @@ RSpec.describe Delayed::Monitor do
|
|
|
273
273
|
end
|
|
274
274
|
end
|
|
275
275
|
end
|
|
276
|
+
|
|
277
|
+
describe '#query_for' do
|
|
278
|
+
let(:monitor) { described_class.new }
|
|
279
|
+
let(:queries) { [] }
|
|
280
|
+
let(:now) { '2025-11-10 17:20:13 UTC' }
|
|
281
|
+
|
|
282
|
+
around { |example| Timecop.freeze(now) { example.run } }
|
|
283
|
+
|
|
284
|
+
before do
|
|
285
|
+
Delayed::Worker.queues = []
|
|
286
|
+
ActiveSupport::Notifications.subscribe('sql.active_record') do |_, _, _, _, details|
|
|
287
|
+
sql = details[:sql]
|
|
288
|
+
details[:binds]&.each do |value|
|
|
289
|
+
value = value.value if value.is_a?(ActiveModel::Attribute)
|
|
290
|
+
sql = sql.sub(/(\?|\$\d)/, ActiveRecord::Base.connection.quote(value))
|
|
291
|
+
end
|
|
292
|
+
queries << sql
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def queries_for(metric)
|
|
297
|
+
monitor.query_for(metric)
|
|
298
|
+
queries.map { |q| QueryUnderTest.for(q) }
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
described_class::METRICS.each do |metric|
|
|
302
|
+
context "('#{metric}')" do
|
|
303
|
+
it "runs the expected #{current_adapter} query for #{metric}" do
|
|
304
|
+
expect(queries_for(metric).map(&:formatted).join("\n")).to match_snapshot
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
it "produces the expected #{current_adapter} query plan for #{metric}" do
|
|
308
|
+
expect(queries_for(metric).map(&:explain).join("\n")).to match_snapshot
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
context 'when using the legacy index', :with_legacy_table_index do
|
|
312
|
+
it "[legacy index] produces the expected #{current_adapter} query plan for #{metric}" do
|
|
313
|
+
expect(queries_for(metric).map(&:explain).join("\n")).to match_snapshot
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
end
|
|
276
319
|
end
|
data/spec/helper.rb
CHANGED
|
@@ -10,6 +10,8 @@ require 'sample_jobs'
|
|
|
10
10
|
|
|
11
11
|
require 'rake'
|
|
12
12
|
|
|
13
|
+
require 'snapshot_testing/rspec'
|
|
14
|
+
|
|
13
15
|
ActiveSupport.on_load(:active_record) do
|
|
14
16
|
require 'global_id/identification'
|
|
15
17
|
include GlobalID::Identification
|
|
@@ -23,15 +25,17 @@ else
|
|
|
23
25
|
ActiveSupport::Deprecation.behavior = :raise
|
|
24
26
|
end
|
|
25
27
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
else
|
|
29
|
-
require 'tempfile'
|
|
28
|
+
Delayed.logger = Logger.new($stdout)
|
|
29
|
+
Delayed.logger.level = ENV['DEBUG_LOGS'] ? Logger::DEBUG : Logger::FATAL
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
def with_log_level(level)
|
|
32
|
+
logger_level_was = Delayed.logger.level
|
|
33
|
+
Delayed.logger.level = level
|
|
34
|
+
yield
|
|
35
|
+
ensure
|
|
36
|
+
Delayed.logger.level = logger_level_was
|
|
34
37
|
end
|
|
38
|
+
|
|
35
39
|
ENV['RAILS_ENV'] = 'test'
|
|
36
40
|
|
|
37
41
|
db_adapter = ENV["ADAPTER"]
|
|
@@ -45,6 +49,12 @@ ActiveRecord::Base.logger = Delayed.logger
|
|
|
45
49
|
ActiveJob::Base.logger = Delayed.logger
|
|
46
50
|
ActiveRecord::Migration.verbose = false
|
|
47
51
|
|
|
52
|
+
if ActiveRecord.respond_to?(:default_timezone=)
|
|
53
|
+
ActiveRecord.default_timezone = :utc
|
|
54
|
+
else
|
|
55
|
+
ActiveRecord::Base.default_timezone = :utc
|
|
56
|
+
end
|
|
57
|
+
|
|
48
58
|
# MySQL 5.7 no longer supports null default values for the primary key
|
|
49
59
|
# Override the default primary key type in Rails <= 4.0
|
|
50
60
|
# https://stackoverflow.com/a/34555109
|
|
@@ -62,11 +72,35 @@ end
|
|
|
62
72
|
Dir['db/migrate/*.rb'].each { |f| require_relative("../#{f}") }
|
|
63
73
|
|
|
64
74
|
ActiveRecord::Schema.define do
|
|
65
|
-
|
|
75
|
+
if ActiveRecord::VERSION::MAJOR >= 7
|
|
76
|
+
drop_table :delayed_jobs, if_exists: true
|
|
77
|
+
elsif ActiveRecord::Base.connection.table_exists?(:delayed_jobs)
|
|
78
|
+
drop_table :delayed_jobs
|
|
79
|
+
end
|
|
66
80
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
81
|
+
# Let's prove reversibility when we set up our test DB:
|
|
82
|
+
def run_migration(klass)
|
|
83
|
+
klass.migrate(:up)
|
|
84
|
+
klass.migrate(:down)
|
|
85
|
+
klass.migrate(:up)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
with_log_level(Logger::WARN) do
|
|
89
|
+
run_migration(CreateDelayedJobs)
|
|
90
|
+
run_migration(AddNameToDelayedJobs)
|
|
91
|
+
run_migration(AddIndexToDelayedJobsName)
|
|
92
|
+
run_migration(IndexLiveJobs)
|
|
93
|
+
run_migration(IndexFailedJobs)
|
|
94
|
+
run_migration(SetPostgresFillfactor)
|
|
95
|
+
run_migration(RemoveLegacyIndex)
|
|
96
|
+
|
|
97
|
+
# Test that these index migrations can be re-applied idempotently.
|
|
98
|
+
# (In case identical indexes had been manually applied previously.)
|
|
99
|
+
AddIndexToDelayedJobsName.migrate(:up)
|
|
100
|
+
IndexLiveJobs.migrate(:up)
|
|
101
|
+
IndexFailedJobs.migrate(:up)
|
|
102
|
+
RemoveLegacyIndex.migrate(:up)
|
|
103
|
+
end
|
|
70
104
|
|
|
71
105
|
create_table :stories, primary_key: :story_id, force: true do |table|
|
|
72
106
|
table.string :text
|
|
@@ -101,6 +135,8 @@ TEST_MIN_RESERVE_INTERVAL = -10
|
|
|
101
135
|
TEST_SLEEP_DELAY = -100
|
|
102
136
|
|
|
103
137
|
RSpec.configure do |config|
|
|
138
|
+
config.include SnapshotTesting::RSpec
|
|
139
|
+
|
|
104
140
|
config.around(:each) do |example|
|
|
105
141
|
aj_priority_was = ActiveJob::Base.priority
|
|
106
142
|
aj_queue_name_was = ActiveJob::Base.queue_name
|
|
@@ -144,6 +180,19 @@ RSpec.configure do |config|
|
|
|
144
180
|
Delayed::Job.delete_all
|
|
145
181
|
end
|
|
146
182
|
|
|
183
|
+
config.around(:each, :with_legacy_table_index) do |example|
|
|
184
|
+
IndexFailedJobs.migrate(:down)
|
|
185
|
+
IndexLiveJobs.migrate(:down)
|
|
186
|
+
AddIndexToDelayedJobsName.migrate(:down)
|
|
187
|
+
RemoveLegacyIndex.migrate(:down)
|
|
188
|
+
example.run
|
|
189
|
+
ensure
|
|
190
|
+
RemoveLegacyIndex.migrate(:up)
|
|
191
|
+
AddIndexToDelayedJobsName.migrate(:up)
|
|
192
|
+
IndexLiveJobs.migrate(:up)
|
|
193
|
+
IndexFailedJobs.migrate(:up)
|
|
194
|
+
end
|
|
195
|
+
|
|
147
196
|
config.expect_with :rspec do |c|
|
|
148
197
|
c.syntax = :expect
|
|
149
198
|
end
|
|
@@ -211,3 +260,72 @@ def current_database
|
|
|
211
260
|
'delayed_job_test'
|
|
212
261
|
end
|
|
213
262
|
end
|
|
263
|
+
|
|
264
|
+
QueryUnderTest = Struct.new(:sql, :connection) do
|
|
265
|
+
def self.for(query, connection: ActiveRecord::Base.connection)
|
|
266
|
+
new(query.respond_to?(:to_sql) ? query.to_sql : query.to_s, connection)
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def formatted
|
|
270
|
+
fmt = sql.squish
|
|
271
|
+
|
|
272
|
+
if ActiveRecord::VERSION::MAJOR < 7
|
|
273
|
+
# Rails 6.0->6.1 optimizes for fewer "OR" parenthesis
|
|
274
|
+
fmt.gsub!(/\(\((.+ OR .+)\)( OR .+)\)/) { "(#{Regexp.last_match(1)}#{Regexp.last_match(2)})" }
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# basic formatting for easier git diffing
|
|
278
|
+
fmt.gsub(/ (SELECT|FROM|WHERE|GROUP BY|ORDER BY) /) { "\n #{Regexp.last_match(1).strip} " }
|
|
279
|
+
.gsub(/ (AND|OR) /) { "\n #{Regexp.last_match(1).strip} " }
|
|
280
|
+
# normalize and truncate 'AS' names/aliases (changes across Rails versions)
|
|
281
|
+
.gsub(/AS ("|`)?(\w+)("|`)?/) { "AS #{Regexp.last_match(2)[0...63]}" }
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def explain
|
|
285
|
+
send(:"#{current_adapter}_explain").strip
|
|
286
|
+
# normalize plan estimates
|
|
287
|
+
.gsub(/\(cost=.+\)/, '(cost=...)')
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
private
|
|
291
|
+
|
|
292
|
+
def postgresql_explain
|
|
293
|
+
connection.execute("SET seq_page_cost = 100")
|
|
294
|
+
connection.execute("SET enable_hashagg = off")
|
|
295
|
+
connection.execute("SET plan_cache_mode TO force_generic_plan")
|
|
296
|
+
connection.execute("EXPLAIN (VERBOSE) #{sql}").values.flatten.join("\n")
|
|
297
|
+
ensure
|
|
298
|
+
connection.execute("RESET plan_cache_mode")
|
|
299
|
+
connection.execute("RESET enable_hashagg")
|
|
300
|
+
connection.execute("RESET seq_page_cost")
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
def mysql2_explain
|
|
304
|
+
seed_rows! # MySQL needs a bit of data to reach for indexes
|
|
305
|
+
connection.execute("ANALYZE TABLE #{Delayed::Job.table_name}")
|
|
306
|
+
connection.execute("SET SESSION max_seeks_for_key = 1")
|
|
307
|
+
connection.execute("EXPLAIN FORMAT=TREE #{sql}").to_a.map(&:first).join("\n")
|
|
308
|
+
ensure
|
|
309
|
+
connection.execute("SET SESSION max_seeks_for_key = DEFAULT")
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def sqlite3_explain
|
|
313
|
+
connection.execute("EXPLAIN QUERY PLAN #{sql}").flat_map { |r| r["detail"] }.join("\n")
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def seed_rows!
|
|
317
|
+
now = Delayed::Job.db_time_now
|
|
318
|
+
10.times do
|
|
319
|
+
[true, false].repeated_combination(5).each_with_index do |(erroring, failed, locked, future), i|
|
|
320
|
+
Delayed::Job.create!(
|
|
321
|
+
run_at: now + (future ? i.minutes : -i.minutes),
|
|
322
|
+
queue: "queue_#{i}",
|
|
323
|
+
handler: "--- !ruby/object:SimpleJob\n",
|
|
324
|
+
attempts: erroring ? i : 0,
|
|
325
|
+
failed_at: failed ? now - i.minutes : nil,
|
|
326
|
+
locked_at: locked ? now - i.seconds : nil,
|
|
327
|
+
)
|
|
328
|
+
end
|
|
329
|
+
end
|
|
330
|
+
end
|
|
331
|
+
end
|
data/spec/sample_jobs.rb
CHANGED
|
@@ -115,3 +115,13 @@ end
|
|
|
115
115
|
class ActiveJobJob < ActiveJob::Base # rubocop:disable Rails/ApplicationJob
|
|
116
116
|
def perform(*args, **kwargs); end
|
|
117
117
|
end
|
|
118
|
+
|
|
119
|
+
class RescuesStandardErrorJob
|
|
120
|
+
cattr_accessor(:runs) { 0 }
|
|
121
|
+
def perform
|
|
122
|
+
self.class.runs += 1
|
|
123
|
+
sleep 10
|
|
124
|
+
rescue StandardError => e
|
|
125
|
+
raise "Rescued: #{e.class}"
|
|
126
|
+
end
|
|
127
|
+
end
|
data/spec/worker_spec.rb
CHANGED
|
@@ -221,6 +221,24 @@ describe Delayed::Worker do
|
|
|
221
221
|
end
|
|
222
222
|
end
|
|
223
223
|
|
|
224
|
+
describe '.max_run_time' do
|
|
225
|
+
before { described_class.max_run_time = 1 }
|
|
226
|
+
after { RescuesStandardErrorJob.runs = 0 }
|
|
227
|
+
|
|
228
|
+
it 'times out and raises a WorkerTimeout that bypasses any StandardError rescuing' do
|
|
229
|
+
Delayed::Job.enqueue RescuesStandardErrorJob.new
|
|
230
|
+
described_class.new.work_off
|
|
231
|
+
|
|
232
|
+
expect(Delayed::Job.count).to eq 1
|
|
233
|
+
expect(RescuesStandardErrorJob.runs).to eq 1
|
|
234
|
+
Delayed::Job.first.tap do |job|
|
|
235
|
+
expect(job.attempts).to eq 1
|
|
236
|
+
expect(job.last_error).to match(/execution expired/)
|
|
237
|
+
expect(job.last_error).to match(/Delayed::Worker.max_run_time is only 1 second/)
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
|
|
224
242
|
describe 'lifecycle callbacks' do
|
|
225
243
|
let(:plugin) do
|
|
226
244
|
Class.new(Delayed::Plugin) do
|