inst-jobs 3.0.3 → 3.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 11059ce779a0ff644edcae25335e23b7ad2a4448dfb48e8cbc95295f3e6c468c
|
|
4
|
+
data.tar.gz: 84dfa8a1185219823013e1363190b0469e397dd423105eb29a205fea1e7207fa
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9e4c5291673edccd5760cd11d087102e0d8829d25793a6ca4f9f3255336b3abb1ce2bcf7426a79a76d0f373b44ec2b0152c395974f78a4552942f7a858d9d499
|
|
7
|
+
data.tar.gz: 77aea9b18c3492e2b98f26a23c0591c8da9a31a34c0736868f00d54b3eadff77f7b2bbeb00b91b0819f93bffcefb7a26d1f2c829ebd82bc6b58a5c5a7bb01b16
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class UpdateAfterDeleteTriggerForSingletonTransitionCases < ActiveRecord::Migration[6.0]
|
|
4
|
+
def up
|
|
5
|
+
execute(<<~SQL)
|
|
6
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
|
7
|
+
DECLARE
|
|
8
|
+
next_strand varchar;
|
|
9
|
+
running_count integer;
|
|
10
|
+
should_lock boolean;
|
|
11
|
+
should_be_precise boolean;
|
|
12
|
+
update_query varchar;
|
|
13
|
+
skip_locked varchar;
|
|
14
|
+
transition boolean;
|
|
15
|
+
BEGIN
|
|
16
|
+
IF OLD.strand IS NOT NULL THEN
|
|
17
|
+
should_lock := true;
|
|
18
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
|
19
|
+
|
|
20
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
|
21
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
22
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
23
|
+
) subquery_for_count);
|
|
24
|
+
should_lock := running_count < OLD.max_concurrent;
|
|
25
|
+
END IF;
|
|
26
|
+
|
|
27
|
+
IF should_lock THEN
|
|
28
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
29
|
+
END IF;
|
|
30
|
+
|
|
31
|
+
-- note that we don't really care if the row we're deleting has a singleton, or if it even
|
|
32
|
+
-- matches the row(s) we're going to update. we just need to make sure that whatever
|
|
33
|
+
-- singleton we grab isn't already running (which is a simple existence check, since
|
|
34
|
+
-- the unique indexes ensure there is at most one singleton running, and one queued)
|
|
35
|
+
update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
|
|
36
|
+
SELECT id FROM delayed_jobs j2
|
|
37
|
+
WHERE next_in_strand=false AND
|
|
38
|
+
j2.strand=$1.strand AND
|
|
39
|
+
(j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
|
|
40
|
+
ORDER BY j2.strand_order_override ASC, j2.id ASC
|
|
41
|
+
LIMIT ';
|
|
42
|
+
|
|
43
|
+
IF should_be_precise THEN
|
|
44
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
45
|
+
SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
46
|
+
) s);
|
|
47
|
+
IF running_count < OLD.max_concurrent THEN
|
|
48
|
+
update_query := update_query || '($1.max_concurrent - $2)';
|
|
49
|
+
ELSE
|
|
50
|
+
-- we have too many running already; just bail
|
|
51
|
+
RETURN OLD;
|
|
52
|
+
END IF;
|
|
53
|
+
ELSE
|
|
54
|
+
update_query := update_query || '1';
|
|
55
|
+
|
|
56
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
|
57
|
+
IF OLD.max_concurrent > 1 THEN
|
|
58
|
+
skip_locked := ' SKIP LOCKED';
|
|
59
|
+
END IF;
|
|
60
|
+
END IF;
|
|
61
|
+
|
|
62
|
+
update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
|
|
63
|
+
EXECUTE update_query USING OLD, running_count;
|
|
64
|
+
END IF;
|
|
65
|
+
|
|
66
|
+
IF OLD.singleton IS NOT NULL THEN
|
|
67
|
+
transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
|
|
68
|
+
|
|
69
|
+
IF transition THEN
|
|
70
|
+
next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
|
|
71
|
+
|
|
72
|
+
IF next_strand IS NOT NULL THEN
|
|
73
|
+
-- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
|
|
74
|
+
IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
|
|
75
|
+
-- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
|
|
76
|
+
RETURN OLD;
|
|
77
|
+
END IF;
|
|
78
|
+
END IF;
|
|
79
|
+
ELSIF OLD.strand IS NOT NULL THEN
|
|
80
|
+
-- if there is no transition and there is a strand then we have already handled this singleton in the case above --
|
|
81
|
+
RETURN OLD;
|
|
82
|
+
END IF;
|
|
83
|
+
|
|
84
|
+
-- handles transitioning a singleton from stranded to not stranded --
|
|
85
|
+
-- handles transitioning a singleton from unstranded to stranded --
|
|
86
|
+
-- handles transitioning a singleton from strand A to strand B --
|
|
87
|
+
-- these transitions are a relatively rare case, so we take a shortcut and --
|
|
88
|
+
-- only start the next singleton if its strand does not currently have any running jobs --
|
|
89
|
+
-- if it does, the next stranded job that finishes will start this singleton if it can --
|
|
90
|
+
UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
|
|
91
|
+
SELECT id FROM delayed_jobs j2
|
|
92
|
+
WHERE next_in_strand=false AND
|
|
93
|
+
j2.singleton=OLD.singleton AND
|
|
94
|
+
j2.locked_by IS NULL AND
|
|
95
|
+
(j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
|
|
96
|
+
FOR UPDATE
|
|
97
|
+
);
|
|
98
|
+
END IF;
|
|
99
|
+
RETURN OLD;
|
|
100
|
+
END;
|
|
101
|
+
$$ LANGUAGE plpgsql;
|
|
102
|
+
SQL
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def down
|
|
106
|
+
execute(<<~SQL)
|
|
107
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
|
108
|
+
DECLARE
|
|
109
|
+
running_count integer;
|
|
110
|
+
should_lock boolean;
|
|
111
|
+
should_be_precise boolean;
|
|
112
|
+
update_query varchar;
|
|
113
|
+
skip_locked varchar;
|
|
114
|
+
BEGIN
|
|
115
|
+
IF OLD.strand IS NOT NULL THEN
|
|
116
|
+
should_lock := true;
|
|
117
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
|
118
|
+
|
|
119
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
|
120
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
121
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
122
|
+
) subquery_for_count);
|
|
123
|
+
should_lock := running_count < OLD.max_concurrent;
|
|
124
|
+
END IF;
|
|
125
|
+
|
|
126
|
+
IF should_lock THEN
|
|
127
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
128
|
+
END IF;
|
|
129
|
+
|
|
130
|
+
-- note that we don't really care if the row we're deleting has a singleton, or if it even
|
|
131
|
+
-- matches the row(s) we're going to update. we just need to make sure that whatever
|
|
132
|
+
-- singleton we grab isn't already running (which is a simple existence check, since
|
|
133
|
+
-- the unique indexes ensure there is at most one singleton running, and one queued)
|
|
134
|
+
update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
|
|
135
|
+
SELECT id FROM delayed_jobs j2
|
|
136
|
+
WHERE next_in_strand=false AND
|
|
137
|
+
j2.strand=$1.strand AND
|
|
138
|
+
(j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
|
|
139
|
+
ORDER BY j2.strand_order_override ASC, j2.id ASC
|
|
140
|
+
LIMIT ';
|
|
141
|
+
|
|
142
|
+
IF should_be_precise THEN
|
|
143
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
144
|
+
SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
145
|
+
) s);
|
|
146
|
+
IF running_count < OLD.max_concurrent THEN
|
|
147
|
+
update_query := update_query || '($1.max_concurrent - $2)';
|
|
148
|
+
ELSE
|
|
149
|
+
-- we have too many running already; just bail
|
|
150
|
+
RETURN OLD;
|
|
151
|
+
END IF;
|
|
152
|
+
ELSE
|
|
153
|
+
update_query := update_query || '1';
|
|
154
|
+
|
|
155
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
|
156
|
+
IF OLD.max_concurrent > 1 THEN
|
|
157
|
+
skip_locked := ' SKIP LOCKED';
|
|
158
|
+
END IF;
|
|
159
|
+
END IF;
|
|
160
|
+
|
|
161
|
+
update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
|
|
162
|
+
EXECUTE update_query USING OLD, running_count;
|
|
163
|
+
ELSIF OLD.singleton IS NOT NULL THEN
|
|
164
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
|
|
165
|
+
END IF;
|
|
166
|
+
RETURN OLD;
|
|
167
|
+
END;
|
|
168
|
+
$$ LANGUAGE plpgsql;
|
|
169
|
+
SQL
|
|
170
|
+
end
|
|
171
|
+
end
|
data/lib/delayed/version.rb
CHANGED
|
@@ -389,7 +389,153 @@ shared_examples_for "a backend" do
|
|
|
389
389
|
expect(job1.reload.handler).to include("ErrorJob")
|
|
390
390
|
end
|
|
391
391
|
|
|
392
|
+
context "next_in_strand management - deadlocks", non_transactional: true do
|
|
393
|
+
# The following unit tests are fairly slow and non-deterministic. It may be
|
|
394
|
+
# easier to make them fail quicker and more consistently by adding a random
|
|
395
|
+
# sleep into the appropriate trigger(s).
|
|
396
|
+
#
|
|
397
|
+
# Example:
|
|
398
|
+
# PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
399
|
+
# PERFORM pg_sleep(random() * 2);
|
|
400
|
+
|
|
401
|
+
def loop_secs(val)
|
|
402
|
+
loop_start = Time.now.utc
|
|
403
|
+
|
|
404
|
+
loop do
|
|
405
|
+
break if Time.now.utc >= loop_start + val
|
|
406
|
+
|
|
407
|
+
yield
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
it "doesn't deadlock when transitioning from strand_a to strand_b" do
|
|
412
|
+
threads = []
|
|
413
|
+
|
|
414
|
+
def thread_body(j1_params, j2_params)
|
|
415
|
+
loop do
|
|
416
|
+
j1 = create_job(**j1_params)
|
|
417
|
+
j2 = create_job(**j2_params)
|
|
418
|
+
|
|
419
|
+
expect(j1.reload.next_in_strand).to eq(true)
|
|
420
|
+
expect(j2.reload.next_in_strand).to eq(false)
|
|
421
|
+
|
|
422
|
+
j1.delete
|
|
423
|
+
|
|
424
|
+
# In case we couldn't acquire a lock, we actually need to wait for
|
|
425
|
+
# the other thread to set this to true.
|
|
426
|
+
loop_secs(10.seconds) do
|
|
427
|
+
break if j2.reload.next_in_strand
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
expect(j2.reload.next_in_strand).to eq(true)
|
|
431
|
+
|
|
432
|
+
j2.delete
|
|
433
|
+
end
|
|
434
|
+
rescue
|
|
435
|
+
Thread.current.thread_variable_set(:fail, true)
|
|
436
|
+
raise
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
threads << Thread.new do
|
|
440
|
+
thread_body(
|
|
441
|
+
{ singleton: "myjobs", strand: "myjobs2", locked_by: "w1" },
|
|
442
|
+
{ singleton: "myjobs", strand: "myjobs" }
|
|
443
|
+
)
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
threads << Thread.new do
|
|
447
|
+
thread_body(
|
|
448
|
+
{ singleton: "myjobs2", strand: "myjobs", locked_by: "w1" },
|
|
449
|
+
{ singleton: "myjobs2", strand: "myjobs2" }
|
|
450
|
+
)
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
begin
|
|
454
|
+
loop_secs(60.seconds) do
|
|
455
|
+
if threads.any? { |x| x.thread_variable_get(:fail) }
|
|
456
|
+
raise "at least one thread hit a deadlock or other error"
|
|
457
|
+
end
|
|
458
|
+
end
|
|
459
|
+
ensure
|
|
460
|
+
threads.each(&:kill)
|
|
461
|
+
threads.each(&:join)
|
|
462
|
+
end
|
|
463
|
+
end
|
|
464
|
+
end
|
|
465
|
+
|
|
392
466
|
context "next_in_strand management" do
|
|
467
|
+
it "handles transitions correctly when going from stranded to not stranded" do
|
|
468
|
+
@job1 = create_job(singleton: "myjobs", strand: "myjobs")
|
|
469
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
470
|
+
@job2 = create_job(singleton: "myjobs")
|
|
471
|
+
|
|
472
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
473
|
+
expect(@job2.reload.next_in_strand).to eq false
|
|
474
|
+
|
|
475
|
+
@job1.destroy
|
|
476
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
it "handles transitions correctly when going from not stranded to stranded" do
|
|
480
|
+
@job1 = create_job(singleton: "myjobs2", strand: "myjobs")
|
|
481
|
+
@job2 = create_job(singleton: "myjobs")
|
|
482
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
483
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
484
|
+
@job3 = create_job(singleton: "myjobs", strand: "myjobs2")
|
|
485
|
+
|
|
486
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
487
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
488
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
489
|
+
|
|
490
|
+
@job2.destroy
|
|
491
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
492
|
+
expect(@job3.reload.next_in_strand).to eq true
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
it "does not violate n_strand=1 constraints when going from not stranded to stranded" do
|
|
496
|
+
@job1 = create_job(singleton: "myjobs2", strand: "myjobs")
|
|
497
|
+
@job2 = create_job(singleton: "myjobs")
|
|
498
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
499
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
500
|
+
@job3 = create_job(singleton: "myjobs", strand: "myjobs")
|
|
501
|
+
|
|
502
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
503
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
504
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
505
|
+
|
|
506
|
+
@job2.destroy
|
|
507
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
508
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
509
|
+
end
|
|
510
|
+
|
|
511
|
+
it "handles transitions correctly when going from stranded to another strand" do
|
|
512
|
+
@job1 = create_job(singleton: "myjobs", strand: "myjobs")
|
|
513
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
514
|
+
@job2 = create_job(singleton: "myjobs", strand: "myjobs2")
|
|
515
|
+
|
|
516
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
517
|
+
expect(@job2.reload.next_in_strand).to eq false
|
|
518
|
+
|
|
519
|
+
@job1.destroy
|
|
520
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
it "does not violate n_strand=1 constraints when going from stranded to another strand" do
|
|
524
|
+
@job1 = create_job(singleton: "myjobs2", strand: "myjobs2")
|
|
525
|
+
@job2 = create_job(singleton: "myjobs", strand: "myjobs")
|
|
526
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
527
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
528
|
+
@job3 = create_job(singleton: "myjobs", strand: "myjobs2")
|
|
529
|
+
|
|
530
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
531
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
532
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
533
|
+
|
|
534
|
+
@job2.destroy
|
|
535
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
536
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
537
|
+
end
|
|
538
|
+
|
|
393
539
|
it "creates first as true, and second as false, then transitions to second when deleted" do
|
|
394
540
|
@job1 = create_job(singleton: "myjobs")
|
|
395
541
|
Delayed::Job.get_and_lock_next_available("w1")
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: inst-jobs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.0.
|
|
4
|
+
version: 3.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Cody Cutrer
|
|
@@ -10,7 +10,7 @@ authors:
|
|
|
10
10
|
autorequire:
|
|
11
11
|
bindir: exe
|
|
12
12
|
cert_chain: []
|
|
13
|
-
date: 2021-
|
|
13
|
+
date: 2021-12-09 00:00:00.000000000 Z
|
|
14
14
|
dependencies:
|
|
15
15
|
- !ruby/object:Gem::Dependency
|
|
16
16
|
name: activerecord
|
|
@@ -466,6 +466,7 @@ files:
|
|
|
466
466
|
- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
|
|
467
467
|
- db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
|
|
468
468
|
- db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
|
|
469
|
+
- db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
|
|
469
470
|
- exe/inst_jobs
|
|
470
471
|
- lib/delayed/backend/active_record.rb
|
|
471
472
|
- lib/delayed/backend/base.rb
|
|
@@ -547,7 +548,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
547
548
|
- !ruby/object:Gem::Version
|
|
548
549
|
version: '0'
|
|
549
550
|
requirements: []
|
|
550
|
-
rubygems_version: 3.
|
|
551
|
+
rubygems_version: 3.1.4
|
|
551
552
|
signing_key:
|
|
552
553
|
specification_version: 4
|
|
553
554
|
summary: Instructure-maintained fork of delayed_job
|