inst-jobs 3.0.3 → 3.0.5
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 11059ce779a0ff644edcae25335e23b7ad2a4448dfb48e8cbc95295f3e6c468c
|
|
4
|
+
data.tar.gz: 84dfa8a1185219823013e1363190b0469e397dd423105eb29a205fea1e7207fa
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9e4c5291673edccd5760cd11d087102e0d8829d25793a6ca4f9f3255336b3abb1ce2bcf7426a79a76d0f373b44ec2b0152c395974f78a4552942f7a858d9d499
|
|
7
|
+
data.tar.gz: 77aea9b18c3492e2b98f26a23c0591c8da9a31a34c0736868f00d54b3eadff77f7b2bbeb00b91b0819f93bffcefb7a26d1f2c829ebd82bc6b58a5c5a7bb01b16
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class UpdateAfterDeleteTriggerForSingletonTransitionCases < ActiveRecord::Migration[6.0]
|
|
4
|
+
def up
|
|
5
|
+
execute(<<~SQL)
|
|
6
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
|
7
|
+
DECLARE
|
|
8
|
+
next_strand varchar;
|
|
9
|
+
running_count integer;
|
|
10
|
+
should_lock boolean;
|
|
11
|
+
should_be_precise boolean;
|
|
12
|
+
update_query varchar;
|
|
13
|
+
skip_locked varchar;
|
|
14
|
+
transition boolean;
|
|
15
|
+
BEGIN
|
|
16
|
+
IF OLD.strand IS NOT NULL THEN
|
|
17
|
+
should_lock := true;
|
|
18
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
|
19
|
+
|
|
20
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
|
21
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
22
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
23
|
+
) subquery_for_count);
|
|
24
|
+
should_lock := running_count < OLD.max_concurrent;
|
|
25
|
+
END IF;
|
|
26
|
+
|
|
27
|
+
IF should_lock THEN
|
|
28
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
29
|
+
END IF;
|
|
30
|
+
|
|
31
|
+
-- note that we don't really care if the row we're deleting has a singleton, or if it even
|
|
32
|
+
-- matches the row(s) we're going to update. we just need to make sure that whatever
|
|
33
|
+
-- singleton we grab isn't already running (which is a simple existence check, since
|
|
34
|
+
-- the unique indexes ensure there is at most one singleton running, and one queued)
|
|
35
|
+
update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
|
|
36
|
+
SELECT id FROM delayed_jobs j2
|
|
37
|
+
WHERE next_in_strand=false AND
|
|
38
|
+
j2.strand=$1.strand AND
|
|
39
|
+
(j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
|
|
40
|
+
ORDER BY j2.strand_order_override ASC, j2.id ASC
|
|
41
|
+
LIMIT ';
|
|
42
|
+
|
|
43
|
+
IF should_be_precise THEN
|
|
44
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
45
|
+
SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
46
|
+
) s);
|
|
47
|
+
IF running_count < OLD.max_concurrent THEN
|
|
48
|
+
update_query := update_query || '($1.max_concurrent - $2)';
|
|
49
|
+
ELSE
|
|
50
|
+
-- we have too many running already; just bail
|
|
51
|
+
RETURN OLD;
|
|
52
|
+
END IF;
|
|
53
|
+
ELSE
|
|
54
|
+
update_query := update_query || '1';
|
|
55
|
+
|
|
56
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
|
57
|
+
IF OLD.max_concurrent > 1 THEN
|
|
58
|
+
skip_locked := ' SKIP LOCKED';
|
|
59
|
+
END IF;
|
|
60
|
+
END IF;
|
|
61
|
+
|
|
62
|
+
update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
|
|
63
|
+
EXECUTE update_query USING OLD, running_count;
|
|
64
|
+
END IF;
|
|
65
|
+
|
|
66
|
+
IF OLD.singleton IS NOT NULL THEN
|
|
67
|
+
transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
|
|
68
|
+
|
|
69
|
+
IF transition THEN
|
|
70
|
+
next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
|
|
71
|
+
|
|
72
|
+
IF next_strand IS NOT NULL THEN
|
|
73
|
+
-- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
|
|
74
|
+
IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
|
|
75
|
+
-- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
|
|
76
|
+
RETURN OLD;
|
|
77
|
+
END IF;
|
|
78
|
+
END IF;
|
|
79
|
+
ELSIF OLD.strand IS NOT NULL THEN
|
|
80
|
+
-- if there is no transition and there is a strand then we have already handled this singleton in the case above --
|
|
81
|
+
RETURN OLD;
|
|
82
|
+
END IF;
|
|
83
|
+
|
|
84
|
+
-- handles transitioning a singleton from stranded to not stranded --
|
|
85
|
+
-- handles transitioning a singleton from unstranded to stranded --
|
|
86
|
+
-- handles transitioning a singleton from strand A to strand B --
|
|
87
|
+
-- these transitions are a relatively rare case, so we take a shortcut and --
|
|
88
|
+
-- only start the next singleton if its strand does not currently have any running jobs --
|
|
89
|
+
-- if it does, the next stranded job that finishes will start this singleton if it can --
|
|
90
|
+
UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
|
|
91
|
+
SELECT id FROM delayed_jobs j2
|
|
92
|
+
WHERE next_in_strand=false AND
|
|
93
|
+
j2.singleton=OLD.singleton AND
|
|
94
|
+
j2.locked_by IS NULL AND
|
|
95
|
+
(j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
|
|
96
|
+
FOR UPDATE
|
|
97
|
+
);
|
|
98
|
+
END IF;
|
|
99
|
+
RETURN OLD;
|
|
100
|
+
END;
|
|
101
|
+
$$ LANGUAGE plpgsql;
|
|
102
|
+
SQL
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def down
|
|
106
|
+
execute(<<~SQL)
|
|
107
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
|
108
|
+
DECLARE
|
|
109
|
+
running_count integer;
|
|
110
|
+
should_lock boolean;
|
|
111
|
+
should_be_precise boolean;
|
|
112
|
+
update_query varchar;
|
|
113
|
+
skip_locked varchar;
|
|
114
|
+
BEGIN
|
|
115
|
+
IF OLD.strand IS NOT NULL THEN
|
|
116
|
+
should_lock := true;
|
|
117
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
|
118
|
+
|
|
119
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
|
120
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
121
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
122
|
+
) subquery_for_count);
|
|
123
|
+
should_lock := running_count < OLD.max_concurrent;
|
|
124
|
+
END IF;
|
|
125
|
+
|
|
126
|
+
IF should_lock THEN
|
|
127
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
128
|
+
END IF;
|
|
129
|
+
|
|
130
|
+
-- note that we don't really care if the row we're deleting has a singleton, or if it even
|
|
131
|
+
-- matches the row(s) we're going to update. we just need to make sure that whatever
|
|
132
|
+
-- singleton we grab isn't already running (which is a simple existence check, since
|
|
133
|
+
-- the unique indexes ensure there is at most one singleton running, and one queued)
|
|
134
|
+
update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
|
|
135
|
+
SELECT id FROM delayed_jobs j2
|
|
136
|
+
WHERE next_in_strand=false AND
|
|
137
|
+
j2.strand=$1.strand AND
|
|
138
|
+
(j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
|
|
139
|
+
ORDER BY j2.strand_order_override ASC, j2.id ASC
|
|
140
|
+
LIMIT ';
|
|
141
|
+
|
|
142
|
+
IF should_be_precise THEN
|
|
143
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
144
|
+
SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
145
|
+
) s);
|
|
146
|
+
IF running_count < OLD.max_concurrent THEN
|
|
147
|
+
update_query := update_query || '($1.max_concurrent - $2)';
|
|
148
|
+
ELSE
|
|
149
|
+
-- we have too many running already; just bail
|
|
150
|
+
RETURN OLD;
|
|
151
|
+
END IF;
|
|
152
|
+
ELSE
|
|
153
|
+
update_query := update_query || '1';
|
|
154
|
+
|
|
155
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
|
156
|
+
IF OLD.max_concurrent > 1 THEN
|
|
157
|
+
skip_locked := ' SKIP LOCKED';
|
|
158
|
+
END IF;
|
|
159
|
+
END IF;
|
|
160
|
+
|
|
161
|
+
update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
|
|
162
|
+
EXECUTE update_query USING OLD, running_count;
|
|
163
|
+
ELSIF OLD.singleton IS NOT NULL THEN
|
|
164
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
|
|
165
|
+
END IF;
|
|
166
|
+
RETURN OLD;
|
|
167
|
+
END;
|
|
168
|
+
$$ LANGUAGE plpgsql;
|
|
169
|
+
SQL
|
|
170
|
+
end
|
|
171
|
+
end
|
data/lib/delayed/version.rb
CHANGED
|
@@ -389,7 +389,153 @@ shared_examples_for "a backend" do
|
|
|
389
389
|
expect(job1.reload.handler).to include("ErrorJob")
|
|
390
390
|
end
|
|
391
391
|
|
|
392
|
+
context "next_in_strand management - deadlocks", non_transactional: true do
|
|
393
|
+
# The following unit tests are fairly slow and non-deterministic. It may be
|
|
394
|
+
# easier to make them fail quicker and more consistently by adding a random
|
|
395
|
+
# sleep into the appropriate trigger(s).
|
|
396
|
+
#
|
|
397
|
+
# Example:
|
|
398
|
+
# PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
399
|
+
# PERFORM pg_sleep(random() * 2);
|
|
400
|
+
|
|
401
|
+
def loop_secs(val)
|
|
402
|
+
loop_start = Time.now.utc
|
|
403
|
+
|
|
404
|
+
loop do
|
|
405
|
+
break if Time.now.utc >= loop_start + val
|
|
406
|
+
|
|
407
|
+
yield
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
it "doesn't deadlock when transitioning from strand_a to strand_b" do
|
|
412
|
+
threads = []
|
|
413
|
+
|
|
414
|
+
def thread_body(j1_params, j2_params)
|
|
415
|
+
loop do
|
|
416
|
+
j1 = create_job(**j1_params)
|
|
417
|
+
j2 = create_job(**j2_params)
|
|
418
|
+
|
|
419
|
+
expect(j1.reload.next_in_strand).to eq(true)
|
|
420
|
+
expect(j2.reload.next_in_strand).to eq(false)
|
|
421
|
+
|
|
422
|
+
j1.delete
|
|
423
|
+
|
|
424
|
+
# In case we couldn't acquire a lock, we actually need to wait for
|
|
425
|
+
# the other thread to set this to true.
|
|
426
|
+
loop_secs(10.seconds) do
|
|
427
|
+
break if j2.reload.next_in_strand
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
expect(j2.reload.next_in_strand).to eq(true)
|
|
431
|
+
|
|
432
|
+
j2.delete
|
|
433
|
+
end
|
|
434
|
+
rescue
|
|
435
|
+
Thread.current.thread_variable_set(:fail, true)
|
|
436
|
+
raise
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
threads << Thread.new do
|
|
440
|
+
thread_body(
|
|
441
|
+
{ singleton: "myjobs", strand: "myjobs2", locked_by: "w1" },
|
|
442
|
+
{ singleton: "myjobs", strand: "myjobs" }
|
|
443
|
+
)
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
threads << Thread.new do
|
|
447
|
+
thread_body(
|
|
448
|
+
{ singleton: "myjobs2", strand: "myjobs", locked_by: "w1" },
|
|
449
|
+
{ singleton: "myjobs2", strand: "myjobs2" }
|
|
450
|
+
)
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
begin
|
|
454
|
+
loop_secs(60.seconds) do
|
|
455
|
+
if threads.any? { |x| x.thread_variable_get(:fail) }
|
|
456
|
+
raise "at least one thread hit a deadlock or other error"
|
|
457
|
+
end
|
|
458
|
+
end
|
|
459
|
+
ensure
|
|
460
|
+
threads.each(&:kill)
|
|
461
|
+
threads.each(&:join)
|
|
462
|
+
end
|
|
463
|
+
end
|
|
464
|
+
end
|
|
465
|
+
|
|
392
466
|
context "next_in_strand management" do
|
|
467
|
+
it "handles transitions correctly when going from stranded to not stranded" do
|
|
468
|
+
@job1 = create_job(singleton: "myjobs", strand: "myjobs")
|
|
469
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
470
|
+
@job2 = create_job(singleton: "myjobs")
|
|
471
|
+
|
|
472
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
473
|
+
expect(@job2.reload.next_in_strand).to eq false
|
|
474
|
+
|
|
475
|
+
@job1.destroy
|
|
476
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
it "handles transitions correctly when going from not stranded to stranded" do
|
|
480
|
+
@job1 = create_job(singleton: "myjobs2", strand: "myjobs")
|
|
481
|
+
@job2 = create_job(singleton: "myjobs")
|
|
482
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
483
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
484
|
+
@job3 = create_job(singleton: "myjobs", strand: "myjobs2")
|
|
485
|
+
|
|
486
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
487
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
488
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
489
|
+
|
|
490
|
+
@job2.destroy
|
|
491
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
492
|
+
expect(@job3.reload.next_in_strand).to eq true
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
it "does not violate n_strand=1 constraints when going from not stranded to stranded" do
|
|
496
|
+
@job1 = create_job(singleton: "myjobs2", strand: "myjobs")
|
|
497
|
+
@job2 = create_job(singleton: "myjobs")
|
|
498
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
499
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
500
|
+
@job3 = create_job(singleton: "myjobs", strand: "myjobs")
|
|
501
|
+
|
|
502
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
503
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
504
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
505
|
+
|
|
506
|
+
@job2.destroy
|
|
507
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
508
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
509
|
+
end
|
|
510
|
+
|
|
511
|
+
it "handles transitions correctly when going from stranded to another strand" do
|
|
512
|
+
@job1 = create_job(singleton: "myjobs", strand: "myjobs")
|
|
513
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
514
|
+
@job2 = create_job(singleton: "myjobs", strand: "myjobs2")
|
|
515
|
+
|
|
516
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
517
|
+
expect(@job2.reload.next_in_strand).to eq false
|
|
518
|
+
|
|
519
|
+
@job1.destroy
|
|
520
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
it "does not violate n_strand=1 constraints when going from stranded to another strand" do
|
|
524
|
+
@job1 = create_job(singleton: "myjobs2", strand: "myjobs2")
|
|
525
|
+
@job2 = create_job(singleton: "myjobs", strand: "myjobs")
|
|
526
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
527
|
+
Delayed::Job.get_and_lock_next_available("w1")
|
|
528
|
+
@job3 = create_job(singleton: "myjobs", strand: "myjobs2")
|
|
529
|
+
|
|
530
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
531
|
+
expect(@job2.reload.next_in_strand).to eq true
|
|
532
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
533
|
+
|
|
534
|
+
@job2.destroy
|
|
535
|
+
expect(@job1.reload.next_in_strand).to eq true
|
|
536
|
+
expect(@job3.reload.next_in_strand).to eq false
|
|
537
|
+
end
|
|
538
|
+
|
|
393
539
|
it "creates first as true, and second as false, then transitions to second when deleted" do
|
|
394
540
|
@job1 = create_job(singleton: "myjobs")
|
|
395
541
|
Delayed::Job.get_and_lock_next_available("w1")
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: inst-jobs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.0.
|
|
4
|
+
version: 3.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Cody Cutrer
|
|
@@ -10,7 +10,7 @@ authors:
|
|
|
10
10
|
autorequire:
|
|
11
11
|
bindir: exe
|
|
12
12
|
cert_chain: []
|
|
13
|
-
date: 2021-
|
|
13
|
+
date: 2021-12-09 00:00:00.000000000 Z
|
|
14
14
|
dependencies:
|
|
15
15
|
- !ruby/object:Gem::Dependency
|
|
16
16
|
name: activerecord
|
|
@@ -466,6 +466,7 @@ files:
|
|
|
466
466
|
- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
|
|
467
467
|
- db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
|
|
468
468
|
- db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
|
|
469
|
+
- db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
|
|
469
470
|
- exe/inst_jobs
|
|
470
471
|
- lib/delayed/backend/active_record.rb
|
|
471
472
|
- lib/delayed/backend/base.rb
|
|
@@ -547,7 +548,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
547
548
|
- !ruby/object:Gem::Version
|
|
548
549
|
version: '0'
|
|
549
550
|
requirements: []
|
|
550
|
-
rubygems_version: 3.
|
|
551
|
+
rubygems_version: 3.1.4
|
|
551
552
|
signing_key:
|
|
552
553
|
specification_version: 4
|
|
553
554
|
summary: Instructure-maintained fork of delayed_job
|