inst-jobs 3.0.3 → 3.0.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fb38d77d225501a3f9ca85561266adf7f2b873d187d0ee9e820e30f773fc5846
4
- data.tar.gz: 0bdbb7e6609d0c228de6906b68ff41b7a5218b89832dd9f8958ed261d7a39fd2
3
+ metadata.gz: 11059ce779a0ff644edcae25335e23b7ad2a4448dfb48e8cbc95295f3e6c468c
4
+ data.tar.gz: 84dfa8a1185219823013e1363190b0469e397dd423105eb29a205fea1e7207fa
5
5
  SHA512:
6
- metadata.gz: 76c59051987d523e8465c98a0aa03edfa7ee6df1fc090ebeb8238f827d3f5487dccd61f92c20acf87df9b53153de03bb866d577f44eb3d0db18bd3cd9fa44f66
7
- data.tar.gz: 0262e34514a1919ff9715b1707d7c99dab2ddf4f3ef0c98e06441b61709ddb2bb36c763b73cc5a5512a03e2b2a253b507818da313213a621c40879d086fa9333
6
+ metadata.gz: 9e4c5291673edccd5760cd11d087102e0d8829d25793a6ca4f9f3255336b3abb1ce2bcf7426a79a76d0f373b44ec2b0152c395974f78a4552942f7a858d9d499
7
+ data.tar.gz: 77aea9b18c3492e2b98f26a23c0591c8da9a31a34c0736868f00d54b3eadff77f7b2bbeb00b91b0819f93bffcefb7a26d1f2c829ebd82bc6b58a5c5a7bb01b16
@@ -0,0 +1,171 @@
1
+ # frozen_string_literal: true
2
+
3
+ class UpdateAfterDeleteTriggerForSingletonTransitionCases < ActiveRecord::Migration[6.0]
4
+ def up
5
+ execute(<<~SQL)
6
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
7
+ DECLARE
8
+ next_strand varchar;
9
+ running_count integer;
10
+ should_lock boolean;
11
+ should_be_precise boolean;
12
+ update_query varchar;
13
+ skip_locked varchar;
14
+ transition boolean;
15
+ BEGIN
16
+ IF OLD.strand IS NOT NULL THEN
17
+ should_lock := true;
18
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
19
+
20
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
21
+ running_count := (SELECT COUNT(*) FROM (
22
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
23
+ ) subquery_for_count);
24
+ should_lock := running_count < OLD.max_concurrent;
25
+ END IF;
26
+
27
+ IF should_lock THEN
28
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
29
+ END IF;
30
+
31
+ -- note that we don't really care if the row we're deleting has a singleton, or if it even
32
+ -- matches the row(s) we're going to update. we just need to make sure that whatever
33
+ -- singleton we grab isn't already running (which is a simple existence check, since
34
+ -- the unique indexes ensure there is at most one singleton running, and one queued)
35
+ update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
36
+ SELECT id FROM delayed_jobs j2
37
+ WHERE next_in_strand=false AND
38
+ j2.strand=$1.strand AND
39
+ (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
40
+ ORDER BY j2.strand_order_override ASC, j2.id ASC
41
+ LIMIT ';
42
+
43
+ IF should_be_precise THEN
44
+ running_count := (SELECT COUNT(*) FROM (
45
+ SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
46
+ ) s);
47
+ IF running_count < OLD.max_concurrent THEN
48
+ update_query := update_query || '($1.max_concurrent - $2)';
49
+ ELSE
50
+ -- we have too many running already; just bail
51
+ RETURN OLD;
52
+ END IF;
53
+ ELSE
54
+ update_query := update_query || '1';
55
+
56
+ -- n-strands don't require precise ordering; we can make this query more performant
57
+ IF OLD.max_concurrent > 1 THEN
58
+ skip_locked := ' SKIP LOCKED';
59
+ END IF;
60
+ END IF;
61
+
62
+ update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
63
+ EXECUTE update_query USING OLD, running_count;
64
+ END IF;
65
+
66
+ IF OLD.singleton IS NOT NULL THEN
67
+ transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
68
+
69
+ IF transition THEN
70
+ next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
71
+
72
+ IF next_strand IS NOT NULL THEN
73
+ -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
74
+ IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
75
+ -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
76
+ RETURN OLD;
77
+ END IF;
78
+ END IF;
79
+ ELSIF OLD.strand IS NOT NULL THEN
80
+ -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
81
+ RETURN OLD;
82
+ END IF;
83
+
84
+ -- handles transitioning a singleton from stranded to not stranded --
85
+ -- handles transitioning a singleton from unstranded to stranded --
86
+ -- handles transitioning a singleton from strand A to strand B --
87
+ -- these transitions are a relatively rare case, so we take a shortcut and --
88
+ -- only start the next singleton if its strand does not currently have any running jobs --
89
+ -- if it does, the next stranded job that finishes will start this singleton if it can --
90
+ UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
91
+ SELECT id FROM delayed_jobs j2
92
+ WHERE next_in_strand=false AND
93
+ j2.singleton=OLD.singleton AND
94
+ j2.locked_by IS NULL AND
95
+ (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
96
+ FOR UPDATE
97
+ );
98
+ END IF;
99
+ RETURN OLD;
100
+ END;
101
+ $$ LANGUAGE plpgsql;
102
+ SQL
103
+ end
104
+
105
+ def down
106
+ execute(<<~SQL)
107
+ CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
108
+ DECLARE
109
+ running_count integer;
110
+ should_lock boolean;
111
+ should_be_precise boolean;
112
+ update_query varchar;
113
+ skip_locked varchar;
114
+ BEGIN
115
+ IF OLD.strand IS NOT NULL THEN
116
+ should_lock := true;
117
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
118
+
119
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
120
+ running_count := (SELECT COUNT(*) FROM (
121
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
122
+ ) subquery_for_count);
123
+ should_lock := running_count < OLD.max_concurrent;
124
+ END IF;
125
+
126
+ IF should_lock THEN
127
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
128
+ END IF;
129
+
130
+ -- note that we don't really care if the row we're deleting has a singleton, or if it even
131
+ -- matches the row(s) we're going to update. we just need to make sure that whatever
132
+ -- singleton we grab isn't already running (which is a simple existence check, since
133
+ -- the unique indexes ensure there is at most one singleton running, and one queued)
134
+ update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
135
+ SELECT id FROM delayed_jobs j2
136
+ WHERE next_in_strand=false AND
137
+ j2.strand=$1.strand AND
138
+ (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
139
+ ORDER BY j2.strand_order_override ASC, j2.id ASC
140
+ LIMIT ';
141
+
142
+ IF should_be_precise THEN
143
+ running_count := (SELECT COUNT(*) FROM (
144
+ SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
145
+ ) s);
146
+ IF running_count < OLD.max_concurrent THEN
147
+ update_query := update_query || '($1.max_concurrent - $2)';
148
+ ELSE
149
+ -- we have too many running already; just bail
150
+ RETURN OLD;
151
+ END IF;
152
+ ELSE
153
+ update_query := update_query || '1';
154
+
155
+ -- n-strands don't require precise ordering; we can make this query more performant
156
+ IF OLD.max_concurrent > 1 THEN
157
+ skip_locked := ' SKIP LOCKED';
158
+ END IF;
159
+ END IF;
160
+
161
+ update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
162
+ EXECUTE update_query USING OLD, running_count;
163
+ ELSIF OLD.singleton IS NOT NULL THEN
164
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
165
+ END IF;
166
+ RETURN OLD;
167
+ END;
168
+ $$ LANGUAGE plpgsql;
169
+ SQL
170
+ end
171
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Delayed
4
- VERSION = "3.0.3"
4
+ VERSION = "3.0.5"
5
5
  end
@@ -389,7 +389,153 @@ shared_examples_for "a backend" do
389
389
  expect(job1.reload.handler).to include("ErrorJob")
390
390
  end
391
391
 
392
+ context "next_in_strand management - deadlocks", non_transactional: true do
393
+ # The following unit tests are fairly slow and non-deterministic. It may be
394
+ # easier to make them fail quicker and more consistently by adding a random
395
+ # sleep into the appropriate trigger(s).
396
+ #
397
+ # Example:
398
+ # PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
399
+ # PERFORM pg_sleep(random() * 2);
400
+
401
+ def loop_secs(val)
402
+ loop_start = Time.now.utc
403
+
404
+ loop do
405
+ break if Time.now.utc >= loop_start + val
406
+
407
+ yield
408
+ end
409
+ end
410
+
411
+ it "doesn't deadlock when transitioning from strand_a to strand_b" do
412
+ threads = []
413
+
414
+ def thread_body(j1_params, j2_params)
415
+ loop do
416
+ j1 = create_job(**j1_params)
417
+ j2 = create_job(**j2_params)
418
+
419
+ expect(j1.reload.next_in_strand).to eq(true)
420
+ expect(j2.reload.next_in_strand).to eq(false)
421
+
422
+ j1.delete
423
+
424
+ # In case we couldn't acquire a lock, we actually need to wait for
425
+ # the other thread to set this to true.
426
+ loop_secs(10.seconds) do
427
+ break if j2.reload.next_in_strand
428
+ end
429
+
430
+ expect(j2.reload.next_in_strand).to eq(true)
431
+
432
+ j2.delete
433
+ end
434
+ rescue
435
+ Thread.current.thread_variable_set(:fail, true)
436
+ raise
437
+ end
438
+
439
+ threads << Thread.new do
440
+ thread_body(
441
+ { singleton: "myjobs", strand: "myjobs2", locked_by: "w1" },
442
+ { singleton: "myjobs", strand: "myjobs" }
443
+ )
444
+ end
445
+
446
+ threads << Thread.new do
447
+ thread_body(
448
+ { singleton: "myjobs2", strand: "myjobs", locked_by: "w1" },
449
+ { singleton: "myjobs2", strand: "myjobs2" }
450
+ )
451
+ end
452
+
453
+ begin
454
+ loop_secs(60.seconds) do
455
+ if threads.any? { |x| x.thread_variable_get(:fail) }
456
+ raise "at least one thread hit a deadlock or other error"
457
+ end
458
+ end
459
+ ensure
460
+ threads.each(&:kill)
461
+ threads.each(&:join)
462
+ end
463
+ end
464
+ end
465
+
392
466
  context "next_in_strand management" do
467
+ it "handles transitions correctly when going from stranded to not stranded" do
468
+ @job1 = create_job(singleton: "myjobs", strand: "myjobs")
469
+ Delayed::Job.get_and_lock_next_available("w1")
470
+ @job2 = create_job(singleton: "myjobs")
471
+
472
+ expect(@job1.reload.next_in_strand).to eq true
473
+ expect(@job2.reload.next_in_strand).to eq false
474
+
475
+ @job1.destroy
476
+ expect(@job2.reload.next_in_strand).to eq true
477
+ end
478
+
479
+ it "handles transitions correctly when going from not stranded to stranded" do
480
+ @job1 = create_job(singleton: "myjobs2", strand: "myjobs")
481
+ @job2 = create_job(singleton: "myjobs")
482
+ Delayed::Job.get_and_lock_next_available("w1")
483
+ Delayed::Job.get_and_lock_next_available("w1")
484
+ @job3 = create_job(singleton: "myjobs", strand: "myjobs2")
485
+
486
+ expect(@job1.reload.next_in_strand).to eq true
487
+ expect(@job2.reload.next_in_strand).to eq true
488
+ expect(@job3.reload.next_in_strand).to eq false
489
+
490
+ @job2.destroy
491
+ expect(@job1.reload.next_in_strand).to eq true
492
+ expect(@job3.reload.next_in_strand).to eq true
493
+ end
494
+
495
+ it "does not violate n_strand=1 constraints when going from not stranded to stranded" do
496
+ @job1 = create_job(singleton: "myjobs2", strand: "myjobs")
497
+ @job2 = create_job(singleton: "myjobs")
498
+ Delayed::Job.get_and_lock_next_available("w1")
499
+ Delayed::Job.get_and_lock_next_available("w1")
500
+ @job3 = create_job(singleton: "myjobs", strand: "myjobs")
501
+
502
+ expect(@job1.reload.next_in_strand).to eq true
503
+ expect(@job2.reload.next_in_strand).to eq true
504
+ expect(@job3.reload.next_in_strand).to eq false
505
+
506
+ @job2.destroy
507
+ expect(@job1.reload.next_in_strand).to eq true
508
+ expect(@job3.reload.next_in_strand).to eq false
509
+ end
510
+
511
+ it "handles transitions correctly when going from stranded to another strand" do
512
+ @job1 = create_job(singleton: "myjobs", strand: "myjobs")
513
+ Delayed::Job.get_and_lock_next_available("w1")
514
+ @job2 = create_job(singleton: "myjobs", strand: "myjobs2")
515
+
516
+ expect(@job1.reload.next_in_strand).to eq true
517
+ expect(@job2.reload.next_in_strand).to eq false
518
+
519
+ @job1.destroy
520
+ expect(@job2.reload.next_in_strand).to eq true
521
+ end
522
+
523
+ it "does not violate n_strand=1 constraints when going from stranded to another strand" do
524
+ @job1 = create_job(singleton: "myjobs2", strand: "myjobs2")
525
+ @job2 = create_job(singleton: "myjobs", strand: "myjobs")
526
+ Delayed::Job.get_and_lock_next_available("w1")
527
+ Delayed::Job.get_and_lock_next_available("w1")
528
+ @job3 = create_job(singleton: "myjobs", strand: "myjobs2")
529
+
530
+ expect(@job1.reload.next_in_strand).to eq true
531
+ expect(@job2.reload.next_in_strand).to eq true
532
+ expect(@job3.reload.next_in_strand).to eq false
533
+
534
+ @job2.destroy
535
+ expect(@job1.reload.next_in_strand).to eq true
536
+ expect(@job3.reload.next_in_strand).to eq false
537
+ end
538
+
393
539
  it "creates first as true, and second as false, then transitions to second when deleted" do
394
540
  @job1 = create_job(singleton: "myjobs")
395
541
  Delayed::Job.get_and_lock_next_available("w1")
data/spec/spec_helper.rb CHANGED
@@ -54,7 +54,8 @@ connection_config = {
54
54
  host: ENV["TEST_DB_HOST"].presence,
55
55
  encoding: "utf8",
56
56
  username: ENV["TEST_DB_USERNAME"],
57
- database: ENV["TEST_DB_DATABASE"]
57
+ database: ENV["TEST_DB_DATABASE"],
58
+ min_messages: "notice"
58
59
  }
59
60
 
60
61
  def migrate(file)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: inst-jobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.3
4
+ version: 3.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2021-11-30 00:00:00.000000000 Z
13
+ date: 2021-12-09 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: activerecord
@@ -466,6 +466,7 @@ files:
466
466
  - db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
467
467
  - db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
468
468
  - db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
469
+ - db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
469
470
  - exe/inst_jobs
470
471
  - lib/delayed/backend/active_record.rb
471
472
  - lib/delayed/backend/base.rb
@@ -547,7 +548,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
547
548
  - !ruby/object:Gem::Version
548
549
  version: '0'
549
550
  requirements: []
550
- rubygems_version: 3.2.15
551
+ rubygems_version: 3.1.4
551
552
  signing_key:
552
553
  specification_version: 4
553
554
  summary: Instructure-maintained fork of delayed_job