inst-jobs 0.15.22 → 0.16.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d1547b62f3c5fd86703cc209ec4b0feb6954db3c6fb4b58f4bcdbcf58dc9a9eb
|
|
4
|
+
data.tar.gz: 95ad1cacd71ac4511ee29eb0d3fda780037768234c232302b6fc10084de9e351
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bd27fc7e6c0ca5a613ba98ea0928d7947f2344d05c57ab14f588ff615c69bab60a0b6f08e60d1b610b6746ad8a467d151237f1469129f6144d5e93ce20ea71ef
|
|
7
|
+
data.tar.gz: 25ce44b9f6eeedae9605c4bc2077a9e9b51a4b7df94ba4d68be1e78f70801af65273c332caba780c11bf523cff31daed9c44d8f068f697a8910781b09a2daf88
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
|
|
2
|
+
def connection
|
|
3
|
+
Delayed::Job.connection
|
|
4
|
+
end
|
|
5
|
+
|
|
6
|
+
def up
|
|
7
|
+
if connection.adapter_name == 'PostgreSQL'
|
|
8
|
+
# tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
|
|
9
|
+
# don't get the count every single time - it's usually safe to just set the next one in line
|
|
10
|
+
# since the max_concurrent doesn't change all that often for a strand
|
|
11
|
+
execute(<<-SQL)
|
|
12
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
|
13
|
+
DECLARE
|
|
14
|
+
running_count integer;
|
|
15
|
+
should_lock boolean;
|
|
16
|
+
should_be_precise boolean;
|
|
17
|
+
BEGIN
|
|
18
|
+
IF OLD.strand IS NOT NULL THEN
|
|
19
|
+
should_lock := true;
|
|
20
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
|
21
|
+
|
|
22
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
|
23
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
24
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
25
|
+
) subquery_for_count);
|
|
26
|
+
should_lock := running_count < OLD.max_concurrent;
|
|
27
|
+
END IF;
|
|
28
|
+
|
|
29
|
+
IF should_lock THEN
|
|
30
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
31
|
+
END IF;
|
|
32
|
+
|
|
33
|
+
IF should_be_precise THEN
|
|
34
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
35
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
36
|
+
) subquery_for_count);
|
|
37
|
+
IF running_count < OLD.max_concurrent THEN
|
|
38
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
|
39
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
40
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
|
41
|
+
);
|
|
42
|
+
END IF;
|
|
43
|
+
ELSE
|
|
44
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
|
45
|
+
IF OLD.max_concurrent > 1 THEN
|
|
46
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
|
47
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
48
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
|
49
|
+
ELSE
|
|
50
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
|
51
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
52
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
|
53
|
+
END IF;
|
|
54
|
+
END IF;
|
|
55
|
+
END IF;
|
|
56
|
+
RETURN OLD;
|
|
57
|
+
END;
|
|
58
|
+
$$ LANGUAGE plpgsql;
|
|
59
|
+
SQL
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def down
|
|
64
|
+
if connection.adapter_name == 'PostgreSQL'
|
|
65
|
+
execute(<<-SQL)
|
|
66
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
|
67
|
+
DECLARE
|
|
68
|
+
running_count integer;
|
|
69
|
+
BEGIN
|
|
70
|
+
IF OLD.strand IS NOT NULL THEN
|
|
71
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
72
|
+
IF OLD.id % 20 = 0 THEN
|
|
73
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
74
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
75
|
+
) subquery_for_count);
|
|
76
|
+
IF running_count < OLD.max_concurrent THEN
|
|
77
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
|
78
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
79
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
|
80
|
+
);
|
|
81
|
+
END IF;
|
|
82
|
+
ELSE
|
|
83
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
|
84
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
85
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
|
86
|
+
END IF;
|
|
87
|
+
END IF;
|
|
88
|
+
RETURN OLD;
|
|
89
|
+
END;
|
|
90
|
+
$$ LANGUAGE plpgsql;
|
|
91
|
+
SQL
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
|
|
2
|
+
disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
|
|
3
|
+
|
|
4
|
+
def connection
|
|
5
|
+
Delayed::Job.connection
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
def up
|
|
9
|
+
add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
|
|
10
|
+
add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
|
|
11
|
+
add_index :delayed_jobs, [:strand, :strand_order_override, :id],
|
|
12
|
+
algorithm: :concurrently,
|
|
13
|
+
where: "strand IS NOT NULL",
|
|
14
|
+
name: "next_in_strand_index"
|
|
15
|
+
|
|
16
|
+
if connection.adapter_name == 'PostgreSQL'
|
|
17
|
+
# Use the strand_order_override as the primary sorting mechanism (useful when moving between jobs queues without preserving ID ordering)
|
|
18
|
+
execute(<<-SQL)
|
|
19
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
|
20
|
+
DECLARE
|
|
21
|
+
running_count integer;
|
|
22
|
+
should_lock boolean;
|
|
23
|
+
should_be_precise boolean;
|
|
24
|
+
BEGIN
|
|
25
|
+
IF OLD.strand IS NOT NULL THEN
|
|
26
|
+
should_lock := true;
|
|
27
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
|
28
|
+
|
|
29
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
|
30
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
31
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
32
|
+
) subquery_for_count);
|
|
33
|
+
should_lock := running_count < OLD.max_concurrent;
|
|
34
|
+
END IF;
|
|
35
|
+
|
|
36
|
+
IF should_lock THEN
|
|
37
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
38
|
+
END IF;
|
|
39
|
+
|
|
40
|
+
IF should_be_precise THEN
|
|
41
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
42
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
43
|
+
) subquery_for_count);
|
|
44
|
+
IF running_count < OLD.max_concurrent THEN
|
|
45
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
|
46
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
47
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
|
48
|
+
);
|
|
49
|
+
END IF;
|
|
50
|
+
ELSE
|
|
51
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
|
52
|
+
IF OLD.max_concurrent > 1 THEN
|
|
53
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
|
54
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
55
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
|
56
|
+
ELSE
|
|
57
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
|
58
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
59
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
|
|
60
|
+
END IF;
|
|
61
|
+
END IF;
|
|
62
|
+
END IF;
|
|
63
|
+
RETURN OLD;
|
|
64
|
+
END;
|
|
65
|
+
$$ LANGUAGE plpgsql;
|
|
66
|
+
SQL
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def down
|
|
71
|
+
remove_column :delayed_jobs, :strand_order_override, :integer
|
|
72
|
+
remove_column :failed_jobs, :strand_order_override, :integer
|
|
73
|
+
|
|
74
|
+
if connection.adapter_name == 'PostgreSQL'
|
|
75
|
+
execute(<<-SQL)
|
|
76
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
|
77
|
+
DECLARE
|
|
78
|
+
running_count integer;
|
|
79
|
+
should_lock boolean;
|
|
80
|
+
should_be_precise boolean;
|
|
81
|
+
BEGIN
|
|
82
|
+
IF OLD.strand IS NOT NULL THEN
|
|
83
|
+
should_lock := true;
|
|
84
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
|
85
|
+
|
|
86
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
|
87
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
88
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
89
|
+
) subquery_for_count);
|
|
90
|
+
should_lock := running_count < OLD.max_concurrent;
|
|
91
|
+
END IF;
|
|
92
|
+
|
|
93
|
+
IF should_lock THEN
|
|
94
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
|
95
|
+
END IF;
|
|
96
|
+
|
|
97
|
+
IF should_be_precise THEN
|
|
98
|
+
running_count := (SELECT COUNT(*) FROM (
|
|
99
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
|
100
|
+
) subquery_for_count);
|
|
101
|
+
IF running_count < OLD.max_concurrent THEN
|
|
102
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
|
103
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
104
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
|
105
|
+
);
|
|
106
|
+
END IF;
|
|
107
|
+
ELSE
|
|
108
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
|
109
|
+
IF OLD.max_concurrent > 1 THEN
|
|
110
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
|
111
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
112
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
|
113
|
+
ELSE
|
|
114
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
|
115
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
|
116
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
|
117
|
+
END IF;
|
|
118
|
+
END IF;
|
|
119
|
+
END IF;
|
|
120
|
+
RETURN OLD;
|
|
121
|
+
END;
|
|
122
|
+
$$ LANGUAGE plpgsql;
|
|
123
|
+
SQL
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -17,6 +17,8 @@ module Delayed
|
|
|
17
17
|
include Delayed::Backend::Base
|
|
18
18
|
self.table_name = :delayed_jobs
|
|
19
19
|
|
|
20
|
+
scope :next_in_strand_order, -> { order(:strand_order_override, :id) }
|
|
21
|
+
|
|
20
22
|
def self.reconnect!
|
|
21
23
|
clear_all_connections!
|
|
22
24
|
end
|
|
@@ -398,7 +400,7 @@ module Delayed
|
|
|
398
400
|
strand = options[:strand]
|
|
399
401
|
on_conflict = options.delete(:on_conflict) || :use_earliest
|
|
400
402
|
transaction_for_singleton(strand, on_conflict) do
|
|
401
|
-
job = self.where(:strand => strand, :locked_at => nil).
|
|
403
|
+
job = self.where(:strand => strand, :locked_at => nil).next_in_strand_order.first
|
|
402
404
|
new_job = new(options)
|
|
403
405
|
if job
|
|
404
406
|
new_job.initialize_defaults
|
data/lib/delayed/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: inst-jobs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.16.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Tobias Luetke
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2020-
|
|
12
|
+
date: 2020-08-25 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: activerecord
|
|
@@ -343,6 +343,8 @@ files:
|
|
|
343
343
|
- db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb
|
|
344
344
|
- db/migrate/20190726154743_make_critical_columns_not_null.rb
|
|
345
345
|
- db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb
|
|
346
|
+
- db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb
|
|
347
|
+
- db/migrate/20200825011002_add_strand_order_override.rb
|
|
346
348
|
- exe/inst_jobs
|
|
347
349
|
- lib/delayed/backend/active_record.rb
|
|
348
350
|
- lib/delayed/backend/base.rb
|