inst-jobs 2.3.3 → 2.4.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/db/migrate/20101216224513_create_delayed_jobs.rb +9 -7
- data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +8 -13
- data/db/migrate/20110610213249_optimize_delayed_jobs.rb +8 -8
- data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +25 -25
- data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +4 -8
- data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +1 -3
- data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +11 -15
- data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +1 -1
- data/db/migrate/20120608191051_add_jobs_run_at_index.rb +2 -2
- data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +1 -1
- data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +2 -3
- data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +9 -13
- data/db/migrate/20151210162949_improve_max_concurrent.rb +4 -8
- data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +3 -2
- data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +13 -17
- data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +8 -8
- data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +72 -77
- data/db/migrate/20200825011002_add_strand_order_override.rb +93 -97
- data/db/migrate/20210809145804_add_n_strand_index.rb +3 -3
- data/db/migrate/20210812210128_add_singleton_column.rb +200 -0
- data/exe/inst_jobs +3 -2
- data/lib/delayed/backend/active_record.rb +183 -149
- data/lib/delayed/backend/base.rb +80 -75
- data/lib/delayed/batch.rb +11 -9
- data/lib/delayed/cli.rb +98 -84
- data/lib/delayed/core_ext/kernel.rb +4 -2
- data/lib/delayed/daemon.rb +70 -74
- data/lib/delayed/job_tracking.rb +26 -25
- data/lib/delayed/lifecycle.rb +27 -24
- data/lib/delayed/log_tailer.rb +17 -17
- data/lib/delayed/logging.rb +13 -16
- data/lib/delayed/message_sending.rb +43 -52
- data/lib/delayed/performable_method.rb +6 -8
- data/lib/delayed/periodic.rb +66 -65
- data/lib/delayed/plugin.rb +2 -4
- data/lib/delayed/pool.rb +198 -193
- data/lib/delayed/server/helpers.rb +6 -6
- data/lib/delayed/server.rb +51 -54
- data/lib/delayed/settings.rb +93 -81
- data/lib/delayed/testing.rb +21 -22
- data/lib/delayed/version.rb +1 -1
- data/lib/delayed/work_queue/in_process.rb +21 -18
- data/lib/delayed/work_queue/parent_process/client.rb +54 -55
- data/lib/delayed/work_queue/parent_process/server.rb +215 -209
- data/lib/delayed/work_queue/parent_process.rb +52 -53
- data/lib/delayed/worker/consul_health_check.rb +21 -19
- data/lib/delayed/worker/health_check.rb +21 -12
- data/lib/delayed/worker/null_health_check.rb +3 -1
- data/lib/delayed/worker/process_helper.rb +8 -9
- data/lib/delayed/worker.rb +271 -265
- data/lib/delayed/yaml_extensions.rb +12 -10
- data/lib/delayed_job.rb +37 -38
- data/lib/inst-jobs.rb +1 -1
- data/spec/active_record_job_spec.rb +129 -136
- data/spec/delayed/cli_spec.rb +7 -7
- data/spec/delayed/daemon_spec.rb +8 -8
- data/spec/delayed/message_sending_spec.rb +16 -9
- data/spec/delayed/periodic_spec.rb +13 -12
- data/spec/delayed/server_spec.rb +38 -38
- data/spec/delayed/settings_spec.rb +26 -25
- data/spec/delayed/work_queue/in_process_spec.rb +7 -7
- data/spec/delayed/work_queue/parent_process/client_spec.rb +15 -11
- data/spec/delayed/work_queue/parent_process/server_spec.rb +43 -40
- data/spec/delayed/work_queue/parent_process_spec.rb +21 -21
- data/spec/delayed/worker/consul_health_check_spec.rb +22 -22
- data/spec/delayed/worker/health_check_spec.rb +51 -49
- data/spec/delayed/worker_spec.rb +28 -25
- data/spec/sample_jobs.rb +45 -15
- data/spec/shared/delayed_batch.rb +74 -67
- data/spec/shared/delayed_method.rb +143 -102
- data/spec/shared/performable_method.rb +39 -38
- data/spec/shared/shared_backend.rb +534 -441
- data/spec/shared/testing.rb +14 -14
- data/spec/shared/worker.rb +155 -147
- data/spec/shared_jobs_specs.rb +13 -13
- data/spec/spec_helper.rb +43 -40
- metadata +81 -59
- data/lib/delayed/backend/redis/bulk_update.lua +0 -50
- data/lib/delayed/backend/redis/destroy_job.lua +0 -2
- data/lib/delayed/backend/redis/enqueue.lua +0 -29
- data/lib/delayed/backend/redis/fail_job.lua +0 -5
- data/lib/delayed/backend/redis/find_available.lua +0 -3
- data/lib/delayed/backend/redis/functions.rb +0 -59
- data/lib/delayed/backend/redis/get_and_lock_next_available.lua +0 -17
- data/lib/delayed/backend/redis/includes/jobs_common.lua +0 -203
- data/lib/delayed/backend/redis/job.rb +0 -528
- data/lib/delayed/backend/redis/set_running.lua +0 -5
- data/lib/delayed/backend/redis/tickle_strand.lua +0 -2
- data/spec/gemfiles/52.gemfile +0 -7
- data/spec/gemfiles/60.gemfile +0 -7
- data/spec/gemfiles/60.gemfile.lock +0 -246
- data/spec/gemfiles/61.gemfile +0 -7
- data/spec/redis_job_spec.rb +0 -148
@@ -6,92 +6,87 @@ class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
|
|
6
6
|
end
|
7
7
|
|
8
8
|
def up
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
9
|
+
# tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
|
10
|
+
# don't get the count every single time - it's usually safe to just set the next one in line
|
11
|
+
# since the max_concurrent doesn't change all that often for a strand
|
12
|
+
execute(<<-SQL)
|
13
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
14
|
+
DECLARE
|
15
|
+
running_count integer;
|
16
|
+
should_lock boolean;
|
17
|
+
should_be_precise boolean;
|
18
|
+
BEGIN
|
19
|
+
IF OLD.strand IS NOT NULL THEN
|
20
|
+
should_lock := true;
|
21
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
22
|
+
|
23
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
24
|
+
running_count := (SELECT COUNT(*) FROM (
|
25
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
26
|
+
) subquery_for_count);
|
27
|
+
should_lock := running_count < OLD.max_concurrent;
|
28
|
+
END IF;
|
29
|
+
|
30
|
+
IF should_lock THEN
|
31
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
32
|
+
END IF;
|
33
|
+
|
34
|
+
IF should_be_precise THEN
|
35
|
+
running_count := (SELECT COUNT(*) FROM (
|
36
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
37
|
+
) subquery_for_count);
|
38
|
+
IF running_count < OLD.max_concurrent THEN
|
39
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
40
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
41
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
42
|
+
);
|
33
43
|
END IF;
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
41
|
-
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
42
|
-
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
43
|
-
);
|
44
|
-
END IF;
|
44
|
+
ELSE
|
45
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
46
|
+
IF OLD.max_concurrent > 1 THEN
|
47
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
48
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
49
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
45
50
|
ELSE
|
46
|
-
|
47
|
-
IF OLD.max_concurrent > 1 THEN
|
48
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
51
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
49
52
|
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
50
|
-
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
|
51
|
-
ELSE
|
52
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
53
|
-
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
54
|
-
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
55
|
-
END IF;
|
53
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
56
54
|
END IF;
|
57
55
|
END IF;
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
56
|
+
END IF;
|
57
|
+
RETURN OLD;
|
58
|
+
END;
|
59
|
+
$$ LANGUAGE plpgsql;
|
60
|
+
SQL
|
63
61
|
end
|
64
62
|
|
65
63
|
def down
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
);
|
83
|
-
END IF;
|
84
|
-
ELSE
|
85
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
86
|
-
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
87
|
-
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
64
|
+
execute(<<-SQL)
|
65
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
66
|
+
DECLARE
|
67
|
+
running_count integer;
|
68
|
+
BEGIN
|
69
|
+
IF OLD.strand IS NOT NULL THEN
|
70
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
71
|
+
IF OLD.id % 20 = 0 THEN
|
72
|
+
running_count := (SELECT COUNT(*) FROM (
|
73
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
74
|
+
) subquery_for_count);
|
75
|
+
IF running_count < OLD.max_concurrent THEN
|
76
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
77
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
78
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
79
|
+
);
|
88
80
|
END IF;
|
81
|
+
ELSE
|
82
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
83
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
84
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
89
85
|
END IF;
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
86
|
+
END IF;
|
87
|
+
RETURN OLD;
|
88
|
+
END;
|
89
|
+
$$ LANGUAGE plpgsql;
|
90
|
+
SQL
|
95
91
|
end
|
96
92
|
end
|
97
|
-
|
@@ -10,119 +10,115 @@ class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
|
|
10
10
|
def up
|
11
11
|
add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
|
12
12
|
add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
|
13
|
-
add_index :delayed_jobs, [
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
add_index :delayed_jobs, %i[strand strand_order_override id],
|
14
|
+
algorithm: :concurrently,
|
15
|
+
where: "strand IS NOT NULL",
|
16
|
+
name: "next_in_strand_index"
|
17
17
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
18
|
+
# Use the strand_order_override as the primary sorting mechanism (useful when moving between jobs queues without preserving ID ordering)
|
19
|
+
execute(<<-SQL)
|
20
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
21
|
+
DECLARE
|
22
|
+
running_count integer;
|
23
|
+
should_lock boolean;
|
24
|
+
should_be_precise boolean;
|
25
|
+
BEGIN
|
26
|
+
IF OLD.strand IS NOT NULL THEN
|
27
|
+
should_lock := true;
|
28
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
29
|
+
|
30
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
31
|
+
running_count := (SELECT COUNT(*) FROM (
|
32
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
33
|
+
) subquery_for_count);
|
34
|
+
should_lock := running_count < OLD.max_concurrent;
|
35
|
+
END IF;
|
36
|
+
|
37
|
+
IF should_lock THEN
|
38
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
39
|
+
END IF;
|
40
|
+
|
41
|
+
IF should_be_precise THEN
|
42
|
+
running_count := (SELECT COUNT(*) FROM (
|
43
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
44
|
+
) subquery_for_count);
|
45
|
+
IF running_count < OLD.max_concurrent THEN
|
46
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
47
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
48
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
49
|
+
);
|
40
50
|
END IF;
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
48
|
-
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
49
|
-
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
50
|
-
);
|
51
|
-
END IF;
|
51
|
+
ELSE
|
52
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
53
|
+
IF OLD.max_concurrent > 1 THEN
|
54
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
55
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
56
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
52
57
|
ELSE
|
53
|
-
|
54
|
-
IF OLD.max_concurrent > 1 THEN
|
55
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
58
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
56
59
|
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
57
|
-
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE
|
58
|
-
ELSE
|
59
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
60
|
-
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
61
|
-
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
|
62
|
-
END IF;
|
60
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
|
63
61
|
END IF;
|
64
62
|
END IF;
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
63
|
+
END IF;
|
64
|
+
RETURN OLD;
|
65
|
+
END;
|
66
|
+
$$ LANGUAGE plpgsql;
|
67
|
+
SQL
|
70
68
|
end
|
71
69
|
|
72
70
|
def down
|
73
71
|
remove_column :delayed_jobs, :strand_order_override, :integer
|
74
72
|
remove_column :failed_jobs, :strand_order_override, :integer
|
75
73
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
74
|
+
execute(<<-SQL)
|
75
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
76
|
+
DECLARE
|
77
|
+
running_count integer;
|
78
|
+
should_lock boolean;
|
79
|
+
should_be_precise boolean;
|
80
|
+
BEGIN
|
81
|
+
IF OLD.strand IS NOT NULL THEN
|
82
|
+
should_lock := true;
|
83
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
84
|
+
|
85
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
86
|
+
running_count := (SELECT COUNT(*) FROM (
|
87
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
88
|
+
) subquery_for_count);
|
89
|
+
should_lock := running_count < OLD.max_concurrent;
|
90
|
+
END IF;
|
91
|
+
|
92
|
+
IF should_lock THEN
|
93
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
94
|
+
END IF;
|
95
|
+
|
96
|
+
IF should_be_precise THEN
|
97
|
+
running_count := (SELECT COUNT(*) FROM (
|
98
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
99
|
+
) subquery_for_count);
|
100
|
+
IF running_count < OLD.max_concurrent THEN
|
101
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
102
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
103
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
104
|
+
);
|
97
105
|
END IF;
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
105
|
-
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
106
|
-
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
107
|
-
);
|
108
|
-
END IF;
|
106
|
+
ELSE
|
107
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
108
|
+
IF OLD.max_concurrent > 1 THEN
|
109
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
110
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
111
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
109
112
|
ELSE
|
110
|
-
|
111
|
-
IF OLD.max_concurrent > 1 THEN
|
112
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
113
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
113
114
|
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
114
|
-
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
|
115
|
-
ELSE
|
116
|
-
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
117
|
-
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
118
|
-
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
119
|
-
END IF;
|
115
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
120
116
|
END IF;
|
121
117
|
END IF;
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
118
|
+
END IF;
|
119
|
+
RETURN OLD;
|
120
|
+
END;
|
121
|
+
$$ LANGUAGE plpgsql;
|
122
|
+
SQL
|
127
123
|
end
|
128
124
|
end
|
@@ -4,9 +4,9 @@ class AddNStrandIndex < ActiveRecord::Migration[5.2]
|
|
4
4
|
disable_ddl_transaction!
|
5
5
|
|
6
6
|
def change
|
7
|
-
add_index :delayed_jobs, [
|
8
|
-
name:
|
9
|
-
where:
|
7
|
+
add_index :delayed_jobs, %i[strand next_in_strand id],
|
8
|
+
name: "n_strand_index",
|
9
|
+
where: "strand IS NOT NULL",
|
10
10
|
algorithm: :concurrently
|
11
11
|
end
|
12
12
|
end
|
@@ -0,0 +1,200 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class AddSingletonColumn < ActiveRecord::Migration[5.2]
|
4
|
+
disable_ddl_transaction!
|
5
|
+
|
6
|
+
def change
|
7
|
+
add_column :delayed_jobs, :singleton, :string, if_not_exists: true
|
8
|
+
add_column :failed_jobs, :singleton, :string, if_not_exists: true
|
9
|
+
# only one job can be queued in a singleton
|
10
|
+
add_index :delayed_jobs,
|
11
|
+
:singleton,
|
12
|
+
where: "singleton IS NOT NULL AND locked_by IS NULL",
|
13
|
+
unique: true,
|
14
|
+
name: "index_delayed_jobs_on_singleton_not_running",
|
15
|
+
algorithm: :concurrently
|
16
|
+
# only one job can be running for a singleton
|
17
|
+
add_index :delayed_jobs,
|
18
|
+
:singleton,
|
19
|
+
where: "singleton IS NOT NULL AND locked_by IS NOT NULL",
|
20
|
+
unique: true,
|
21
|
+
name: "index_delayed_jobs_on_singleton_running",
|
22
|
+
algorithm: :concurrently
|
23
|
+
|
24
|
+
reversible do |direction|
|
25
|
+
direction.up do
|
26
|
+
execute(<<~SQL)
|
27
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
28
|
+
DECLARE
|
29
|
+
running_count integer;
|
30
|
+
should_lock boolean;
|
31
|
+
should_be_precise boolean;
|
32
|
+
update_query varchar;
|
33
|
+
skip_locked varchar;
|
34
|
+
BEGIN
|
35
|
+
IF OLD.strand IS NOT NULL THEN
|
36
|
+
should_lock := true;
|
37
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
38
|
+
|
39
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
40
|
+
running_count := (SELECT COUNT(*) FROM (
|
41
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
42
|
+
) subquery_for_count);
|
43
|
+
should_lock := running_count < OLD.max_concurrent;
|
44
|
+
END IF;
|
45
|
+
|
46
|
+
IF should_lock THEN
|
47
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
48
|
+
END IF;
|
49
|
+
|
50
|
+
-- note that we don't really care if the row we're deleting has a singleton, or if it even
|
51
|
+
-- matches the row(s) we're going to update. we just need to make sure that whatever
|
52
|
+
-- singleton we grab isn't already running (which is a simple existence check, since
|
53
|
+
-- the unique indexes ensure there is at most one singleton running, and one queued)
|
54
|
+
update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
|
55
|
+
SELECT id FROM delayed_jobs j2
|
56
|
+
WHERE next_in_strand=false AND
|
57
|
+
j2.strand=$1.strand AND
|
58
|
+
(j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id))
|
59
|
+
ORDER BY j2.strand_order_override ASC, j2.id ASC
|
60
|
+
LIMIT ';
|
61
|
+
|
62
|
+
IF should_be_precise THEN
|
63
|
+
running_count := (SELECT COUNT(*) FROM (
|
64
|
+
SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
65
|
+
) s);
|
66
|
+
IF running_count < OLD.max_concurrent THEN
|
67
|
+
update_query := update_query || '($1.max_concurrent - $2)';
|
68
|
+
ELSE
|
69
|
+
-- we have too many running already; just bail
|
70
|
+
RETURN OLD;
|
71
|
+
END IF;
|
72
|
+
ELSE
|
73
|
+
update_query := update_query || '1';
|
74
|
+
|
75
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
76
|
+
IF OLD.max_concurrent > 1 THEN
|
77
|
+
skip_locked := ' SKIP LOCKED';
|
78
|
+
END IF;
|
79
|
+
END IF;
|
80
|
+
|
81
|
+
update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
|
82
|
+
EXECUTE update_query USING OLD, running_count;
|
83
|
+
ELSIF OLD.singleton IS NOT NULL THEN
|
84
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false;
|
85
|
+
END IF;
|
86
|
+
RETURN OLD;
|
87
|
+
END;
|
88
|
+
$$ LANGUAGE plpgsql;
|
89
|
+
SQL
|
90
|
+
execute(<<~SQL)
|
91
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
|
92
|
+
BEGIN
|
93
|
+
IF NEW.strand IS NOT NULL THEN
|
94
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
95
|
+
IF (SELECT COUNT(*) FROM (
|
96
|
+
SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
|
97
|
+
) s) = NEW.max_concurrent THEN
|
98
|
+
NEW.next_in_strand := false;
|
99
|
+
END IF;
|
100
|
+
END IF;
|
101
|
+
IF NEW.singleton IS NOT NULL THEN
|
102
|
+
PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
|
103
|
+
IF FOUND THEN
|
104
|
+
NEW.next_in_strand := false;
|
105
|
+
END IF;
|
106
|
+
END IF;
|
107
|
+
RETURN NEW;
|
108
|
+
END;
|
109
|
+
$$ LANGUAGE plpgsql;
|
110
|
+
SQL
|
111
|
+
end
|
112
|
+
direction.down do
|
113
|
+
execute(<<~SQL)
|
114
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
|
115
|
+
DECLARE
|
116
|
+
running_count integer;
|
117
|
+
should_lock boolean;
|
118
|
+
should_be_precise boolean;
|
119
|
+
BEGIN
|
120
|
+
IF OLD.strand IS NOT NULL THEN
|
121
|
+
should_lock := true;
|
122
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
123
|
+
|
124
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
125
|
+
running_count := (SELECT COUNT(*) FROM (
|
126
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
127
|
+
) subquery_for_count);
|
128
|
+
should_lock := running_count < OLD.max_concurrent;
|
129
|
+
END IF;
|
130
|
+
|
131
|
+
IF should_lock THEN
|
132
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
133
|
+
END IF;
|
134
|
+
|
135
|
+
IF should_be_precise THEN
|
136
|
+
running_count := (SELECT COUNT(*) FROM (
|
137
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
138
|
+
) subquery_for_count);
|
139
|
+
IF running_count < OLD.max_concurrent THEN
|
140
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
141
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
142
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
143
|
+
);
|
144
|
+
END IF;
|
145
|
+
ELSE
|
146
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
147
|
+
IF OLD.max_concurrent > 1 THEN
|
148
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
149
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
150
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
151
|
+
ELSE
|
152
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
153
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
154
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
|
155
|
+
END IF;
|
156
|
+
END IF;
|
157
|
+
END IF;
|
158
|
+
RETURN OLD;
|
159
|
+
END;
|
160
|
+
$$ LANGUAGE plpgsql;
|
161
|
+
SQL
|
162
|
+
execute(<<~SQL)
|
163
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
|
164
|
+
BEGIN
|
165
|
+
IF NEW.strand IS NOT NULL THEN
|
166
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
167
|
+
IF (SELECT COUNT(*) FROM (
|
168
|
+
SELECT 1 AS one FROM delayed_jobs WHERE strand = NEW.strand LIMIT NEW.max_concurrent
|
169
|
+
) subquery_for_count) = NEW.max_concurrent THEN
|
170
|
+
NEW.next_in_strand := 'f';
|
171
|
+
END IF;
|
172
|
+
END IF;
|
173
|
+
RETURN NEW;
|
174
|
+
END;
|
175
|
+
$$ LANGUAGE plpgsql;
|
176
|
+
SQL
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
connection.transaction do
|
181
|
+
reversible do |direction|
|
182
|
+
direction.up do
|
183
|
+
drop_triggers
|
184
|
+
execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON delayed_jobs FOR EACH ROW WHEN (NEW.strand IS NOT NULL OR NEW.singleton IS NOT NULL) EXECUTE PROCEDURE delayed_jobs_before_insert_row_tr_fn()")
|
185
|
+
execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON delayed_jobs FOR EACH ROW WHEN ((OLD.strand IS NOT NULL OR OLD.singleton IS NOT NULL) AND OLD.next_in_strand=true) EXECUTE PROCEDURE delayed_jobs_after_delete_row_tr_fn()")
|
186
|
+
end
|
187
|
+
direction.down do
|
188
|
+
drop_triggers
|
189
|
+
execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON delayed_jobs FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE delayed_jobs_before_insert_row_tr_fn()")
|
190
|
+
execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON delayed_jobs FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE delayed_jobs_after_delete_row_tr_fn()")
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def drop_triggers
|
197
|
+
execute("DROP TRIGGER delayed_jobs_before_insert_row_tr ON delayed_jobs")
|
198
|
+
execute("DROP TRIGGER delayed_jobs_after_delete_row_tr ON delayed_jobs")
|
199
|
+
end
|
200
|
+
end
|
data/exe/inst_jobs
CHANGED