switchman-inst-jobs 1.5.1 → 3.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/db/migrate/20101216224513_create_delayed_jobs.rb +42 -0
- data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +14 -0
- data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +13 -0
- data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +14 -0
- data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +26 -0
- data/db/migrate/20110610213249_optimize_delayed_jobs.rb +40 -0
- data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +52 -0
- data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +31 -0
- data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +15 -0
- data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +80 -0
- data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +15 -0
- data/db/migrate/20120608191051_add_jobs_run_at_index.rb +15 -0
- data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +13 -0
- data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +13 -0
- data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +12 -0
- data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +13 -0
- data/db/migrate/20140512213941_add_source_to_jobs.rb +15 -0
- data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +70 -0
- data/db/migrate/20151123210429_add_expires_at_to_jobs.rb +15 -0
- data/db/migrate/20151210162949_improve_max_concurrent.rb +50 -0
- data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +39 -0
- data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +95 -0
- data/db/migrate/20190726154743_make_critical_columns_not_null.rb +15 -0
- data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +25 -0
- data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +94 -0
- data/db/migrate/20200825011002_add_strand_order_override.rb +126 -0
- data/lib/switchman_inst_jobs.rb +3 -3
- data/lib/switchman_inst_jobs/active_record/migration.rb +10 -0
- data/lib/switchman_inst_jobs/delayed/backend/base.rb +9 -9
- data/lib/switchman_inst_jobs/delayed/message_sending.rb +3 -2
- data/lib/switchman_inst_jobs/delayed/worker/health_check.rb +1 -3
- data/lib/switchman_inst_jobs/engine.rb +2 -2
- data/lib/switchman_inst_jobs/{shackles.rb → guard_rail.rb} +1 -1
- data/lib/switchman_inst_jobs/jobs_migrator.rb +9 -7
- data/lib/switchman_inst_jobs/version.rb +1 -1
- metadata +49 -30
- data/db/migrate/20180628153808_set_search_paths_on_functions.rb +0 -15
@@ -0,0 +1,15 @@
|
|
1
|
+
class AddSourceToJobs < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
add_column :delayed_jobs, :source, :string
|
8
|
+
add_column :failed_jobs, :source, :string
|
9
|
+
end
|
10
|
+
|
11
|
+
def down
|
12
|
+
remove_column :delayed_jobs, :source
|
13
|
+
remove_column :failed_jobs, :source
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
class AddMaxConcurrentToJobs < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
add_column :delayed_jobs, :max_concurrent, :integer, default: 1, null: false
|
8
|
+
|
9
|
+
if connection.adapter_name == 'PostgreSQL'
|
10
|
+
execute(<<-CODE)
|
11
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
|
12
|
+
BEGIN
|
13
|
+
IF NEW.strand IS NOT NULL THEN
|
14
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
15
|
+
IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
|
16
|
+
NEW.next_in_strand := 'f';
|
17
|
+
END IF;
|
18
|
+
END IF;
|
19
|
+
RETURN NEW;
|
20
|
+
END;
|
21
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
22
|
+
CODE
|
23
|
+
|
24
|
+
execute(<<-CODE)
|
25
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
26
|
+
BEGIN
|
27
|
+
IF OLD.strand IS NOT NULL THEN
|
28
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
29
|
+
IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
|
30
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
|
31
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
32
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
|
33
|
+
);
|
34
|
+
END IF;
|
35
|
+
END IF;
|
36
|
+
RETURN OLD;
|
37
|
+
END;
|
38
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
39
|
+
CODE
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def down
|
44
|
+
remove_column :delayed_jobs, :max_concurrent
|
45
|
+
|
46
|
+
if connection.adapter_name == 'PostgreSQL'
|
47
|
+
execute(<<-CODE)
|
48
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
|
49
|
+
BEGIN
|
50
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
51
|
+
IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
|
52
|
+
NEW.next_in_strand := 'f';
|
53
|
+
END IF;
|
54
|
+
RETURN NEW;
|
55
|
+
END;
|
56
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
57
|
+
CODE
|
58
|
+
|
59
|
+
execute(<<-CODE)
|
60
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
61
|
+
BEGIN
|
62
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
63
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
|
64
|
+
RETURN OLD;
|
65
|
+
END;
|
66
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
67
|
+
CODE
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class AddExpiresAtToJobs < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
add_column :delayed_jobs, :expires_at, :datetime
|
8
|
+
add_column :failed_jobs, :expires_at, :datetime
|
9
|
+
end
|
10
|
+
|
11
|
+
def down
|
12
|
+
remove_column :delayed_jobs, :expires_at
|
13
|
+
remove_column :failed_jobs, :expires_at
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
class ImproveMaxConcurrent < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
if connection.adapter_name == 'PostgreSQL'
|
8
|
+
execute(<<-CODE)
|
9
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
10
|
+
DECLARE
|
11
|
+
running_count integer;
|
12
|
+
BEGIN
|
13
|
+
IF OLD.strand IS NOT NULL THEN
|
14
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
15
|
+
running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
|
16
|
+
IF running_count < OLD.max_concurrent THEN
|
17
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
18
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
19
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
20
|
+
);
|
21
|
+
END IF;
|
22
|
+
END IF;
|
23
|
+
RETURN OLD;
|
24
|
+
END;
|
25
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
26
|
+
CODE
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def down
|
31
|
+
if connection.adapter_name == 'PostgreSQL'
|
32
|
+
execute(<<-CODE)
|
33
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
34
|
+
BEGIN
|
35
|
+
IF OLD.strand IS NOT NULL THEN
|
36
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
37
|
+
IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
|
38
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
|
39
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
40
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
|
41
|
+
);
|
42
|
+
END IF;
|
43
|
+
END IF;
|
44
|
+
RETURN OLD;
|
45
|
+
END;
|
46
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
47
|
+
CODE
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
class AddBackDefaultStringLimitsJobs < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
drop_triggers
|
8
|
+
|
9
|
+
add_string_limit_if_missing :delayed_jobs, :queue
|
10
|
+
add_string_limit_if_missing :delayed_jobs, :locked_by
|
11
|
+
add_string_limit_if_missing :delayed_jobs, :tag
|
12
|
+
add_string_limit_if_missing :delayed_jobs, :strand
|
13
|
+
add_string_limit_if_missing :delayed_jobs, :source
|
14
|
+
|
15
|
+
add_string_limit_if_missing :failed_jobs, :queue
|
16
|
+
add_string_limit_if_missing :failed_jobs, :locked_by
|
17
|
+
add_string_limit_if_missing :failed_jobs, :tag
|
18
|
+
add_string_limit_if_missing :failed_jobs, :strand
|
19
|
+
add_string_limit_if_missing :failed_jobs, :source
|
20
|
+
|
21
|
+
readd_triggers
|
22
|
+
end
|
23
|
+
|
24
|
+
def drop_triggers
|
25
|
+
execute %(DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name})
|
26
|
+
execute %(DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name})
|
27
|
+
end
|
28
|
+
|
29
|
+
def readd_triggers
|
30
|
+
execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
|
31
|
+
execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
|
32
|
+
end
|
33
|
+
|
34
|
+
def add_string_limit_if_missing(table, column)
|
35
|
+
return if column_exists?(table, column, :string, limit: 255)
|
36
|
+
|
37
|
+
change_column table, column, :string, limit: 255
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
class SpeedUpMaxConcurrentTriggers < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
if connection.adapter_name == 'PostgreSQL'
|
8
|
+
# tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
|
9
|
+
# don't get the count every single time - it's usually safe to just set the next one in line
|
10
|
+
# since the max_concurrent doesn't change all that often for a strand
|
11
|
+
execute(<<-CODE)
|
12
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
13
|
+
DECLARE
|
14
|
+
running_count integer;
|
15
|
+
BEGIN
|
16
|
+
IF OLD.strand IS NOT NULL THEN
|
17
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
18
|
+
IF OLD.id % 20 = 0 THEN
|
19
|
+
running_count := (SELECT COUNT(*) FROM (
|
20
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
21
|
+
) subquery_for_count);
|
22
|
+
IF running_count < OLD.max_concurrent THEN
|
23
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
24
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
25
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
26
|
+
);
|
27
|
+
END IF;
|
28
|
+
ELSE
|
29
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
30
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
31
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
32
|
+
END IF;
|
33
|
+
END IF;
|
34
|
+
RETURN OLD;
|
35
|
+
END;
|
36
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
37
|
+
CODE
|
38
|
+
|
39
|
+
# don't need the full count on insert
|
40
|
+
execute(<<-CODE)
|
41
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
|
42
|
+
BEGIN
|
43
|
+
IF NEW.strand IS NOT NULL THEN
|
44
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
45
|
+
IF (SELECT COUNT(*) FROM (
|
46
|
+
SELECT 1 AS one FROM delayed_jobs WHERE strand = NEW.strand LIMIT NEW.max_concurrent
|
47
|
+
) subquery_for_count) = NEW.max_concurrent THEN
|
48
|
+
NEW.next_in_strand := 'f';
|
49
|
+
END IF;
|
50
|
+
END IF;
|
51
|
+
RETURN NEW;
|
52
|
+
END;
|
53
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
54
|
+
CODE
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def down
|
59
|
+
if connection.adapter_name == 'PostgreSQL'
|
60
|
+
execute(<<-CODE)
|
61
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
62
|
+
DECLARE
|
63
|
+
running_count integer;
|
64
|
+
BEGIN
|
65
|
+
IF OLD.strand IS NOT NULL THEN
|
66
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
67
|
+
running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
|
68
|
+
IF running_count < OLD.max_concurrent THEN
|
69
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
70
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
71
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
72
|
+
);
|
73
|
+
END IF;
|
74
|
+
END IF;
|
75
|
+
RETURN OLD;
|
76
|
+
END;
|
77
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
78
|
+
CODE
|
79
|
+
|
80
|
+
execute(<<-CODE)
|
81
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
|
82
|
+
BEGIN
|
83
|
+
IF NEW.strand IS NOT NULL THEN
|
84
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
85
|
+
IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
|
86
|
+
NEW.next_in_strand := 'f';
|
87
|
+
END IF;
|
88
|
+
END IF;
|
89
|
+
RETURN NEW;
|
90
|
+
END;
|
91
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
92
|
+
CODE
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class MakeCriticalColumnsNotNull < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
change_column_null :delayed_jobs, :run_at, false
|
8
|
+
change_column_null :delayed_jobs, :queue, false
|
9
|
+
end
|
10
|
+
|
11
|
+
def down
|
12
|
+
change_column_null :delayed_jobs, :run_at, true
|
13
|
+
change_column_null :delayed_jobs, :queue, true
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
class AddIdToGetDelayedJobsIndex < ActiveRecord::Migration[4.2]
|
2
|
+
disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
|
3
|
+
|
4
|
+
def connection
|
5
|
+
Delayed::Job.connection
|
6
|
+
end
|
7
|
+
|
8
|
+
def up
|
9
|
+
rename_index :delayed_jobs, 'get_delayed_jobs_index', 'get_delayed_jobs_index_old'
|
10
|
+
add_index :delayed_jobs, %i[queue priority run_at id],
|
11
|
+
algorithm: :concurrently,
|
12
|
+
where: 'locked_at IS NULL AND next_in_strand',
|
13
|
+
name: 'get_delayed_jobs_index'
|
14
|
+
remove_index :delayed_jobs, name: 'get_delayed_jobs_index_old'
|
15
|
+
end
|
16
|
+
|
17
|
+
def down
|
18
|
+
rename_index :delayed_jobs, 'get_delayed_jobs_index', 'get_delayed_jobs_index_old'
|
19
|
+
add_index :delayed_jobs, %i[priority run_at queue],
|
20
|
+
algorithm: :concurrently,
|
21
|
+
where: 'locked_at IS NULL AND next_in_strand',
|
22
|
+
name: 'get_delayed_jobs_index'
|
23
|
+
remove_index :delayed_jobs, name: 'get_delayed_jobs_index_old'
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
if connection.adapter_name == 'PostgreSQL'
|
8
|
+
# tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
|
9
|
+
# don't get the count every single time - it's usually safe to just set the next one in line
|
10
|
+
# since the max_concurrent doesn't change all that often for a strand
|
11
|
+
execute(<<-SQL)
|
12
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
13
|
+
DECLARE
|
14
|
+
running_count integer;
|
15
|
+
should_lock boolean;
|
16
|
+
should_be_precise boolean;
|
17
|
+
BEGIN
|
18
|
+
IF OLD.strand IS NOT NULL THEN
|
19
|
+
should_lock := true;
|
20
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
21
|
+
|
22
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
23
|
+
running_count := (SELECT COUNT(*) FROM (
|
24
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
25
|
+
) subquery_for_count);
|
26
|
+
should_lock := running_count < OLD.max_concurrent;
|
27
|
+
END IF;
|
28
|
+
|
29
|
+
IF should_lock THEN
|
30
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
31
|
+
END IF;
|
32
|
+
|
33
|
+
IF should_be_precise THEN
|
34
|
+
running_count := (SELECT COUNT(*) FROM (
|
35
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
36
|
+
) subquery_for_count);
|
37
|
+
IF running_count < OLD.max_concurrent THEN
|
38
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
39
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
40
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
41
|
+
);
|
42
|
+
END IF;
|
43
|
+
ELSE
|
44
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
45
|
+
IF OLD.max_concurrent > 1 THEN
|
46
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
47
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
48
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
49
|
+
ELSE
|
50
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
51
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
52
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
53
|
+
END IF;
|
54
|
+
END IF;
|
55
|
+
END IF;
|
56
|
+
RETURN OLD;
|
57
|
+
END;
|
58
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
59
|
+
SQL
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def down
|
64
|
+
if connection.adapter_name == 'PostgreSQL'
|
65
|
+
execute(<<-SQL)
|
66
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
67
|
+
DECLARE
|
68
|
+
running_count integer;
|
69
|
+
BEGIN
|
70
|
+
IF OLD.strand IS NOT NULL THEN
|
71
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
72
|
+
IF OLD.id % 20 = 0 THEN
|
73
|
+
running_count := (SELECT COUNT(*) FROM (
|
74
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
75
|
+
) subquery_for_count);
|
76
|
+
IF running_count < OLD.max_concurrent THEN
|
77
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
78
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
79
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
80
|
+
);
|
81
|
+
END IF;
|
82
|
+
ELSE
|
83
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
84
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
85
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
86
|
+
END IF;
|
87
|
+
END IF;
|
88
|
+
RETURN OLD;
|
89
|
+
END;
|
90
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
91
|
+
SQL
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,126 @@
|
|
1
|
+
class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
|
2
|
+
disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
|
3
|
+
|
4
|
+
def connection
|
5
|
+
Delayed::Job.connection
|
6
|
+
end
|
7
|
+
|
8
|
+
def up
|
9
|
+
add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
|
10
|
+
add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
|
11
|
+
add_index :delayed_jobs, %i[strand strand_order_override id],
|
12
|
+
algorithm: :concurrently,
|
13
|
+
where: 'strand IS NOT NULL',
|
14
|
+
name: 'next_in_strand_index'
|
15
|
+
|
16
|
+
if connection.adapter_name == 'PostgreSQL'
|
17
|
+
# Use the strand_order_override as the primary sorting mechanism (useful when moving between jobs queues without preserving ID ordering)
|
18
|
+
execute(<<-SQL)
|
19
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
20
|
+
DECLARE
|
21
|
+
running_count integer;
|
22
|
+
should_lock boolean;
|
23
|
+
should_be_precise boolean;
|
24
|
+
BEGIN
|
25
|
+
IF OLD.strand IS NOT NULL THEN
|
26
|
+
should_lock := true;
|
27
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
28
|
+
|
29
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
30
|
+
running_count := (SELECT COUNT(*) FROM (
|
31
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
32
|
+
) subquery_for_count);
|
33
|
+
should_lock := running_count < OLD.max_concurrent;
|
34
|
+
END IF;
|
35
|
+
|
36
|
+
IF should_lock THEN
|
37
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
38
|
+
END IF;
|
39
|
+
|
40
|
+
IF should_be_precise THEN
|
41
|
+
running_count := (SELECT COUNT(*) FROM (
|
42
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
43
|
+
) subquery_for_count);
|
44
|
+
IF running_count < OLD.max_concurrent THEN
|
45
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
46
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
47
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
48
|
+
);
|
49
|
+
END IF;
|
50
|
+
ELSE
|
51
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
52
|
+
IF OLD.max_concurrent > 1 THEN
|
53
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
54
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
55
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
56
|
+
ELSE
|
57
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
58
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
59
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
|
60
|
+
END IF;
|
61
|
+
END IF;
|
62
|
+
END IF;
|
63
|
+
RETURN OLD;
|
64
|
+
END;
|
65
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
66
|
+
SQL
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def down
|
71
|
+
remove_column :delayed_jobs, :strand_order_override, :integer
|
72
|
+
remove_column :failed_jobs, :strand_order_override, :integer
|
73
|
+
|
74
|
+
if connection.adapter_name == 'PostgreSQL'
|
75
|
+
execute(<<-SQL)
|
76
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
77
|
+
DECLARE
|
78
|
+
running_count integer;
|
79
|
+
should_lock boolean;
|
80
|
+
should_be_precise boolean;
|
81
|
+
BEGIN
|
82
|
+
IF OLD.strand IS NOT NULL THEN
|
83
|
+
should_lock := true;
|
84
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
85
|
+
|
86
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
87
|
+
running_count := (SELECT COUNT(*) FROM (
|
88
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
89
|
+
) subquery_for_count);
|
90
|
+
should_lock := running_count < OLD.max_concurrent;
|
91
|
+
END IF;
|
92
|
+
|
93
|
+
IF should_lock THEN
|
94
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
95
|
+
END IF;
|
96
|
+
|
97
|
+
IF should_be_precise THEN
|
98
|
+
running_count := (SELECT COUNT(*) FROM (
|
99
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
100
|
+
) subquery_for_count);
|
101
|
+
IF running_count < OLD.max_concurrent THEN
|
102
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
103
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
104
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
105
|
+
);
|
106
|
+
END IF;
|
107
|
+
ELSE
|
108
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
109
|
+
IF OLD.max_concurrent > 1 THEN
|
110
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
111
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
112
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
113
|
+
ELSE
|
114
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
115
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
116
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
117
|
+
END IF;
|
118
|
+
END IF;
|
119
|
+
END IF;
|
120
|
+
RETURN OLD;
|
121
|
+
END;
|
122
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
123
|
+
SQL
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|