switchman-inst-jobs 1.5.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/db/migrate/20101216224513_create_delayed_jobs.rb +42 -0
- data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +14 -0
- data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +13 -0
- data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +14 -0
- data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +26 -0
- data/db/migrate/20110610213249_optimize_delayed_jobs.rb +40 -0
- data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +52 -0
- data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +31 -0
- data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +15 -0
- data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +80 -0
- data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +15 -0
- data/db/migrate/20120608191051_add_jobs_run_at_index.rb +15 -0
- data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +13 -0
- data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +13 -0
- data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +12 -0
- data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +13 -0
- data/db/migrate/20140512213941_add_source_to_jobs.rb +15 -0
- data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +70 -0
- data/db/migrate/20151123210429_add_expires_at_to_jobs.rb +15 -0
- data/db/migrate/20151210162949_improve_max_concurrent.rb +50 -0
- data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +39 -0
- data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +95 -0
- data/db/migrate/20190726154743_make_critical_columns_not_null.rb +15 -0
- data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +25 -0
- data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +94 -0
- data/db/migrate/20200825011002_add_strand_order_override.rb +126 -0
- data/lib/switchman_inst_jobs.rb +3 -3
- data/lib/switchman_inst_jobs/active_record/migration.rb +10 -0
- data/lib/switchman_inst_jobs/delayed/backend/base.rb +9 -9
- data/lib/switchman_inst_jobs/delayed/message_sending.rb +2 -2
- data/lib/switchman_inst_jobs/delayed/worker/health_check.rb +1 -3
- data/lib/switchman_inst_jobs/engine.rb +2 -2
- data/lib/switchman_inst_jobs/{shackles.rb → guard_rail.rb} +1 -1
- data/lib/switchman_inst_jobs/jobs_migrator.rb +9 -7
- data/lib/switchman_inst_jobs/switchman/default_shard.rb +4 -0
- data/lib/switchman_inst_jobs/version.rb +1 -1
- metadata +45 -32
- data/db/migrate/20180628153808_set_search_paths_on_functions.rb +0 -15
@@ -0,0 +1,15 @@
|
|
1
|
+
class AddSourceToJobs < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
add_column :delayed_jobs, :source, :string
|
8
|
+
add_column :failed_jobs, :source, :string
|
9
|
+
end
|
10
|
+
|
11
|
+
def down
|
12
|
+
remove_column :delayed_jobs, :source
|
13
|
+
remove_column :failed_jobs, :source
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
class AddMaxConcurrentToJobs < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
add_column :delayed_jobs, :max_concurrent, :integer, default: 1, null: false
|
8
|
+
|
9
|
+
if connection.adapter_name == 'PostgreSQL'
|
10
|
+
execute(<<-CODE)
|
11
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
|
12
|
+
BEGIN
|
13
|
+
IF NEW.strand IS NOT NULL THEN
|
14
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
15
|
+
IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
|
16
|
+
NEW.next_in_strand := 'f';
|
17
|
+
END IF;
|
18
|
+
END IF;
|
19
|
+
RETURN NEW;
|
20
|
+
END;
|
21
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
22
|
+
CODE
|
23
|
+
|
24
|
+
execute(<<-CODE)
|
25
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
26
|
+
BEGIN
|
27
|
+
IF OLD.strand IS NOT NULL THEN
|
28
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
29
|
+
IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
|
30
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
|
31
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
32
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
|
33
|
+
);
|
34
|
+
END IF;
|
35
|
+
END IF;
|
36
|
+
RETURN OLD;
|
37
|
+
END;
|
38
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
39
|
+
CODE
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def down
|
44
|
+
remove_column :delayed_jobs, :max_concurrent
|
45
|
+
|
46
|
+
if connection.adapter_name == 'PostgreSQL'
|
47
|
+
execute(<<-CODE)
|
48
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
|
49
|
+
BEGIN
|
50
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
51
|
+
IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
|
52
|
+
NEW.next_in_strand := 'f';
|
53
|
+
END IF;
|
54
|
+
RETURN NEW;
|
55
|
+
END;
|
56
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
57
|
+
CODE
|
58
|
+
|
59
|
+
execute(<<-CODE)
|
60
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
61
|
+
BEGIN
|
62
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
63
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
|
64
|
+
RETURN OLD;
|
65
|
+
END;
|
66
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
67
|
+
CODE
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class AddExpiresAtToJobs < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
add_column :delayed_jobs, :expires_at, :datetime
|
8
|
+
add_column :failed_jobs, :expires_at, :datetime
|
9
|
+
end
|
10
|
+
|
11
|
+
def down
|
12
|
+
remove_column :delayed_jobs, :expires_at
|
13
|
+
remove_column :failed_jobs, :expires_at
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
class ImproveMaxConcurrent < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
if connection.adapter_name == 'PostgreSQL'
|
8
|
+
execute(<<-CODE)
|
9
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
10
|
+
DECLARE
|
11
|
+
running_count integer;
|
12
|
+
BEGIN
|
13
|
+
IF OLD.strand IS NOT NULL THEN
|
14
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
15
|
+
running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
|
16
|
+
IF running_count < OLD.max_concurrent THEN
|
17
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
18
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
19
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
20
|
+
);
|
21
|
+
END IF;
|
22
|
+
END IF;
|
23
|
+
RETURN OLD;
|
24
|
+
END;
|
25
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
26
|
+
CODE
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def down
|
31
|
+
if connection.adapter_name == 'PostgreSQL'
|
32
|
+
execute(<<-CODE)
|
33
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
34
|
+
BEGIN
|
35
|
+
IF OLD.strand IS NOT NULL THEN
|
36
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
37
|
+
IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
|
38
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
|
39
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
40
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
|
41
|
+
);
|
42
|
+
END IF;
|
43
|
+
END IF;
|
44
|
+
RETURN OLD;
|
45
|
+
END;
|
46
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
47
|
+
CODE
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
class AddBackDefaultStringLimitsJobs < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
drop_triggers
|
8
|
+
|
9
|
+
add_string_limit_if_missing :delayed_jobs, :queue
|
10
|
+
add_string_limit_if_missing :delayed_jobs, :locked_by
|
11
|
+
add_string_limit_if_missing :delayed_jobs, :tag
|
12
|
+
add_string_limit_if_missing :delayed_jobs, :strand
|
13
|
+
add_string_limit_if_missing :delayed_jobs, :source
|
14
|
+
|
15
|
+
add_string_limit_if_missing :failed_jobs, :queue
|
16
|
+
add_string_limit_if_missing :failed_jobs, :locked_by
|
17
|
+
add_string_limit_if_missing :failed_jobs, :tag
|
18
|
+
add_string_limit_if_missing :failed_jobs, :strand
|
19
|
+
add_string_limit_if_missing :failed_jobs, :source
|
20
|
+
|
21
|
+
readd_triggers
|
22
|
+
end
|
23
|
+
|
24
|
+
def drop_triggers
|
25
|
+
execute %(DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name})
|
26
|
+
execute %(DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name})
|
27
|
+
end
|
28
|
+
|
29
|
+
def readd_triggers
|
30
|
+
execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
|
31
|
+
execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
|
32
|
+
end
|
33
|
+
|
34
|
+
def add_string_limit_if_missing(table, column)
|
35
|
+
return if column_exists?(table, column, :string, limit: 255)
|
36
|
+
|
37
|
+
change_column table, column, :string, limit: 255
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
class SpeedUpMaxConcurrentTriggers < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
if connection.adapter_name == 'PostgreSQL'
|
8
|
+
# tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
|
9
|
+
# don't get the count every single time - it's usually safe to just set the next one in line
|
10
|
+
# since the max_concurrent doesn't change all that often for a strand
|
11
|
+
execute(<<-CODE)
|
12
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
13
|
+
DECLARE
|
14
|
+
running_count integer;
|
15
|
+
BEGIN
|
16
|
+
IF OLD.strand IS NOT NULL THEN
|
17
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
18
|
+
IF OLD.id % 20 = 0 THEN
|
19
|
+
running_count := (SELECT COUNT(*) FROM (
|
20
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
21
|
+
) subquery_for_count);
|
22
|
+
IF running_count < OLD.max_concurrent THEN
|
23
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
24
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
25
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
26
|
+
);
|
27
|
+
END IF;
|
28
|
+
ELSE
|
29
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
30
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
31
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
32
|
+
END IF;
|
33
|
+
END IF;
|
34
|
+
RETURN OLD;
|
35
|
+
END;
|
36
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
37
|
+
CODE
|
38
|
+
|
39
|
+
# don't need the full count on insert
|
40
|
+
execute(<<-CODE)
|
41
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
|
42
|
+
BEGIN
|
43
|
+
IF NEW.strand IS NOT NULL THEN
|
44
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
45
|
+
IF (SELECT COUNT(*) FROM (
|
46
|
+
SELECT 1 AS one FROM delayed_jobs WHERE strand = NEW.strand LIMIT NEW.max_concurrent
|
47
|
+
) subquery_for_count) = NEW.max_concurrent THEN
|
48
|
+
NEW.next_in_strand := 'f';
|
49
|
+
END IF;
|
50
|
+
END IF;
|
51
|
+
RETURN NEW;
|
52
|
+
END;
|
53
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
54
|
+
CODE
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def down
|
59
|
+
if connection.adapter_name == 'PostgreSQL'
|
60
|
+
execute(<<-CODE)
|
61
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
62
|
+
DECLARE
|
63
|
+
running_count integer;
|
64
|
+
BEGIN
|
65
|
+
IF OLD.strand IS NOT NULL THEN
|
66
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
67
|
+
running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
|
68
|
+
IF running_count < OLD.max_concurrent THEN
|
69
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
70
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
71
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
72
|
+
);
|
73
|
+
END IF;
|
74
|
+
END IF;
|
75
|
+
RETURN OLD;
|
76
|
+
END;
|
77
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
78
|
+
CODE
|
79
|
+
|
80
|
+
execute(<<-CODE)
|
81
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
|
82
|
+
BEGIN
|
83
|
+
IF NEW.strand IS NOT NULL THEN
|
84
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
85
|
+
IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
|
86
|
+
NEW.next_in_strand := 'f';
|
87
|
+
END IF;
|
88
|
+
END IF;
|
89
|
+
RETURN NEW;
|
90
|
+
END;
|
91
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
92
|
+
CODE
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class MakeCriticalColumnsNotNull < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
change_column_null :delayed_jobs, :run_at, false
|
8
|
+
change_column_null :delayed_jobs, :queue, false
|
9
|
+
end
|
10
|
+
|
11
|
+
def down
|
12
|
+
change_column_null :delayed_jobs, :run_at, true
|
13
|
+
change_column_null :delayed_jobs, :queue, true
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
class AddIdToGetDelayedJobsIndex < ActiveRecord::Migration[4.2]
|
2
|
+
disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
|
3
|
+
|
4
|
+
def connection
|
5
|
+
Delayed::Job.connection
|
6
|
+
end
|
7
|
+
|
8
|
+
def up
|
9
|
+
rename_index :delayed_jobs, 'get_delayed_jobs_index', 'get_delayed_jobs_index_old'
|
10
|
+
add_index :delayed_jobs, %i[queue priority run_at id],
|
11
|
+
algorithm: :concurrently,
|
12
|
+
where: 'locked_at IS NULL AND next_in_strand',
|
13
|
+
name: 'get_delayed_jobs_index'
|
14
|
+
remove_index :delayed_jobs, name: 'get_delayed_jobs_index_old'
|
15
|
+
end
|
16
|
+
|
17
|
+
def down
|
18
|
+
rename_index :delayed_jobs, 'get_delayed_jobs_index', 'get_delayed_jobs_index_old'
|
19
|
+
add_index :delayed_jobs, %i[priority run_at queue],
|
20
|
+
algorithm: :concurrently,
|
21
|
+
where: 'locked_at IS NULL AND next_in_strand',
|
22
|
+
name: 'get_delayed_jobs_index'
|
23
|
+
remove_index :delayed_jobs, name: 'get_delayed_jobs_index_old'
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
|
2
|
+
def connection
|
3
|
+
Delayed::Job.connection
|
4
|
+
end
|
5
|
+
|
6
|
+
def up
|
7
|
+
if connection.adapter_name == 'PostgreSQL'
|
8
|
+
# tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
|
9
|
+
# don't get the count every single time - it's usually safe to just set the next one in line
|
10
|
+
# since the max_concurrent doesn't change all that often for a strand
|
11
|
+
execute(<<-SQL)
|
12
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
13
|
+
DECLARE
|
14
|
+
running_count integer;
|
15
|
+
should_lock boolean;
|
16
|
+
should_be_precise boolean;
|
17
|
+
BEGIN
|
18
|
+
IF OLD.strand IS NOT NULL THEN
|
19
|
+
should_lock := true;
|
20
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
21
|
+
|
22
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
23
|
+
running_count := (SELECT COUNT(*) FROM (
|
24
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
25
|
+
) subquery_for_count);
|
26
|
+
should_lock := running_count < OLD.max_concurrent;
|
27
|
+
END IF;
|
28
|
+
|
29
|
+
IF should_lock THEN
|
30
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
31
|
+
END IF;
|
32
|
+
|
33
|
+
IF should_be_precise THEN
|
34
|
+
running_count := (SELECT COUNT(*) FROM (
|
35
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
36
|
+
) subquery_for_count);
|
37
|
+
IF running_count < OLD.max_concurrent THEN
|
38
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
39
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
40
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
41
|
+
);
|
42
|
+
END IF;
|
43
|
+
ELSE
|
44
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
45
|
+
IF OLD.max_concurrent > 1 THEN
|
46
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
47
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
48
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
49
|
+
ELSE
|
50
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
51
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
52
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
53
|
+
END IF;
|
54
|
+
END IF;
|
55
|
+
END IF;
|
56
|
+
RETURN OLD;
|
57
|
+
END;
|
58
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
59
|
+
SQL
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def down
|
64
|
+
if connection.adapter_name == 'PostgreSQL'
|
65
|
+
execute(<<-SQL)
|
66
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
67
|
+
DECLARE
|
68
|
+
running_count integer;
|
69
|
+
BEGIN
|
70
|
+
IF OLD.strand IS NOT NULL THEN
|
71
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
72
|
+
IF OLD.id % 20 = 0 THEN
|
73
|
+
running_count := (SELECT COUNT(*) FROM (
|
74
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
75
|
+
) subquery_for_count);
|
76
|
+
IF running_count < OLD.max_concurrent THEN
|
77
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
78
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
79
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
80
|
+
);
|
81
|
+
END IF;
|
82
|
+
ELSE
|
83
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
84
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
85
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
86
|
+
END IF;
|
87
|
+
END IF;
|
88
|
+
RETURN OLD;
|
89
|
+
END;
|
90
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
91
|
+
SQL
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,126 @@
|
|
1
|
+
class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
|
2
|
+
disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
|
3
|
+
|
4
|
+
def connection
|
5
|
+
Delayed::Job.connection
|
6
|
+
end
|
7
|
+
|
8
|
+
def up
|
9
|
+
add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
|
10
|
+
add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
|
11
|
+
add_index :delayed_jobs, %i[strand strand_order_override id],
|
12
|
+
algorithm: :concurrently,
|
13
|
+
where: 'strand IS NOT NULL',
|
14
|
+
name: 'next_in_strand_index'
|
15
|
+
|
16
|
+
if connection.adapter_name == 'PostgreSQL'
|
17
|
+
# Use the strand_order_override as the primary sorting mechanism (useful when moving between jobs queues without preserving ID ordering)
|
18
|
+
execute(<<-SQL)
|
19
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
20
|
+
DECLARE
|
21
|
+
running_count integer;
|
22
|
+
should_lock boolean;
|
23
|
+
should_be_precise boolean;
|
24
|
+
BEGIN
|
25
|
+
IF OLD.strand IS NOT NULL THEN
|
26
|
+
should_lock := true;
|
27
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
28
|
+
|
29
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
30
|
+
running_count := (SELECT COUNT(*) FROM (
|
31
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
32
|
+
) subquery_for_count);
|
33
|
+
should_lock := running_count < OLD.max_concurrent;
|
34
|
+
END IF;
|
35
|
+
|
36
|
+
IF should_lock THEN
|
37
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
38
|
+
END IF;
|
39
|
+
|
40
|
+
IF should_be_precise THEN
|
41
|
+
running_count := (SELECT COUNT(*) FROM (
|
42
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
43
|
+
) subquery_for_count);
|
44
|
+
IF running_count < OLD.max_concurrent THEN
|
45
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
46
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
47
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
48
|
+
);
|
49
|
+
END IF;
|
50
|
+
ELSE
|
51
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
52
|
+
IF OLD.max_concurrent > 1 THEN
|
53
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
54
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
55
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
56
|
+
ELSE
|
57
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
58
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
59
|
+
j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
|
60
|
+
END IF;
|
61
|
+
END IF;
|
62
|
+
END IF;
|
63
|
+
RETURN OLD;
|
64
|
+
END;
|
65
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
66
|
+
SQL
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def down
|
71
|
+
remove_column :delayed_jobs, :strand_order_override, :integer
|
72
|
+
remove_column :failed_jobs, :strand_order_override, :integer
|
73
|
+
|
74
|
+
if connection.adapter_name == 'PostgreSQL'
|
75
|
+
execute(<<-SQL)
|
76
|
+
CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
|
77
|
+
DECLARE
|
78
|
+
running_count integer;
|
79
|
+
should_lock boolean;
|
80
|
+
should_be_precise boolean;
|
81
|
+
BEGIN
|
82
|
+
IF OLD.strand IS NOT NULL THEN
|
83
|
+
should_lock := true;
|
84
|
+
should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
|
85
|
+
|
86
|
+
IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
|
87
|
+
running_count := (SELECT COUNT(*) FROM (
|
88
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
89
|
+
) subquery_for_count);
|
90
|
+
should_lock := running_count < OLD.max_concurrent;
|
91
|
+
END IF;
|
92
|
+
|
93
|
+
IF should_lock THEN
|
94
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
|
95
|
+
END IF;
|
96
|
+
|
97
|
+
IF should_be_precise THEN
|
98
|
+
running_count := (SELECT COUNT(*) FROM (
|
99
|
+
SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
|
100
|
+
) subquery_for_count);
|
101
|
+
IF running_count < OLD.max_concurrent THEN
|
102
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
|
103
|
+
SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
104
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
|
105
|
+
);
|
106
|
+
END IF;
|
107
|
+
ELSE
|
108
|
+
-- n-strands don't require precise ordering; we can make this query more performant
|
109
|
+
IF OLD.max_concurrent > 1 THEN
|
110
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
111
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
112
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
|
113
|
+
ELSE
|
114
|
+
UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
|
115
|
+
(SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
|
116
|
+
j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
|
117
|
+
END IF;
|
118
|
+
END IF;
|
119
|
+
END IF;
|
120
|
+
RETURN OLD;
|
121
|
+
END;
|
122
|
+
$$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
|
123
|
+
SQL
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|