switchman-inst-jobs 1.5.1 → 3.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/db/migrate/20101216224513_create_delayed_jobs.rb +42 -0
  3. data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +14 -0
  4. data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +13 -0
  5. data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +14 -0
  6. data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +26 -0
  7. data/db/migrate/20110610213249_optimize_delayed_jobs.rb +40 -0
  8. data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +52 -0
  9. data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +31 -0
  10. data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +15 -0
  11. data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +80 -0
  12. data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +15 -0
  13. data/db/migrate/20120608191051_add_jobs_run_at_index.rb +15 -0
  14. data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +13 -0
  15. data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +13 -0
  16. data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +12 -0
  17. data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +13 -0
  18. data/db/migrate/20140512213941_add_source_to_jobs.rb +15 -0
  19. data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +70 -0
  20. data/db/migrate/20151123210429_add_expires_at_to_jobs.rb +15 -0
  21. data/db/migrate/20151210162949_improve_max_concurrent.rb +50 -0
  22. data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +39 -0
  23. data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +95 -0
  24. data/db/migrate/20190726154743_make_critical_columns_not_null.rb +15 -0
  25. data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +25 -0
  26. data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +94 -0
  27. data/db/migrate/20200825011002_add_strand_order_override.rb +126 -0
  28. data/lib/switchman_inst_jobs.rb +3 -3
  29. data/lib/switchman_inst_jobs/active_record/migration.rb +10 -0
  30. data/lib/switchman_inst_jobs/delayed/backend/base.rb +9 -9
  31. data/lib/switchman_inst_jobs/delayed/message_sending.rb +3 -2
  32. data/lib/switchman_inst_jobs/delayed/worker/health_check.rb +1 -3
  33. data/lib/switchman_inst_jobs/engine.rb +2 -2
  34. data/lib/switchman_inst_jobs/{shackles.rb → guard_rail.rb} +1 -1
  35. data/lib/switchman_inst_jobs/jobs_migrator.rb +9 -7
  36. data/lib/switchman_inst_jobs/version.rb +1 -1
  37. metadata +49 -30
  38. data/db/migrate/20180628153808_set_search_paths_on_functions.rb +0 -15
@@ -0,0 +1,15 @@
1
+ class AddSourceToJobs < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :source, :string
8
+ add_column :failed_jobs, :source, :string
9
+ end
10
+
11
+ def down
12
+ remove_column :delayed_jobs, :source
13
+ remove_column :failed_jobs, :source
14
+ end
15
+ end
@@ -0,0 +1,70 @@
1
+ class AddMaxConcurrentToJobs < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :max_concurrent, :integer, default: 1, null: false
8
+
9
+ if connection.adapter_name == 'PostgreSQL'
10
+ execute(<<-CODE)
11
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
12
+ BEGIN
13
+ IF NEW.strand IS NOT NULL THEN
14
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
15
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
16
+ NEW.next_in_strand := 'f';
17
+ END IF;
18
+ END IF;
19
+ RETURN NEW;
20
+ END;
21
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
22
+ CODE
23
+
24
+ execute(<<-CODE)
25
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
26
+ BEGIN
27
+ IF OLD.strand IS NOT NULL THEN
28
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
29
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
30
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
31
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
32
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
33
+ );
34
+ END IF;
35
+ END IF;
36
+ RETURN OLD;
37
+ END;
38
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
39
+ CODE
40
+ end
41
+ end
42
+
43
+ def down
44
+ remove_column :delayed_jobs, :max_concurrent
45
+
46
+ if connection.adapter_name == 'PostgreSQL'
47
+ execute(<<-CODE)
48
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
49
+ BEGIN
50
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
51
+ IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
52
+ NEW.next_in_strand := 'f';
53
+ END IF;
54
+ RETURN NEW;
55
+ END;
56
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
57
+ CODE
58
+
59
+ execute(<<-CODE)
60
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
61
+ BEGIN
62
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
63
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
64
+ RETURN OLD;
65
+ END;
66
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
67
+ CODE
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,15 @@
1
+ class AddExpiresAtToJobs < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :expires_at, :datetime
8
+ add_column :failed_jobs, :expires_at, :datetime
9
+ end
10
+
11
+ def down
12
+ remove_column :delayed_jobs, :expires_at
13
+ remove_column :failed_jobs, :expires_at
14
+ end
15
+ end
@@ -0,0 +1,50 @@
1
+ class ImproveMaxConcurrent < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ execute(<<-CODE)
9
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
10
+ DECLARE
11
+ running_count integer;
12
+ BEGIN
13
+ IF OLD.strand IS NOT NULL THEN
14
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
15
+ running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
16
+ IF running_count < OLD.max_concurrent THEN
17
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
18
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
19
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
20
+ );
21
+ END IF;
22
+ END IF;
23
+ RETURN OLD;
24
+ END;
25
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
26
+ CODE
27
+ end
28
+ end
29
+
30
+ def down
31
+ if connection.adapter_name == 'PostgreSQL'
32
+ execute(<<-CODE)
33
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
34
+ BEGIN
35
+ IF OLD.strand IS NOT NULL THEN
36
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
37
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
38
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
39
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
40
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
41
+ );
42
+ END IF;
43
+ END IF;
44
+ RETURN OLD;
45
+ END;
46
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
47
+ CODE
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,39 @@
1
+ class AddBackDefaultStringLimitsJobs < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ drop_triggers
8
+
9
+ add_string_limit_if_missing :delayed_jobs, :queue
10
+ add_string_limit_if_missing :delayed_jobs, :locked_by
11
+ add_string_limit_if_missing :delayed_jobs, :tag
12
+ add_string_limit_if_missing :delayed_jobs, :strand
13
+ add_string_limit_if_missing :delayed_jobs, :source
14
+
15
+ add_string_limit_if_missing :failed_jobs, :queue
16
+ add_string_limit_if_missing :failed_jobs, :locked_by
17
+ add_string_limit_if_missing :failed_jobs, :tag
18
+ add_string_limit_if_missing :failed_jobs, :strand
19
+ add_string_limit_if_missing :failed_jobs, :source
20
+
21
+ readd_triggers
22
+ end
23
+
24
+ def drop_triggers
25
+ execute %(DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name})
26
+ execute %(DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name})
27
+ end
28
+
29
+ def readd_triggers
30
+ execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
31
+ execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
32
+ end
33
+
34
+ def add_string_limit_if_missing(table, column)
35
+ return if column_exists?(table, column, :string, limit: 255)
36
+
37
+ change_column table, column, :string, limit: 255
38
+ end
39
+ end
@@ -0,0 +1,95 @@
1
+ class SpeedUpMaxConcurrentTriggers < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
9
+ # don't get the count every single time - it's usually safe to just set the next one in line
10
+ # since the max_concurrent doesn't change all that often for a strand
11
+ execute(<<-CODE)
12
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
13
+ DECLARE
14
+ running_count integer;
15
+ BEGIN
16
+ IF OLD.strand IS NOT NULL THEN
17
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
18
+ IF OLD.id % 20 = 0 THEN
19
+ running_count := (SELECT COUNT(*) FROM (
20
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
21
+ ) subquery_for_count);
22
+ IF running_count < OLD.max_concurrent THEN
23
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
24
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
25
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
26
+ );
27
+ END IF;
28
+ ELSE
29
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
30
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
31
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
32
+ END IF;
33
+ END IF;
34
+ RETURN OLD;
35
+ END;
36
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
37
+ CODE
38
+
39
+ # don't need the full count on insert
40
+ execute(<<-CODE)
41
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
42
+ BEGIN
43
+ IF NEW.strand IS NOT NULL THEN
44
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
45
+ IF (SELECT COUNT(*) FROM (
46
+ SELECT 1 AS one FROM delayed_jobs WHERE strand = NEW.strand LIMIT NEW.max_concurrent
47
+ ) subquery_for_count) = NEW.max_concurrent THEN
48
+ NEW.next_in_strand := 'f';
49
+ END IF;
50
+ END IF;
51
+ RETURN NEW;
52
+ END;
53
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
54
+ CODE
55
+ end
56
+ end
57
+
58
+ def down
59
+ if connection.adapter_name == 'PostgreSQL'
60
+ execute(<<-CODE)
61
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
62
+ DECLARE
63
+ running_count integer;
64
+ BEGIN
65
+ IF OLD.strand IS NOT NULL THEN
66
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
67
+ running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
68
+ IF running_count < OLD.max_concurrent THEN
69
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
70
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
71
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
72
+ );
73
+ END IF;
74
+ END IF;
75
+ RETURN OLD;
76
+ END;
77
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
78
+ CODE
79
+
80
+ execute(<<-CODE)
81
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
82
+ BEGIN
83
+ IF NEW.strand IS NOT NULL THEN
84
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
85
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
86
+ NEW.next_in_strand := 'f';
87
+ END IF;
88
+ END IF;
89
+ RETURN NEW;
90
+ END;
91
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
92
+ CODE
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,15 @@
1
+ class MakeCriticalColumnsNotNull < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ change_column_null :delayed_jobs, :run_at, false
8
+ change_column_null :delayed_jobs, :queue, false
9
+ end
10
+
11
+ def down
12
+ change_column_null :delayed_jobs, :run_at, true
13
+ change_column_null :delayed_jobs, :queue, true
14
+ end
15
+ end
@@ -0,0 +1,25 @@
1
+ class AddIdToGetDelayedJobsIndex < ActiveRecord::Migration[4.2]
2
+ disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
3
+
4
+ def connection
5
+ Delayed::Job.connection
6
+ end
7
+
8
+ def up
9
+ rename_index :delayed_jobs, 'get_delayed_jobs_index', 'get_delayed_jobs_index_old'
10
+ add_index :delayed_jobs, %i[queue priority run_at id],
11
+ algorithm: :concurrently,
12
+ where: 'locked_at IS NULL AND next_in_strand',
13
+ name: 'get_delayed_jobs_index'
14
+ remove_index :delayed_jobs, name: 'get_delayed_jobs_index_old'
15
+ end
16
+
17
+ def down
18
+ rename_index :delayed_jobs, 'get_delayed_jobs_index', 'get_delayed_jobs_index_old'
19
+ add_index :delayed_jobs, %i[priority run_at queue],
20
+ algorithm: :concurrently,
21
+ where: 'locked_at IS NULL AND next_in_strand',
22
+ name: 'get_delayed_jobs_index'
23
+ remove_index :delayed_jobs, name: 'get_delayed_jobs_index_old'
24
+ end
25
+ end
@@ -0,0 +1,94 @@
1
+ class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
9
+ # don't get the count every single time - it's usually safe to just set the next one in line
10
+ # since the max_concurrent doesn't change all that often for a strand
11
+ execute(<<-SQL)
12
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
13
+ DECLARE
14
+ running_count integer;
15
+ should_lock boolean;
16
+ should_be_precise boolean;
17
+ BEGIN
18
+ IF OLD.strand IS NOT NULL THEN
19
+ should_lock := true;
20
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
21
+
22
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
23
+ running_count := (SELECT COUNT(*) FROM (
24
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
25
+ ) subquery_for_count);
26
+ should_lock := running_count < OLD.max_concurrent;
27
+ END IF;
28
+
29
+ IF should_lock THEN
30
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
31
+ END IF;
32
+
33
+ IF should_be_precise THEN
34
+ running_count := (SELECT COUNT(*) FROM (
35
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
36
+ ) subquery_for_count);
37
+ IF running_count < OLD.max_concurrent THEN
38
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
39
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
40
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
41
+ );
42
+ END IF;
43
+ ELSE
44
+ -- n-strands don't require precise ordering; we can make this query more performant
45
+ IF OLD.max_concurrent > 1 THEN
46
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
47
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
48
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
49
+ ELSE
50
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
51
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
52
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
53
+ END IF;
54
+ END IF;
55
+ END IF;
56
+ RETURN OLD;
57
+ END;
58
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
59
+ SQL
60
+ end
61
+ end
62
+
63
+ def down
64
+ if connection.adapter_name == 'PostgreSQL'
65
+ execute(<<-SQL)
66
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
67
+ DECLARE
68
+ running_count integer;
69
+ BEGIN
70
+ IF OLD.strand IS NOT NULL THEN
71
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
72
+ IF OLD.id % 20 = 0 THEN
73
+ running_count := (SELECT COUNT(*) FROM (
74
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
75
+ ) subquery_for_count);
76
+ IF running_count < OLD.max_concurrent THEN
77
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
78
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
79
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
80
+ );
81
+ END IF;
82
+ ELSE
83
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
84
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
85
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
86
+ END IF;
87
+ END IF;
88
+ RETURN OLD;
89
+ END;
90
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
91
+ SQL
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,126 @@
1
+ class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
2
+ disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
3
+
4
+ def connection
5
+ Delayed::Job.connection
6
+ end
7
+
8
+ def up
9
+ add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
10
+ add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
11
+ add_index :delayed_jobs, %i[strand strand_order_override id],
12
+ algorithm: :concurrently,
13
+ where: 'strand IS NOT NULL',
14
+ name: 'next_in_strand_index'
15
+
16
+ if connection.adapter_name == 'PostgreSQL'
17
+ # Use the strand_order_override as the primary sorting mechanism (useful when moving between jobs queues without preserving ID ordering)
18
+ execute(<<-SQL)
19
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
20
+ DECLARE
21
+ running_count integer;
22
+ should_lock boolean;
23
+ should_be_precise boolean;
24
+ BEGIN
25
+ IF OLD.strand IS NOT NULL THEN
26
+ should_lock := true;
27
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
28
+
29
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
30
+ running_count := (SELECT COUNT(*) FROM (
31
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
32
+ ) subquery_for_count);
33
+ should_lock := running_count < OLD.max_concurrent;
34
+ END IF;
35
+
36
+ IF should_lock THEN
37
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
38
+ END IF;
39
+
40
+ IF should_be_precise THEN
41
+ running_count := (SELECT COUNT(*) FROM (
42
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
43
+ ) subquery_for_count);
44
+ IF running_count < OLD.max_concurrent THEN
45
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
46
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
47
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
48
+ );
49
+ END IF;
50
+ ELSE
51
+ -- n-strands don't require precise ordering; we can make this query more performant
52
+ IF OLD.max_concurrent > 1 THEN
53
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
54
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
55
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
56
+ ELSE
57
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
58
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
59
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
60
+ END IF;
61
+ END IF;
62
+ END IF;
63
+ RETURN OLD;
64
+ END;
65
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
66
+ SQL
67
+ end
68
+ end
69
+
70
+ def down
71
+ remove_column :delayed_jobs, :strand_order_override, :integer
72
+ remove_column :failed_jobs, :strand_order_override, :integer
73
+
74
+ if connection.adapter_name == 'PostgreSQL'
75
+ execute(<<-SQL)
76
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
77
+ DECLARE
78
+ running_count integer;
79
+ should_lock boolean;
80
+ should_be_precise boolean;
81
+ BEGIN
82
+ IF OLD.strand IS NOT NULL THEN
83
+ should_lock := true;
84
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
85
+
86
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
87
+ running_count := (SELECT COUNT(*) FROM (
88
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
89
+ ) subquery_for_count);
90
+ should_lock := running_count < OLD.max_concurrent;
91
+ END IF;
92
+
93
+ IF should_lock THEN
94
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
95
+ END IF;
96
+
97
+ IF should_be_precise THEN
98
+ running_count := (SELECT COUNT(*) FROM (
99
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
100
+ ) subquery_for_count);
101
+ IF running_count < OLD.max_concurrent THEN
102
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
103
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
104
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
105
+ );
106
+ END IF;
107
+ ELSE
108
+ -- n-strands don't require precise ordering; we can make this query more performant
109
+ IF OLD.max_concurrent > 1 THEN
110
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
111
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
112
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
113
+ ELSE
114
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
115
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
116
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
117
+ END IF;
118
+ END IF;
119
+ END IF;
120
+ RETURN OLD;
121
+ END;
122
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
123
+ SQL
124
+ end
125
+ end
126
+ end