switchman-inst-jobs 1.5.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/db/migrate/20101216224513_create_delayed_jobs.rb +42 -0
  3. data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +14 -0
  4. data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +13 -0
  5. data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +14 -0
  6. data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +26 -0
  7. data/db/migrate/20110610213249_optimize_delayed_jobs.rb +40 -0
  8. data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +52 -0
  9. data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +31 -0
  10. data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +15 -0
  11. data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +80 -0
  12. data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +15 -0
  13. data/db/migrate/20120608191051_add_jobs_run_at_index.rb +15 -0
  14. data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +13 -0
  15. data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +13 -0
  16. data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +12 -0
  17. data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +13 -0
  18. data/db/migrate/20140512213941_add_source_to_jobs.rb +15 -0
  19. data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +70 -0
  20. data/db/migrate/20151123210429_add_expires_at_to_jobs.rb +15 -0
  21. data/db/migrate/20151210162949_improve_max_concurrent.rb +50 -0
  22. data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +39 -0
  23. data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +95 -0
  24. data/db/migrate/20190726154743_make_critical_columns_not_null.rb +15 -0
  25. data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +25 -0
  26. data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +94 -0
  27. data/db/migrate/20200825011002_add_strand_order_override.rb +126 -0
  28. data/lib/switchman_inst_jobs.rb +3 -3
  29. data/lib/switchman_inst_jobs/active_record/migration.rb +10 -0
  30. data/lib/switchman_inst_jobs/delayed/backend/base.rb +9 -9
  31. data/lib/switchman_inst_jobs/delayed/message_sending.rb +2 -2
  32. data/lib/switchman_inst_jobs/delayed/worker/health_check.rb +1 -3
  33. data/lib/switchman_inst_jobs/engine.rb +2 -2
  34. data/lib/switchman_inst_jobs/{shackles.rb → guard_rail.rb} +1 -1
  35. data/lib/switchman_inst_jobs/jobs_migrator.rb +9 -7
  36. data/lib/switchman_inst_jobs/switchman/default_shard.rb +4 -0
  37. data/lib/switchman_inst_jobs/version.rb +1 -1
  38. metadata +45 -32
  39. data/db/migrate/20180628153808_set_search_paths_on_functions.rb +0 -15
@@ -0,0 +1,15 @@
1
+ class AddSourceToJobs < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :source, :string
8
+ add_column :failed_jobs, :source, :string
9
+ end
10
+
11
+ def down
12
+ remove_column :delayed_jobs, :source
13
+ remove_column :failed_jobs, :source
14
+ end
15
+ end
@@ -0,0 +1,70 @@
1
+ class AddMaxConcurrentToJobs < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :max_concurrent, :integer, default: 1, null: false
8
+
9
+ if connection.adapter_name == 'PostgreSQL'
10
+ execute(<<-CODE)
11
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
12
+ BEGIN
13
+ IF NEW.strand IS NOT NULL THEN
14
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
15
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
16
+ NEW.next_in_strand := 'f';
17
+ END IF;
18
+ END IF;
19
+ RETURN NEW;
20
+ END;
21
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
22
+ CODE
23
+
24
+ execute(<<-CODE)
25
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
26
+ BEGIN
27
+ IF OLD.strand IS NOT NULL THEN
28
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
29
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
30
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
31
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
32
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
33
+ );
34
+ END IF;
35
+ END IF;
36
+ RETURN OLD;
37
+ END;
38
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
39
+ CODE
40
+ end
41
+ end
42
+
43
+ def down
44
+ remove_column :delayed_jobs, :max_concurrent
45
+
46
+ if connection.adapter_name == 'PostgreSQL'
47
+ execute(<<-CODE)
48
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
49
+ BEGIN
50
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
51
+ IF (SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand LIMIT 1) = 1 THEN
52
+ NEW.next_in_strand := 'f';
53
+ END IF;
54
+ RETURN NEW;
55
+ END;
56
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
57
+ CODE
58
+
59
+ execute(<<-CODE)
60
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
61
+ BEGIN
62
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
63
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (SELECT id FROM delayed_jobs j2 WHERE j2.strand = OLD.strand ORDER BY j2.strand, j2.id ASC LIMIT 1 FOR UPDATE);
64
+ RETURN OLD;
65
+ END;
66
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
67
+ CODE
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,15 @@
1
+ class AddExpiresAtToJobs < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ add_column :delayed_jobs, :expires_at, :datetime
8
+ add_column :failed_jobs, :expires_at, :datetime
9
+ end
10
+
11
+ def down
12
+ remove_column :delayed_jobs, :expires_at
13
+ remove_column :failed_jobs, :expires_at
14
+ end
15
+ end
@@ -0,0 +1,50 @@
1
+ class ImproveMaxConcurrent < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ execute(<<-CODE)
9
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
10
+ DECLARE
11
+ running_count integer;
12
+ BEGIN
13
+ IF OLD.strand IS NOT NULL THEN
14
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
15
+ running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
16
+ IF running_count < OLD.max_concurrent THEN
17
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
18
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
19
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
20
+ );
21
+ END IF;
22
+ END IF;
23
+ RETURN OLD;
24
+ END;
25
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
26
+ CODE
27
+ end
28
+ end
29
+
30
+ def down
31
+ if connection.adapter_name == 'PostgreSQL'
32
+ execute(<<-CODE)
33
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
34
+ BEGIN
35
+ IF OLD.strand IS NOT NULL THEN
36
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
37
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't') < OLD.max_concurrent THEN
38
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id = (
39
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
40
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE
41
+ );
42
+ END IF;
43
+ END IF;
44
+ RETURN OLD;
45
+ END;
46
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
47
+ CODE
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,39 @@
1
+ class AddBackDefaultStringLimitsJobs < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ drop_triggers
8
+
9
+ add_string_limit_if_missing :delayed_jobs, :queue
10
+ add_string_limit_if_missing :delayed_jobs, :locked_by
11
+ add_string_limit_if_missing :delayed_jobs, :tag
12
+ add_string_limit_if_missing :delayed_jobs, :strand
13
+ add_string_limit_if_missing :delayed_jobs, :source
14
+
15
+ add_string_limit_if_missing :failed_jobs, :queue
16
+ add_string_limit_if_missing :failed_jobs, :locked_by
17
+ add_string_limit_if_missing :failed_jobs, :tag
18
+ add_string_limit_if_missing :failed_jobs, :strand
19
+ add_string_limit_if_missing :failed_jobs, :source
20
+
21
+ readd_triggers
22
+ end
23
+
24
+ def drop_triggers
25
+ execute %(DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name})
26
+ execute %(DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name})
27
+ end
28
+
29
+ def readd_triggers
30
+ execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
31
+ execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
32
+ end
33
+
34
+ def add_string_limit_if_missing(table, column)
35
+ return if column_exists?(table, column, :string, limit: 255)
36
+
37
+ change_column table, column, :string, limit: 255
38
+ end
39
+ end
@@ -0,0 +1,95 @@
1
+ class SpeedUpMaxConcurrentTriggers < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
9
+ # don't get the count every single time - it's usually safe to just set the next one in line
10
+ # since the max_concurrent doesn't change all that often for a strand
11
+ execute(<<-CODE)
12
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
13
+ DECLARE
14
+ running_count integer;
15
+ BEGIN
16
+ IF OLD.strand IS NOT NULL THEN
17
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
18
+ IF OLD.id % 20 = 0 THEN
19
+ running_count := (SELECT COUNT(*) FROM (
20
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
21
+ ) subquery_for_count);
22
+ IF running_count < OLD.max_concurrent THEN
23
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
24
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
25
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
26
+ );
27
+ END IF;
28
+ ELSE
29
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
30
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
31
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
32
+ END IF;
33
+ END IF;
34
+ RETURN OLD;
35
+ END;
36
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
37
+ CODE
38
+
39
+ # don't need the full count on insert
40
+ execute(<<-CODE)
41
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
42
+ BEGIN
43
+ IF NEW.strand IS NOT NULL THEN
44
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
45
+ IF (SELECT COUNT(*) FROM (
46
+ SELECT 1 AS one FROM delayed_jobs WHERE strand = NEW.strand LIMIT NEW.max_concurrent
47
+ ) subquery_for_count) = NEW.max_concurrent THEN
48
+ NEW.next_in_strand := 'f';
49
+ END IF;
50
+ END IF;
51
+ RETURN NEW;
52
+ END;
53
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
54
+ CODE
55
+ end
56
+ end
57
+
58
+ def down
59
+ if connection.adapter_name == 'PostgreSQL'
60
+ execute(<<-CODE)
61
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
62
+ DECLARE
63
+ running_count integer;
64
+ BEGIN
65
+ IF OLD.strand IS NOT NULL THEN
66
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
67
+ running_count := (SELECT COUNT(*) FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't');
68
+ IF running_count < OLD.max_concurrent THEN
69
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
70
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
71
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
72
+ );
73
+ END IF;
74
+ END IF;
75
+ RETURN OLD;
76
+ END;
77
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
78
+ CODE
79
+
80
+ execute(<<-CODE)
81
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
82
+ BEGIN
83
+ IF NEW.strand IS NOT NULL THEN
84
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
85
+ IF (SELECT COUNT(*) FROM delayed_jobs WHERE strand = NEW.strand) >= NEW.max_concurrent THEN
86
+ NEW.next_in_strand := 'f';
87
+ END IF;
88
+ END IF;
89
+ RETURN NEW;
90
+ END;
91
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
92
+ CODE
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,15 @@
1
+ class MakeCriticalColumnsNotNull < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ change_column_null :delayed_jobs, :run_at, false
8
+ change_column_null :delayed_jobs, :queue, false
9
+ end
10
+
11
+ def down
12
+ change_column_null :delayed_jobs, :run_at, true
13
+ change_column_null :delayed_jobs, :queue, true
14
+ end
15
+ end
@@ -0,0 +1,25 @@
1
+ class AddIdToGetDelayedJobsIndex < ActiveRecord::Migration[4.2]
2
+ disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
3
+
4
+ def connection
5
+ Delayed::Job.connection
6
+ end
7
+
8
+ def up
9
+ rename_index :delayed_jobs, 'get_delayed_jobs_index', 'get_delayed_jobs_index_old'
10
+ add_index :delayed_jobs, %i[queue priority run_at id],
11
+ algorithm: :concurrently,
12
+ where: 'locked_at IS NULL AND next_in_strand',
13
+ name: 'get_delayed_jobs_index'
14
+ remove_index :delayed_jobs, name: 'get_delayed_jobs_index_old'
15
+ end
16
+
17
+ def down
18
+ rename_index :delayed_jobs, 'get_delayed_jobs_index', 'get_delayed_jobs_index_old'
19
+ add_index :delayed_jobs, %i[priority run_at queue],
20
+ algorithm: :concurrently,
21
+ where: 'locked_at IS NULL AND next_in_strand',
22
+ name: 'get_delayed_jobs_index'
23
+ remove_index :delayed_jobs, name: 'get_delayed_jobs_index_old'
24
+ end
25
+ end
@@ -0,0 +1,94 @@
1
+ class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
2
+ def connection
3
+ Delayed::Job.connection
4
+ end
5
+
6
+ def up
7
+ if connection.adapter_name == 'PostgreSQL'
8
+ # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
9
+ # don't get the count every single time - it's usually safe to just set the next one in line
10
+ # since the max_concurrent doesn't change all that often for a strand
11
+ execute(<<-SQL)
12
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
13
+ DECLARE
14
+ running_count integer;
15
+ should_lock boolean;
16
+ should_be_precise boolean;
17
+ BEGIN
18
+ IF OLD.strand IS NOT NULL THEN
19
+ should_lock := true;
20
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
21
+
22
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
23
+ running_count := (SELECT COUNT(*) FROM (
24
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
25
+ ) subquery_for_count);
26
+ should_lock := running_count < OLD.max_concurrent;
27
+ END IF;
28
+
29
+ IF should_lock THEN
30
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
31
+ END IF;
32
+
33
+ IF should_be_precise THEN
34
+ running_count := (SELECT COUNT(*) FROM (
35
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
36
+ ) subquery_for_count);
37
+ IF running_count < OLD.max_concurrent THEN
38
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
39
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
40
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
41
+ );
42
+ END IF;
43
+ ELSE
44
+ -- n-strands don't require precise ordering; we can make this query more performant
45
+ IF OLD.max_concurrent > 1 THEN
46
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
47
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
48
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
49
+ ELSE
50
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
51
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
52
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
53
+ END IF;
54
+ END IF;
55
+ END IF;
56
+ RETURN OLD;
57
+ END;
58
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
59
+ SQL
60
+ end
61
+ end
62
+
63
+ def down
64
+ if connection.adapter_name == 'PostgreSQL'
65
+ execute(<<-SQL)
66
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
67
+ DECLARE
68
+ running_count integer;
69
+ BEGIN
70
+ IF OLD.strand IS NOT NULL THEN
71
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
72
+ IF OLD.id % 20 = 0 THEN
73
+ running_count := (SELECT COUNT(*) FROM (
74
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
75
+ ) subquery_for_count);
76
+ IF running_count < OLD.max_concurrent THEN
77
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
78
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
79
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
80
+ );
81
+ END IF;
82
+ ELSE
83
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
84
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
85
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
86
+ END IF;
87
+ END IF;
88
+ RETURN OLD;
89
+ END;
90
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
91
+ SQL
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,126 @@
1
+ class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
2
+ disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
3
+
4
+ def connection
5
+ Delayed::Job.connection
6
+ end
7
+
8
+ def up
9
+ add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
10
+ add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
11
+ add_index :delayed_jobs, %i[strand strand_order_override id],
12
+ algorithm: :concurrently,
13
+ where: 'strand IS NOT NULL',
14
+ name: 'next_in_strand_index'
15
+
16
+ if connection.adapter_name == 'PostgreSQL'
17
+ # Use the strand_order_override as the primary sorting mechanism (useful when moving between jobs queues without preserving ID ordering)
18
+ execute(<<-SQL)
19
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
20
+ DECLARE
21
+ running_count integer;
22
+ should_lock boolean;
23
+ should_be_precise boolean;
24
+ BEGIN
25
+ IF OLD.strand IS NOT NULL THEN
26
+ should_lock := true;
27
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
28
+
29
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
30
+ running_count := (SELECT COUNT(*) FROM (
31
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
32
+ ) subquery_for_count);
33
+ should_lock := running_count < OLD.max_concurrent;
34
+ END IF;
35
+
36
+ IF should_lock THEN
37
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
38
+ END IF;
39
+
40
+ IF should_be_precise THEN
41
+ running_count := (SELECT COUNT(*) FROM (
42
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
43
+ ) subquery_for_count);
44
+ IF running_count < OLD.max_concurrent THEN
45
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
46
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
47
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
48
+ );
49
+ END IF;
50
+ ELSE
51
+ -- n-strands don't require precise ordering; we can make this query more performant
52
+ IF OLD.max_concurrent > 1 THEN
53
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
54
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
55
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
56
+ ELSE
57
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
58
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
59
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
60
+ END IF;
61
+ END IF;
62
+ END IF;
63
+ RETURN OLD;
64
+ END;
65
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
66
+ SQL
67
+ end
68
+ end
69
+
70
+ def down
71
+ remove_column :delayed_jobs, :strand_order_override, :integer
72
+ remove_column :failed_jobs, :strand_order_override, :integer
73
+
74
+ if connection.adapter_name == 'PostgreSQL'
75
+ execute(<<-SQL)
76
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
77
+ DECLARE
78
+ running_count integer;
79
+ should_lock boolean;
80
+ should_be_precise boolean;
81
+ BEGIN
82
+ IF OLD.strand IS NOT NULL THEN
83
+ should_lock := true;
84
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
85
+
86
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
87
+ running_count := (SELECT COUNT(*) FROM (
88
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
89
+ ) subquery_for_count);
90
+ should_lock := running_count < OLD.max_concurrent;
91
+ END IF;
92
+
93
+ IF should_lock THEN
94
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
95
+ END IF;
96
+
97
+ IF should_be_precise THEN
98
+ running_count := (SELECT COUNT(*) FROM (
99
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
100
+ ) subquery_for_count);
101
+ IF running_count < OLD.max_concurrent THEN
102
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
103
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
104
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
105
+ );
106
+ END IF;
107
+ ELSE
108
+ -- n-strands don't require precise ordering; we can make this query more performant
109
+ IF OLD.max_concurrent > 1 THEN
110
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
111
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
112
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
113
+ ELSE
114
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
115
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
116
+ j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
117
+ END IF;
118
+ END IF;
119
+ END IF;
120
+ RETURN OLD;
121
+ END;
122
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
123
+ SQL
124
+ end
125
+ end
126
+ end