switchman-inst-jobs 3.2.6 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/db/migrate/20101216224513_create_delayed_jobs.rb +1 -1
  3. data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +1 -1
  4. data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +1 -1
  5. data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +1 -1
  6. data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +1 -1
  7. data/db/migrate/20110610213249_optimize_delayed_jobs.rb +1 -1
  8. data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +1 -1
  9. data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +1 -1
  10. data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +1 -1
  11. data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +1 -1
  12. data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +2 -2
  13. data/db/migrate/20120608191051_add_jobs_run_at_index.rb +2 -2
  14. data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +1 -1
  15. data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +1 -1
  16. data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +2 -2
  17. data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +1 -1
  18. data/db/migrate/20140512213941_add_source_to_jobs.rb +1 -1
  19. data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +1 -1
  20. data/db/migrate/20151123210429_add_expires_at_to_jobs.rb +1 -1
  21. data/db/migrate/20151210162949_improve_max_concurrent.rb +1 -1
  22. data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +1 -1
  23. data/db/migrate/20170308045400_add_shard_id_to_delayed_jobs.rb +1 -11
  24. data/db/migrate/20170308045401_add_delayed_jobs_shard_id_to_switchman_shards.rb +5 -0
  25. data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +1 -1
  26. data/db/migrate/20190726154743_make_critical_columns_not_null.rb +1 -1
  27. data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +2 -2
  28. data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +1 -1
  29. data/db/migrate/20200825011002_add_strand_order_override.rb +2 -2
  30. data/lib/switchman_inst_jobs/active_record/connection_adapters/connection_pool.rb +15 -0
  31. data/lib/switchman_inst_jobs/delayed/backend/base.rb +10 -9
  32. data/lib/switchman_inst_jobs/delayed/pool.rb +1 -1
  33. data/lib/switchman_inst_jobs/delayed/worker/health_check.rb +10 -12
  34. data/lib/switchman_inst_jobs/delayed/worker.rb +2 -2
  35. data/lib/switchman_inst_jobs/engine.rb +6 -4
  36. data/lib/switchman_inst_jobs/jobs_migrator.rb +23 -21
  37. data/lib/switchman_inst_jobs/switchman/shard.rb +8 -21
  38. data/lib/switchman_inst_jobs/version.rb +1 -1
  39. data/lib/switchman_inst_jobs.rb +8 -0
  40. metadata +44 -45
  41. data/db/migrate/20210809145804_add_n_strand_index.rb +0 -12
  42. data/db/migrate/20210812210128_add_singleton_column.rb +0 -200
  43. data/db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb +0 -27
  44. data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb +0 -56
  45. data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb +0 -27
  46. data/db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb +0 -137
  47. data/db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb +0 -171
data/db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb DELETED
@@ -1,137 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class UpdateAfterDeleteTriggerForSingletonIndex < ActiveRecord::Migration[5.2]
4
- def up
5
- execute(<<~SQL)
6
- CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
7
- DECLARE
8
- running_count integer;
9
- should_lock boolean;
10
- should_be_precise boolean;
11
- update_query varchar;
12
- skip_locked varchar;
13
- BEGIN
14
- IF OLD.strand IS NOT NULL THEN
15
- should_lock := true;
16
- should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
17
-
18
- IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
19
- running_count := (SELECT COUNT(*) FROM (
20
- SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
21
- ) subquery_for_count);
22
- should_lock := running_count < OLD.max_concurrent;
23
- END IF;
24
-
25
- IF should_lock THEN
26
- PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
27
- END IF;
28
-
29
- -- note that we don't really care if the row we're deleting has a singleton, or if it even
30
- -- matches the row(s) we're going to update. we just need to make sure that whatever
31
- -- singleton we grab isn't already running (which is a simple existence check, since
32
- -- the unique indexes ensure there is at most one singleton running, and one queued)
33
- update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
34
- SELECT id FROM delayed_jobs j2
35
- WHERE next_in_strand=false AND
36
- j2.strand=$1.strand AND
37
- (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
38
- ORDER BY j2.strand_order_override ASC, j2.id ASC
39
- LIMIT ';
40
-
41
- IF should_be_precise THEN
42
- running_count := (SELECT COUNT(*) FROM (
43
- SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
44
- ) s);
45
- IF running_count < OLD.max_concurrent THEN
46
- update_query := update_query || '($1.max_concurrent - $2)';
47
- ELSE
48
- -- we have too many running already; just bail
49
- RETURN OLD;
50
- END IF;
51
- ELSE
52
- update_query := update_query || '1';
53
-
54
- -- n-strands don't require precise ordering; we can make this query more performant
55
- IF OLD.max_concurrent > 1 THEN
56
- skip_locked := ' SKIP LOCKED';
57
- END IF;
58
- END IF;
59
-
60
- update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
61
- EXECUTE update_query USING OLD, running_count;
62
- ELSIF OLD.singleton IS NOT NULL THEN
63
- UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
64
- END IF;
65
- RETURN OLD;
66
- END;
67
- $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
68
- SQL
69
- end
70
-
71
- def down
72
- execute(<<~SQL)
73
- CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
74
- DECLARE
75
- running_count integer;
76
- should_lock boolean;
77
- should_be_precise boolean;
78
- update_query varchar;
79
- skip_locked varchar;
80
- BEGIN
81
- IF OLD.strand IS NOT NULL THEN
82
- should_lock := true;
83
- should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
84
-
85
- IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
86
- running_count := (SELECT COUNT(*) FROM (
87
- SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
88
- ) subquery_for_count);
89
- should_lock := running_count < OLD.max_concurrent;
90
- END IF;
91
-
92
- IF should_lock THEN
93
- PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
94
- END IF;
95
-
96
- -- note that we don't really care if the row we're deleting has a singleton, or if it even
97
- -- matches the row(s) we're going to update. we just need to make sure that whatever
98
- -- singleton we grab isn't already running (which is a simple existence check, since
99
- -- the unique indexes ensure there is at most one singleton running, and one queued)
100
- update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
101
- SELECT id FROM delayed_jobs j2
102
- WHERE next_in_strand=false AND
103
- j2.strand=$1.strand AND
104
- (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id))
105
- ORDER BY j2.strand_order_override ASC, j2.id ASC
106
- LIMIT ';
107
-
108
- IF should_be_precise THEN
109
- running_count := (SELECT COUNT(*) FROM (
110
- SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
111
- ) s);
112
- IF running_count < OLD.max_concurrent THEN
113
- update_query := update_query || '($1.max_concurrent - $2)';
114
- ELSE
115
- -- we have too many running already; just bail
116
- RETURN OLD;
117
- END IF;
118
- ELSE
119
- update_query := update_query || '1';
120
-
121
- -- n-strands don't require precise ordering; we can make this query more performant
122
- IF OLD.max_concurrent > 1 THEN
123
- skip_locked := ' SKIP LOCKED';
124
- END IF;
125
- END IF;
126
-
127
- update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
128
- EXECUTE update_query USING OLD, running_count;
129
- ELSIF OLD.singleton IS NOT NULL THEN
130
- UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false;
131
- END IF;
132
- RETURN OLD;
133
- END;
134
- $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
135
- SQL
136
- end
137
- end
data/db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb DELETED
@@ -1,171 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- class UpdateAfterDeleteTriggerForSingletonTransitionCases < ActiveRecord::Migration[5.2]
4
- def up
5
- execute(<<~SQL)
6
- CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
7
- DECLARE
8
- next_strand varchar;
9
- running_count integer;
10
- should_lock boolean;
11
- should_be_precise boolean;
12
- update_query varchar;
13
- skip_locked varchar;
14
- transition boolean;
15
- BEGIN
16
- IF OLD.strand IS NOT NULL THEN
17
- should_lock := true;
18
- should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
19
-
20
- IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
21
- running_count := (SELECT COUNT(*) FROM (
22
- SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
23
- ) subquery_for_count);
24
- should_lock := running_count < OLD.max_concurrent;
25
- END IF;
26
-
27
- IF should_lock THEN
28
- PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
29
- END IF;
30
-
31
- -- note that we don't really care if the row we're deleting has a singleton, or if it even
32
- -- matches the row(s) we're going to update. we just need to make sure that whatever
33
- -- singleton we grab isn't already running (which is a simple existence check, since
34
- -- the unique indexes ensure there is at most one singleton running, and one queued)
35
- update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
36
- SELECT id FROM delayed_jobs j2
37
- WHERE next_in_strand=false AND
38
- j2.strand=$1.strand AND
39
- (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
40
- ORDER BY j2.strand_order_override ASC, j2.id ASC
41
- LIMIT ';
42
-
43
- IF should_be_precise THEN
44
- running_count := (SELECT COUNT(*) FROM (
45
- SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
46
- ) s);
47
- IF running_count < OLD.max_concurrent THEN
48
- update_query := update_query || '($1.max_concurrent - $2)';
49
- ELSE
50
- -- we have too many running already; just bail
51
- RETURN OLD;
52
- END IF;
53
- ELSE
54
- update_query := update_query || '1';
55
-
56
- -- n-strands don't require precise ordering; we can make this query more performant
57
- IF OLD.max_concurrent > 1 THEN
58
- skip_locked := ' SKIP LOCKED';
59
- END IF;
60
- END IF;
61
-
62
- update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
63
- EXECUTE update_query USING OLD, running_count;
64
- END IF;
65
-
66
- IF OLD.singleton IS NOT NULL THEN
67
- transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
68
-
69
- IF transition THEN
70
- next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
71
-
72
- IF next_strand IS NOT NULL THEN
73
- -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
74
- IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
75
- -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
76
- RETURN OLD;
77
- END IF;
78
- END IF;
79
- ELSIF OLD.strand IS NOT NULL THEN
80
- -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
81
- RETURN OLD;
82
- END IF;
83
-
84
- -- handles transitioning a singleton from stranded to not stranded --
85
- -- handles transitioning a singleton from unstranded to stranded --
86
- -- handles transitioning a singleton from strand A to strand B --
87
- -- these transitions are a relatively rare case, so we take a shortcut and --
88
- -- only start the next singleton if its strand does not currently have any running jobs --
89
- -- if it does, the next stranded job that finishes will start this singleton if it can --
90
- UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
91
- SELECT id FROM delayed_jobs j2
92
- WHERE next_in_strand=false AND
93
- j2.singleton=OLD.singleton AND
94
- j2.locked_by IS NULL AND
95
- (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
96
- FOR UPDATE
97
- );
98
- END IF;
99
- RETURN OLD;
100
- END;
101
- $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
102
- SQL
103
- end
104
-
105
- def down
106
- execute(<<~SQL)
107
- CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
108
- DECLARE
109
- running_count integer;
110
- should_lock boolean;
111
- should_be_precise boolean;
112
- update_query varchar;
113
- skip_locked varchar;
114
- BEGIN
115
- IF OLD.strand IS NOT NULL THEN
116
- should_lock := true;
117
- should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
118
-
119
- IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
120
- running_count := (SELECT COUNT(*) FROM (
121
- SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
122
- ) subquery_for_count);
123
- should_lock := running_count < OLD.max_concurrent;
124
- END IF;
125
-
126
- IF should_lock THEN
127
- PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
128
- END IF;
129
-
130
- -- note that we don't really care if the row we're deleting has a singleton, or if it even
131
- -- matches the row(s) we're going to update. we just need to make sure that whatever
132
- -- singleton we grab isn't already running (which is a simple existence check, since
133
- -- the unique indexes ensure there is at most one singleton running, and one queued)
134
- update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
135
- SELECT id FROM delayed_jobs j2
136
- WHERE next_in_strand=false AND
137
- j2.strand=$1.strand AND
138
- (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
139
- ORDER BY j2.strand_order_override ASC, j2.id ASC
140
- LIMIT ';
141
-
142
- IF should_be_precise THEN
143
- running_count := (SELECT COUNT(*) FROM (
144
- SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
145
- ) s);
146
- IF running_count < OLD.max_concurrent THEN
147
- update_query := update_query || '($1.max_concurrent - $2)';
148
- ELSE
149
- -- we have too many running already; just bail
150
- RETURN OLD;
151
- END IF;
152
- ELSE
153
- update_query := update_query || '1';
154
-
155
- -- n-strands don't require precise ordering; we can make this query more performant
156
- IF OLD.max_concurrent > 1 THEN
157
- skip_locked := ' SKIP LOCKED';
158
- END IF;
159
- END IF;
160
-
161
- update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
162
- EXECUTE update_query USING OLD, running_count;
163
- ELSIF OLD.singleton IS NOT NULL THEN
164
- UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
165
- END IF;
166
- RETURN OLD;
167
- END;
168
- $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
169
- SQL
170
- end
171
- end