switchman-inst-jobs 3.1.2 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4efbe05e879c9162df06f945af688c64876b9c56cc870509687160549359fd39
4
- data.tar.gz: e32c8a6dfecd0b6c760d07a4d45eeaf6651d6cc7e70e7b6418d82c5a112f70cd
3
+ metadata.gz: e3ee43d75f18fc491bbbb56d90ff3e8acef32162540b6e364ba89b53ec7167e2
4
+ data.tar.gz: 8a0436d82d777d481b57f7c851b31c0739424afe96b2ad33500c4fd12c899440
5
5
  SHA512:
6
- metadata.gz: 4fe0103d1b4bc305f0c85ba2135366968eb7c33bb6482b6d302dde9b9532db1d3d09fac64d90dad16fe6c6a438236ede5732d16a22d8538053d814010c9d2c87
7
- data.tar.gz: feb2e546ee06bc38f964f58e63cfb59fdc25757952762ef8f8faf0521050d264e9f018ea8e6bb0131b277997d63035f68562cf45a7af13886fbfd80f481f5515
6
+ metadata.gz: 2ba59612057feac9cd6b7fa6f2b66d65b8d0cafaaa3e9d951d2b4d66dd7f203299db2fbff646d0edf7ce32bb7aef07063a7b467a5dfb33c91360e2d79be8aefe
7
+ data.tar.gz: 0f7b9d22fccd21b2b09dbed4ed23dbd021c7aa55bb1468ec8c96a7b3b2aa743df23da66771c42940f23d92d708b4141354391a20022a4d3389fafc7be176cc18
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddNStrandIndex < ActiveRecord::Migration[5.2]
4
+ disable_ddl_transaction!
5
+
6
+ def change
7
+ add_index :delayed_jobs, %i[strand next_in_strand id],
8
+ name: 'n_strand_index',
9
+ where: 'strand IS NOT NULL',
10
+ algorithm: :concurrently
11
+ end
12
+ end
@@ -0,0 +1,203 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddSingletonColumn < ActiveRecord::Migration[5.2]
4
+ disable_ddl_transaction!
5
+
6
+ def change
7
+ add_column :delayed_jobs, :singleton, :string, if_not_exists: true
8
+ add_column :failed_jobs, :singleton, :string, if_not_exists: true
9
+ # only one job can be queued in a singleton
10
+ add_index :delayed_jobs,
11
+ :singleton,
12
+ where: 'singleton IS NOT NULL AND locked_by IS NULL',
13
+ unique: true,
14
+ name: 'index_delayed_jobs_on_singleton_not_running',
15
+ algorithm: :concurrently
16
+ # only one job can be running for a singleton
17
+ add_index :delayed_jobs,
18
+ :singleton,
19
+ where: 'singleton IS NOT NULL AND locked_by IS NOT NULL',
20
+ unique: true,
21
+ name: 'index_delayed_jobs_on_singleton_running',
22
+ algorithm: :concurrently
23
+
24
+ reversible do |direction|
25
+ direction.up do
26
+ execute(<<~SQL)
27
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
28
+ DECLARE
29
+ running_count integer;
30
+ should_lock boolean;
31
+ should_be_precise boolean;
32
+ update_query varchar;
33
+ skip_locked varchar;
34
+ BEGIN
35
+ IF OLD.strand IS NOT NULL THEN
36
+ should_lock := true;
37
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
38
+
39
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
40
+ running_count := (SELECT COUNT(*) FROM (
41
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
42
+ ) subquery_for_count);
43
+ should_lock := running_count < OLD.max_concurrent;
44
+ END IF;
45
+
46
+ IF should_lock THEN
47
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
48
+ END IF;
49
+
50
+ -- note that we don't really care if the row we're deleting has a singleton, or if it even
51
+ -- matches the row(s) we're going to update. we just need to make sure that whatever
52
+ -- singleton we grab isn't already running (which is a simple existence check, since
53
+ -- the unique indexes ensure there is at most one singleton running, and one queued)
54
+ update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
55
+ SELECT id FROM delayed_jobs j2
56
+ WHERE next_in_strand=false AND
57
+ j2.strand=$1.strand AND
58
+ (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id))
59
+ ORDER BY j2.strand_order_override ASC, j2.id ASC
60
+ LIMIT ';
61
+
62
+ IF should_be_precise THEN
63
+ running_count := (SELECT COUNT(*) FROM (
64
+ SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
65
+ ) s);
66
+ IF running_count < OLD.max_concurrent THEN
67
+ update_query := update_query || '($1.max_concurrent - $2)';
68
+ ELSE
69
+ -- we have too many running already; just bail
70
+ RETURN OLD;
71
+ END IF;
72
+ ELSE
73
+ update_query := update_query || '1';
74
+
75
+ -- n-strands don't require precise ordering; we can make this query more performant
76
+ IF OLD.max_concurrent > 1 THEN
77
+ skip_locked := ' SKIP LOCKED';
78
+ END IF;
79
+ END IF;
80
+
81
+ update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
82
+ EXECUTE update_query USING OLD, running_count;
83
+ ELSIF OLD.singleton IS NOT NULL THEN
84
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false;
85
+ END IF;
86
+ RETURN OLD;
87
+ END;
88
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
89
+ SQL
90
+ execute(<<~SQL)
91
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
92
+ BEGIN
93
+ RAISE NOTICE 'inserting job';
94
+ IF NEW.strand IS NOT NULL THEN
95
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
96
+ IF (SELECT COUNT(*) FROM (
97
+ SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
98
+ ) s) = NEW.max_concurrent THEN
99
+ NEW.next_in_strand := false;
100
+ END IF;
101
+ END IF;
102
+ IF NEW.singleton IS NOT NULL THEN
103
+ RAISE NOTICE 'inserting job that is a singleton';
104
+ PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
105
+ IF FOUND THEN
106
+ RAISE NOTICE 'and not first';
107
+ NEW.next_in_strand := false;
108
+ END IF;
109
+ END IF;
110
+ RETURN NEW;
111
+ END;
112
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
113
+ SQL
114
+ end
115
+ direction.down do
116
+ execute(<<~SQL)
117
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
118
+ DECLARE
119
+ running_count integer;
120
+ should_lock boolean;
121
+ should_be_precise boolean;
122
+ BEGIN
123
+ IF OLD.strand IS NOT NULL THEN
124
+ should_lock := true;
125
+ should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
126
+
127
+ IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
128
+ running_count := (SELECT COUNT(*) FROM (
129
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
130
+ ) subquery_for_count);
131
+ should_lock := running_count < OLD.max_concurrent;
132
+ END IF;
133
+
134
+ IF should_lock THEN
135
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
136
+ END IF;
137
+
138
+ IF should_be_precise THEN
139
+ running_count := (SELECT COUNT(*) FROM (
140
+ SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
141
+ ) subquery_for_count);
142
+ IF running_count < OLD.max_concurrent THEN
143
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
144
+ SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
145
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
146
+ );
147
+ END IF;
148
+ ELSE
149
+ -- n-strands don't require precise ordering; we can make this query more performant
150
+ IF OLD.max_concurrent > 1 THEN
151
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
152
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
153
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
154
+ ELSE
155
+ UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
156
+ (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
157
+ j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
158
+ END IF;
159
+ END IF;
160
+ END IF;
161
+ RETURN OLD;
162
+ END;
163
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
164
+ SQL
165
+ execute(<<~SQL)
166
+ CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
167
+ BEGIN
168
+ IF NEW.strand IS NOT NULL THEN
169
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
170
+ IF (SELECT COUNT(*) FROM (
171
+ SELECT 1 AS one FROM delayed_jobs WHERE strand = NEW.strand LIMIT NEW.max_concurrent
172
+ ) subquery_for_count) = NEW.max_concurrent THEN
173
+ NEW.next_in_strand := 'f';
174
+ END IF;
175
+ END IF;
176
+ RETURN NEW;
177
+ END;
178
+ $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
179
+ SQL
180
+ end
181
+ end
182
+
183
+ connection.transaction do
184
+ reversible do |direction|
185
+ direction.up do
186
+ drop_triggers
187
+ execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL OR NEW.singleton IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
188
+ execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN ((OLD.strand IS NOT NULL OR OLD.singleton IS NOT NULL) AND OLD.next_in_strand=true) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
189
+ end
190
+ direction.down do
191
+ drop_triggers
192
+ execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
193
+ execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn()')}")
194
+ end
195
+ end
196
+ end
197
+ end
198
+
199
+ def drop_triggers
200
+ execute("DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name}")
201
+ execute("DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name}")
202
+ end
203
+ end
@@ -18,7 +18,7 @@ module SwitchmanInstJobs
18
18
  enqueue_options = options.merge(
19
19
  current_shard: current_shard
20
20
  )
21
- enqueue_job = -> { ::GuardRail.activate(:master) { super(object, **enqueue_options) } }
21
+ enqueue_job = -> { ::GuardRail.activate(:primary) { super(object, **enqueue_options) } }
22
22
 
23
23
  # Another dj shard must be currently manually activated, so just use that
24
24
  # In general this will only happen in unusual circumstances like tests
@@ -81,6 +81,8 @@ module SwitchmanInstJobs
81
81
  end
82
82
 
83
83
  def invoke_job
84
+ raise ShardNotFoundError, shard_id unless current_shard
85
+
84
86
  current_shard.activate { super }
85
87
  end
86
88
 
@@ -1,7 +1,3 @@
1
- # Just disabling all the rubocop metrics for this file for now,
2
- # as it is a direct port-in of existing code
3
-
4
- # rubocop:disable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength
5
1
  require 'set'
6
2
  require 'parallel'
7
3
 
@@ -310,5 +306,3 @@ module SwitchmanInstJobs
310
306
  end
311
307
  end
312
308
  end
313
-
314
- # rubocop:enable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength
@@ -1,3 +1,3 @@
1
1
  module SwitchmanInstJobs
2
- VERSION = '3.1.2'.freeze
2
+ VERSION = '3.2.0'.freeze
3
3
  end
@@ -14,10 +14,6 @@ module SwitchmanInstJobs
14
14
  ::Delayed::Backend::ActiveRecord::Job.prepend(
15
15
  Delayed::Backend::Base
16
16
  )
17
- ::Delayed::Backend::Redis::Job.prepend(
18
- Delayed::Backend::Base
19
- )
20
- ::Delayed::Backend::Redis::Job.column :shard_id, :integer
21
17
  ::Delayed::Pool.prepend Delayed::Pool
22
18
  ::Delayed::Worker.prepend Delayed::Worker
23
19
  ::Delayed::Worker::HealthCheck.prepend Delayed::Worker::HealthCheck
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: switchman-inst-jobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.2
4
+ version: 3.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bryan Petty
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-01-26 00:00:00.000000000 Z
11
+ date: 2021-09-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: inst-jobs
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 1.0.3
19
+ version: 2.4.0
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: '3.0'
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 1.0.3
29
+ version: 2.4.0
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '3.0'
@@ -107,7 +107,7 @@ dependencies:
107
107
  - !ruby/object:Gem::Version
108
108
  version: '0'
109
109
  - !ruby/object:Gem::Dependency
110
- name: imperium
110
+ name: diplomat
111
111
  requirement: !ruby/object:Gem::Requirement
112
112
  requirements:
113
113
  - - ">="
@@ -260,7 +260,7 @@ dependencies:
260
260
  - - "~>"
261
261
  - !ruby/object:Gem::Version
262
262
  version: '1.4'
263
- description:
263
+ description:
264
264
  email:
265
265
  - bpetty@instructure.com
266
266
  executables: []
@@ -296,6 +296,8 @@ files:
296
296
  - db/migrate/20200822014259_add_block_stranded_to_switchman_shards.rb
297
297
  - db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb
298
298
  - db/migrate/20200825011002_add_strand_order_override.rb
299
+ - db/migrate/20210809145804_add_n_strand_index.rb
300
+ - db/migrate/20210812210128_add_singleton_column.rb
299
301
  - lib/switchman-inst-jobs.rb
300
302
  - lib/switchman_inst_jobs.rb
301
303
  - lib/switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter.rb
@@ -320,7 +322,7 @@ homepage: https://github.com/instructure/switchman-inst-jobs
320
322
  licenses:
321
323
  - MIT
322
324
  metadata: {}
323
- post_install_message:
325
+ post_install_message:
324
326
  rdoc_options: []
325
327
  require_paths:
326
328
  - lib
@@ -328,15 +330,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
328
330
  requirements:
329
331
  - - ">="
330
332
  - !ruby/object:Gem::Version
331
- version: '2.5'
333
+ version: '2.6'
332
334
  required_rubygems_version: !ruby/object:Gem::Requirement
333
335
  requirements:
334
336
  - - ">="
335
337
  - !ruby/object:Gem::Version
336
338
  version: '0'
337
339
  requirements: []
338
- rubygems_version: 3.0.3
339
- signing_key:
340
+ rubygems_version: 3.2.24
341
+ signing_key:
340
342
  specification_version: 4
341
343
  summary: Switchman and Instructure Jobs compatibility gem.
342
344
  test_files: []