switchman-inst-jobs 3.2.10 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/db/migrate/20101216224513_create_delayed_jobs.rb +1 -1
- data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +1 -1
- data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +1 -1
- data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +1 -1
- data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +1 -1
- data/db/migrate/20110610213249_optimize_delayed_jobs.rb +1 -1
- data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +1 -1
- data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +1 -1
- data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +1 -1
- data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +1 -1
- data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +2 -2
- data/db/migrate/20120608191051_add_jobs_run_at_index.rb +2 -2
- data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +1 -1
- data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +1 -1
- data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +2 -2
- data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +1 -1
- data/db/migrate/20140512213941_add_source_to_jobs.rb +1 -1
- data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +1 -1
- data/db/migrate/20151123210429_add_expires_at_to_jobs.rb +1 -1
- data/db/migrate/20151210162949_improve_max_concurrent.rb +1 -1
- data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +1 -1
- data/db/migrate/20170308045400_add_shard_id_to_delayed_jobs.rb +1 -11
- data/db/migrate/20170308045401_add_delayed_jobs_shard_id_to_switchman_shards.rb +5 -0
- data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +1 -1
- data/db/migrate/20190726154743_make_critical_columns_not_null.rb +1 -1
- data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +2 -2
- data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +1 -1
- data/db/migrate/20200825011002_add_strand_order_override.rb +2 -2
- data/lib/switchman_inst_jobs/active_record/connection_adapters/connection_pool.rb +15 -0
- data/lib/switchman_inst_jobs/delayed/backend/base.rb +10 -9
- data/lib/switchman_inst_jobs/delayed/pool.rb +1 -1
- data/lib/switchman_inst_jobs/delayed/worker/health_check.rb +10 -12
- data/lib/switchman_inst_jobs/delayed/worker.rb +2 -2
- data/lib/switchman_inst_jobs/engine.rb +8 -10
- data/lib/switchman_inst_jobs/jobs_migrator.rb +88 -144
- data/lib/switchman_inst_jobs/switchman/shard.rb +8 -21
- data/lib/switchman_inst_jobs/version.rb +1 -1
- data/lib/switchman_inst_jobs.rb +8 -0
- metadata +44 -51
- data/db/migrate/20210809145804_add_n_strand_index.rb +0 -12
- data/db/migrate/20210812210128_add_singleton_column.rb +0 -200
- data/db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb +0 -27
- data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb +0 -56
- data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb +0 -27
- data/db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb +0 -137
- data/db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb +0 -171
- data/db/migrate/20211220112800_fix_singleton_race_condition_insert.rb +0 -59
- data/db/migrate/20211220113000_fix_singleton_race_condition_delete.rb +0 -207
- data/db/migrate/20220127091200_fix_singleton_unique_constraint.rb +0 -31
- data/db/migrate/20220128084800_update_insert_trigger_for_singleton_unique_constraint_change.rb +0 -60
- data/db/migrate/20220128084900_update_delete_trigger_for_singleton_unique_constraint_change.rb +0 -209
- data/db/migrate/20220203063200_remove_old_singleton_index.rb +0 -31
@@ -1,3 +1,7 @@
|
|
1
|
+
# Just disabling all the rubocop metrics for this file for now,
|
2
|
+
# as it is a direct port-in of existing code
|
3
|
+
|
4
|
+
# rubocop:disable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength
|
1
5
|
require 'set'
|
2
6
|
require 'parallel'
|
3
7
|
|
@@ -13,10 +17,10 @@ module SwitchmanInstJobs
|
|
13
17
|
return yield if shards.empty?
|
14
18
|
|
15
19
|
shard = shards.pop
|
16
|
-
current_shard = ::Switchman::Shard.current(
|
17
|
-
shard.activate(
|
20
|
+
current_shard = ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
|
21
|
+
shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
18
22
|
::Delayed::Job.transaction do
|
19
|
-
current_shard.activate(
|
23
|
+
current_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
20
24
|
transaction_on(shards, &block)
|
21
25
|
end
|
22
26
|
end
|
@@ -45,11 +49,11 @@ module SwitchmanInstJobs
|
|
45
49
|
# rubocop:disable Style/CombinableLoops
|
46
50
|
# We first migrate strands so that we can stop blocking strands before we migrate unstranded jobs
|
47
51
|
source_shards.each do |s|
|
48
|
-
::Switchman::Shard.lookup(s).activate(
|
52
|
+
::Switchman::Shard.lookup(s).activate(::Delayed::Backend::ActiveRecord::AbstractJob) { migrate_strands }
|
49
53
|
end
|
50
54
|
|
51
55
|
source_shards.each do |s|
|
52
|
-
::Switchman::Shard.lookup(s).activate(
|
56
|
+
::Switchman::Shard.lookup(s).activate(::Delayed::Backend::ActiveRecord::AbstractJob) { migrate_everything }
|
53
57
|
end
|
54
58
|
ensure_unblock_stranded_for(shard_map.map(&:first))
|
55
59
|
# rubocop:enable Style/CombinableLoops
|
@@ -89,9 +93,7 @@ module SwitchmanInstJobs
|
|
89
93
|
migrate_everything
|
90
94
|
end
|
91
95
|
|
92
|
-
def migrate_strands
|
93
|
-
source_shard = ::Switchman::Shard.current(:delayed_jobs)
|
94
|
-
|
96
|
+
def migrate_strands
|
95
97
|
# there are 4 scenarios to deal with here
|
96
98
|
# 1) no running job, no jobs moved: do nothing
|
97
99
|
# 2) running job, no jobs moved; create blocker with next_in_strand=false
|
@@ -100,89 +102,60 @@ module SwitchmanInstJobs
|
|
100
102
|
# those (= do nothing since it should already be false)
|
101
103
|
# 4) no running job, jobs moved: set next_in_strand=true on the first of
|
102
104
|
# those (= do nothing since it should already be true)
|
103
|
-
handler = lambda { |scope, column, blocker_job_kwargs = {}, advisory_lock_cb = nil|
|
104
|
-
shard_map = build_shard_map(scope, source_shard)
|
105
|
-
shard_map.each do |(target_shard, source_shard_ids)|
|
106
|
-
shard_scope = scope.where(shard_id: source_shard_ids)
|
107
|
-
|
108
|
-
# 1) is taken care of because it should not show up here in strands
|
109
|
-
values = shard_scope.distinct.order(column).pluck(column)
|
110
|
-
|
111
|
-
target_shard.activate(:delayed_jobs) do
|
112
|
-
values.each do |value|
|
113
|
-
transaction_on([source_shard, target_shard]) do
|
114
|
-
source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
115
|
-
advisory_lock_cb&.call(value)
|
116
|
-
end
|
117
|
-
|
118
|
-
value_scope = shard_scope.where(**{ column => value })
|
119
|
-
# we want to copy all the jobs except the one that is still running.
|
120
|
-
jobs_scope = value_scope.where(locked_by: nil)
|
121
|
-
|
122
|
-
# 2) and part of 3) are taken care of here by creating a blocker
|
123
|
-
# job with next_in_strand = false. as soon as the current
|
124
|
-
# running job is finished it should set next_in_strand
|
125
|
-
# We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
|
126
|
-
# Since we only unlock it on the new jobs queue *after* deleting from the original
|
127
|
-
# the lock ensures the blocker always gets unlocked
|
128
|
-
first = value_scope.where.not(locked_by: nil).next_in_strand_order.lock.first
|
129
|
-
if first
|
130
|
-
create_blocker_job(
|
131
|
-
queue: first.queue,
|
132
|
-
shard_id: first.shard_id,
|
133
|
-
**{ column => value },
|
134
|
-
**blocker_job_kwargs
|
135
|
-
)
|
136
|
-
|
137
|
-
# the rest of 3) is taken care of here
|
138
|
-
# make sure that all the jobs moved over are NOT next in strand
|
139
|
-
::Delayed::Job.where(next_in_strand: true, locked_by: nil, **{ column => value }).
|
140
|
-
update_all(next_in_strand: false)
|
141
|
-
end
|
142
|
-
|
143
|
-
# 4) is taken care of here, by leaving next_in_strand alone and
|
144
|
-
# it should execute on the new shard
|
145
|
-
batch_move_jobs(
|
146
|
-
target_shard: target_shard,
|
147
|
-
source_shard: source_shard,
|
148
|
-
scope: jobs_scope,
|
149
|
-
batch_size: batch_size
|
150
|
-
) do |job, new_job|
|
151
|
-
# This ensures jobs enqueued on the old jobs shard run before jobs on the new jobs queue
|
152
|
-
new_job.strand_order_override = job.strand_order_override - 1
|
153
|
-
end
|
154
|
-
end
|
155
|
-
end
|
156
|
-
end
|
157
|
-
end
|
158
|
-
}
|
159
105
|
|
106
|
+
source_shard = ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
|
160
107
|
strand_scope = ::Delayed::Job.shard(source_shard).where.not(strand: nil)
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
108
|
+
shard_map = build_shard_map(strand_scope, source_shard)
|
109
|
+
shard_map.each do |(target_shard, source_shard_ids)|
|
110
|
+
shard_scope = strand_scope.where(shard_id: source_shard_ids)
|
111
|
+
|
112
|
+
# 1) is taken care of because it should not show up here in strands
|
113
|
+
strands = shard_scope.distinct.order(:strand).pluck(:strand)
|
114
|
+
|
115
|
+
target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
116
|
+
strands.each do |strand|
|
117
|
+
transaction_on([source_shard, target_shard]) do
|
118
|
+
this_strand_scope = shard_scope.where(strand: strand)
|
119
|
+
# we want to copy all the jobs except the one that is still running.
|
120
|
+
jobs_scope = this_strand_scope.where(locked_by: nil)
|
121
|
+
|
122
|
+
# 2) and part of 3) are taken care of here by creating a blocker
|
123
|
+
# job with next_in_strand = false. as soon as the current
|
124
|
+
# running job is finished it should set next_in_strand
|
125
|
+
# We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
|
126
|
+
# Since we only unlock it on the new jobs queue *after* deleting from the original
|
127
|
+
# the lock ensures the blocker always gets unlocked
|
128
|
+
first = this_strand_scope.where.not(locked_by: nil).next_in_strand_order.lock.first
|
129
|
+
if first
|
130
|
+
first_job = ::Delayed::Job.create!(strand: strand, next_in_strand: false)
|
131
|
+
first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, args: [0])
|
132
|
+
first_job.queue = first.queue
|
133
|
+
first_job.tag = 'Kernel.sleep'
|
134
|
+
first_job.source = 'JobsMigrator::StrandBlocker'
|
135
|
+
first_job.max_attempts = 1
|
136
|
+
# If we ever have jobs left over from 9999 jobs moves of a single shard,
|
137
|
+
# something has gone terribly wrong
|
138
|
+
first_job.strand_order_override = -9999
|
139
|
+
first_job.save!
|
140
|
+
# the rest of 3) is taken care of here
|
141
|
+
# make sure that all the jobs moved over are NOT next in strand
|
142
|
+
::Delayed::Job.where(next_in_strand: true, strand: strand, locked_by: nil).
|
143
|
+
update_all(next_in_strand: false)
|
144
|
+
end
|
179
145
|
|
180
|
-
|
181
|
-
|
146
|
+
# 4) is taken care of here, by leaving next_in_strand alone and
|
147
|
+
# it should execute on the new shard
|
148
|
+
batch_move_jobs(
|
149
|
+
target_shard: target_shard,
|
150
|
+
source_shard: source_shard,
|
151
|
+
scope: jobs_scope
|
152
|
+
) do |job, new_job|
|
153
|
+
# This ensures jobs enqueued on the old jobs shard run before jobs on the new jobs queue
|
154
|
+
new_job.strand_order_override = job.strand_order_override - 1
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
182
158
|
|
183
|
-
shard_map = build_shard_map(all_scope, source_shard)
|
184
|
-
shard_map.each do |(target_shard, source_shard_ids)|
|
185
|
-
target_shard.activate(:delayed_jobs) do
|
186
159
|
updated = ::Switchman::Shard.where(id: source_shard_ids, block_stranded: true).
|
187
160
|
update_all(block_stranded: false)
|
188
161
|
# If this is being manually re-run for some reason to clean something up, don't wait for nothing to happen
|
@@ -197,68 +170,41 @@ module SwitchmanInstJobs
|
|
197
170
|
end
|
198
171
|
end
|
199
172
|
|
200
|
-
def unblock_strands(target_shard
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
173
|
+
def unblock_strands(target_shard)
|
174
|
+
target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
175
|
+
loop do
|
176
|
+
# We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
|
177
|
+
# to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
|
178
|
+
# this target shard, so we shouldn't unlock them yet. We only ever unlock one job here to keep the
|
179
|
+
# logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
|
180
|
+
# batches
|
181
|
+
break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
|
182
|
+
where.not(strand: nil).
|
183
|
+
where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
|
208
184
|
::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
|
209
185
|
where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
|
210
|
-
where(
|
211
|
-
).
|
212
|
-
order(column, :strand_order_override, :id)).limit(batch_size)
|
213
|
-
}
|
214
|
-
|
215
|
-
target_shard.activate(:delayed_jobs) do
|
216
|
-
# We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
|
217
|
-
# to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
|
218
|
-
# this target shard, so we shouldn't unlock them yet. We only ever unlock one job here to keep the
|
219
|
-
# logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
|
220
|
-
# batches
|
221
|
-
|
222
|
-
loop do
|
223
|
-
break if query.call(:strand, 'strand IS NOT NULL').update_all(next_in_strand: true).zero?
|
224
|
-
end
|
225
|
-
|
226
|
-
loop do
|
227
|
-
break if query.call(:singleton,
|
228
|
-
'strand IS NULL AND singleton IS NOT NULL').update_all(next_in_strand: true).zero?
|
186
|
+
where('dj2.strand = delayed_jobs.strand').arel.exists.not
|
187
|
+
).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
|
229
188
|
end
|
230
189
|
end
|
231
190
|
end
|
232
191
|
|
233
|
-
def migrate_everything
|
234
|
-
source_shard = ::Switchman::Shard.current(
|
235
|
-
scope = ::Delayed::Job.shard(source_shard).where(
|
192
|
+
def migrate_everything
|
193
|
+
source_shard = ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
|
194
|
+
scope = ::Delayed::Job.shard(source_shard).where(strand: nil)
|
236
195
|
|
237
196
|
shard_map = build_shard_map(scope, source_shard)
|
238
197
|
shard_map.each do |(target_shard, source_shard_ids)|
|
239
198
|
batch_move_jobs(
|
240
199
|
target_shard: target_shard,
|
241
200
|
source_shard: source_shard,
|
242
|
-
scope: scope.where(shard_id: source_shard_ids).where(locked_by: nil)
|
243
|
-
batch_size: batch_size
|
201
|
+
scope: scope.where(shard_id: source_shard_ids).where(locked_by: nil)
|
244
202
|
)
|
245
203
|
end
|
246
204
|
end
|
247
205
|
|
248
206
|
private
|
249
207
|
|
250
|
-
def create_blocker_job(**kwargs)
|
251
|
-
first_job = ::Delayed::Job.create!(**kwargs, next_in_strand: false)
|
252
|
-
first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, args: [0])
|
253
|
-
first_job.tag = 'Kernel.sleep'
|
254
|
-
first_job.source = 'JobsMigrator::StrandBlocker'
|
255
|
-
first_job.max_attempts = 1
|
256
|
-
# If we ever have jobs left over from 9999 jobs moves of a single shard,
|
257
|
-
# something has gone terribly wrong
|
258
|
-
first_job.strand_order_override = -9999
|
259
|
-
first_job.save!
|
260
|
-
end
|
261
|
-
|
262
208
|
def build_shard_map(scope, source_shard)
|
263
209
|
shard_ids = scope.distinct.pluck(:shard_id)
|
264
210
|
|
@@ -273,12 +219,12 @@ module SwitchmanInstJobs
|
|
273
219
|
shard_map
|
274
220
|
end
|
275
221
|
|
276
|
-
def batch_move_jobs(target_shard:, source_shard:, scope
|
222
|
+
def batch_move_jobs(target_shard:, source_shard:, scope:)
|
277
223
|
while scope.exists?
|
278
224
|
# Adapted from get_and_lock_next_available in delayed/backend/active_record.rb
|
279
|
-
target_jobs = scope.limit(
|
225
|
+
target_jobs = scope.limit(1000).lock('FOR UPDATE SKIP LOCKED')
|
280
226
|
|
281
|
-
query = source_shard.activate(
|
227
|
+
query = source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
282
228
|
"WITH limited_jobs AS (#{target_jobs.to_sql}) " \
|
283
229
|
"UPDATE #{::Delayed::Job.quoted_table_name} " \
|
284
230
|
"SET locked_by = #{::Delayed::Job.connection.quote(::Delayed::Backend::Base::ON_HOLD_LOCKED_BY)}, " \
|
@@ -287,7 +233,9 @@ module SwitchmanInstJobs
|
|
287
233
|
"RETURNING #{::Delayed::Job.quoted_table_name}.*"
|
288
234
|
end
|
289
235
|
|
290
|
-
jobs = source_shard.activate(
|
236
|
+
jobs = source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
237
|
+
::Delayed::Job.find_by_sql(query)
|
238
|
+
end
|
291
239
|
new_jobs = jobs.map do |job|
|
292
240
|
new_job = job.dup
|
293
241
|
new_job.shard = target_shard
|
@@ -305,10 +253,10 @@ module SwitchmanInstJobs
|
|
305
253
|
new_job
|
306
254
|
end
|
307
255
|
transaction_on([source_shard, target_shard]) do
|
308
|
-
target_shard.activate(
|
256
|
+
target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
309
257
|
bulk_insert_jobs(new_jobs)
|
310
258
|
end
|
311
|
-
source_shard.activate(
|
259
|
+
source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
312
260
|
::Delayed::Job.delete(jobs)
|
313
261
|
end
|
314
262
|
end
|
@@ -335,10 +283,7 @@ module SwitchmanInstJobs
|
|
335
283
|
connection = ::Delayed::Job.connection
|
336
284
|
quoted_keys = keys.map { |k| connection.quote_column_name(k) }.join(', ')
|
337
285
|
|
338
|
-
connection.execute
|
339
|
-
connection.execute "CREATE TEMPORARY TABLE delayed_jobs_bulk_copy
|
340
|
-
(LIKE #{::Delayed::Job.quoted_table_name} INCLUDING DEFAULTS)"
|
341
|
-
connection.execute "COPY delayed_jobs_bulk_copy (#{quoted_keys}) FROM STDIN"
|
286
|
+
connection.execute "COPY #{::Delayed::Job.quoted_table_name} (#{quoted_keys}) FROM STDIN"
|
342
287
|
records.map do |record|
|
343
288
|
connection.raw_connection.put_copy_data("#{keys.map { |k| quote_text(record[k]) }.join("\t")}\n")
|
344
289
|
end
|
@@ -350,9 +295,6 @@ module SwitchmanInstJobs
|
|
350
295
|
rescue StandardError => e
|
351
296
|
raise connection.send(:translate_exception, e, 'COPY FROM STDIN')
|
352
297
|
end
|
353
|
-
connection.execute "INSERT INTO #{::Delayed::Job.quoted_table_name} (#{quoted_keys})
|
354
|
-
SELECT #{quoted_keys} FROM delayed_jobs_bulk_copy
|
355
|
-
ON CONFLICT (singleton) WHERE singleton IS NOT NULL AND locked_by IS NULL DO NOTHING"
|
356
298
|
result.cmd_tuples
|
357
299
|
end
|
358
300
|
|
@@ -370,3 +312,5 @@ module SwitchmanInstJobs
|
|
370
312
|
end
|
371
313
|
end
|
372
314
|
end
|
315
|
+
|
316
|
+
# rubocop:enable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength
|
@@ -22,12 +22,12 @@ module SwitchmanInstJobs
|
|
22
22
|
def hold_jobs!(wait: false)
|
23
23
|
self.jobs_held = true
|
24
24
|
save! if changed?
|
25
|
-
delayed_jobs_shard.activate(
|
25
|
+
delayed_jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
26
26
|
lock_jobs_for_hold
|
27
27
|
end
|
28
28
|
return unless wait
|
29
29
|
|
30
|
-
delayed_jobs_shard.activate(
|
30
|
+
delayed_jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
31
31
|
while ::Delayed::Job.where(shard_id: id).
|
32
32
|
where.not(locked_at: nil).
|
33
33
|
where.not(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY).exists?
|
@@ -47,7 +47,7 @@ module SwitchmanInstJobs
|
|
47
47
|
Rails.logger.debug('Waiting for caches to clear')
|
48
48
|
sleep(65)
|
49
49
|
end
|
50
|
-
delayed_jobs_shard.activate(
|
50
|
+
delayed_jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
51
51
|
::Delayed::Job.where(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY, shard_id: id).
|
52
52
|
in_batches(of: 10_000).
|
53
53
|
update_all(
|
@@ -75,22 +75,14 @@ module SwitchmanInstJobs
|
|
75
75
|
remove_instance_variable(:@delayed_jobs_shards) if instance_variable_defined?(:@delayed_jobs_shards)
|
76
76
|
end
|
77
77
|
|
78
|
-
def current(category = :primary)
|
79
|
-
if category == :delayed_jobs
|
80
|
-
active_shards[category] || super(:primary).delayed_jobs_shard
|
81
|
-
else
|
82
|
-
super
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
78
|
def activate!(categories)
|
87
79
|
if !@skip_delayed_job_auto_activation &&
|
88
|
-
!categories[
|
89
|
-
categories[
|
90
|
-
categories[
|
80
|
+
!categories[::Delayed::Backend::ActiveRecord::AbstractJob] &&
|
81
|
+
categories[::ActiveRecord::Base] &&
|
82
|
+
categories[::ActiveRecord::Base] != ::Switchman::Shard.current(::ActiveRecord::Base)
|
91
83
|
skip_delayed_job_auto_activation do
|
92
|
-
categories[
|
93
|
-
categories[
|
84
|
+
categories[::Delayed::Backend::ActiveRecord::AbstractJob] =
|
85
|
+
categories[::ActiveRecord::Base].delayed_jobs_shard
|
94
86
|
end
|
95
87
|
end
|
96
88
|
super
|
@@ -104,11 +96,6 @@ module SwitchmanInstJobs
|
|
104
96
|
@skip_delayed_job_auto_activation = was
|
105
97
|
end
|
106
98
|
|
107
|
-
def create
|
108
|
-
db = ::Switchman::DatabaseServer.server_for_new_shard
|
109
|
-
db.create_new_shard
|
110
|
-
end
|
111
|
-
|
112
99
|
def periodic_clear_shard_cache
|
113
100
|
# TODO: make this configurable
|
114
101
|
@timed_cache ||= TimedCache.new(-> { 60.to_i.seconds.ago }) do
|
data/lib/switchman_inst_jobs.rb
CHANGED
@@ -5,6 +5,9 @@ module SwitchmanInstJobs
|
|
5
5
|
cattr_accessor :delayed_jobs_shard_fallback
|
6
6
|
|
7
7
|
def self.initialize_active_record
|
8
|
+
::ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(
|
9
|
+
ActiveRecord::ConnectionAdapters::ConnectionPool
|
10
|
+
)
|
8
11
|
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(
|
9
12
|
ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
|
10
13
|
)
|
@@ -14,6 +17,10 @@ module SwitchmanInstJobs
|
|
14
17
|
::Delayed::Backend::ActiveRecord::Job.prepend(
|
15
18
|
Delayed::Backend::Base
|
16
19
|
)
|
20
|
+
::Delayed::Backend::Redis::Job.prepend(
|
21
|
+
Delayed::Backend::Base
|
22
|
+
)
|
23
|
+
::Delayed::Backend::Redis::Job.column :shard_id, :integer
|
17
24
|
::Delayed::Pool.prepend Delayed::Pool
|
18
25
|
::Delayed::Worker.prepend Delayed::Worker
|
19
26
|
::Delayed::Worker::HealthCheck.prepend Delayed::Worker::HealthCheck
|
@@ -32,6 +39,7 @@ module SwitchmanInstJobs
|
|
32
39
|
end
|
33
40
|
end
|
34
41
|
|
42
|
+
require 'switchman_inst_jobs/active_record/connection_adapters/connection_pool'
|
35
43
|
require 'switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter'
|
36
44
|
require 'switchman_inst_jobs/active_record/migration'
|
37
45
|
require 'switchman_inst_jobs/delayed/settings'
|