switchman-inst-jobs 3.0.3 → 3.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '08102eee5c1da6d032170565b624d61f90fcbc1c6232e1214babda846c9674c9'
4
- data.tar.gz: 5f9c75d442f40b706dff2a45bfab8dc99ff8599470c096bbea594f1a763d7640
3
+ metadata.gz: f447f30d1f541b4fe85ad0d867276957b5ed583ddd01e70c509834c0441892bc
4
+ data.tar.gz: 677a35494a2b54c609017b335019a98510ca3db04e26e61778ad5a8abc0f9377
5
5
  SHA512:
6
- metadata.gz: 9118a73e2a47f1e6083737f967cc298674b8884c8cc23c9d16efff9e087951d63e54129f51a1c5595f2ec4cadb4332208a2a0262e7992b5777c39ac08ae8efa1
7
- data.tar.gz: 791c382a9fbb141dc0e2f763b936d953bbd575ac01fc869a652628992c93b47c3cf93b590d95bd7867698c9410cbb278932ed1404e1e6d9f582cf00825cda8b0
6
+ metadata.gz: 4dead22acfe97a4a5c06ba7ae348916c39e22cef000754c98d7b4e3ecb9a3312fb2fdffaba1e889812bd77efa9ab2d0ecc9ef6bba3d8325fafb18fe99f70278b
7
+ data.tar.gz: 05513257e9b30f44e2cb10ee5eb497a66906c38b8e525fb2373326ec81d6cb84df413f02283d46faf3a85d8e996f7da66cbf4b59335e23a743329ad1c1ef6f7c
@@ -29,7 +29,7 @@ class OptimizeDelayedJobs < ActiveRecord::Migration[4.2]
29
29
  add_index :delayed_jobs, %w[strand id], name: 'index_delayed_jobs_on_strand'
30
30
 
31
31
  # move all failed jobs to the new failed table
32
- Delayed::Backend::ActiveRecord::Job.where('failed_at IS NOT NULL').find_each do |job|
32
+ Delayed::Backend::ActiveRecord::Job.where.not(failed_at: nil).find_each do |job|
33
33
  job.fail! unless job.on_hold?
34
34
  end
35
35
  end
@@ -93,7 +93,7 @@ module SwitchmanInstJobs
93
93
  # likely a missing shard with a stale cache
94
94
  current_shard.send(:clear_cache)
95
95
  ::Switchman::Shard.clear_cache
96
- raise ShardNotFoundError, shard_id unless ::Switchman::Shard.where(id: shard_id).exists?
96
+ raise ShardNotFoundError, shard_id unless ::Switchman::Shard.exists?(id: shard_id)
97
97
 
98
98
  raise
99
99
  end
@@ -17,7 +17,7 @@ module SwitchmanInstJobs
17
17
  # We purposely don't .compact to remove nils here, since if any
18
18
  # workers are on the default jobs shard we want to unlock against
19
19
  # that shard too.
20
- shard_ids = @config[:workers].map { |c| c[:shard] }.uniq
20
+ shard_ids = @config[:workers].pluck(:shard).uniq
21
21
  shards = shard_ids.map { |shard_id| ::Delayed::Worker.shard(shard_id) }
22
22
  end
23
23
  ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs]) do
@@ -29,26 +29,41 @@ module SwitchmanInstJobs
29
29
 
30
30
  def migrate_shards(shard_map)
31
31
  source_shards = Set[]
32
+ target_shards = Hash.new([])
32
33
  shard_map.each do |(shard, target_shard)|
33
34
  shard = ::Switchman::Shard.find(shard) unless shard.is_a?(::Switchman::Shard)
34
35
  source_shards << shard.delayed_jobs_shard.id
35
- # If target_shard is an int, it won't have an id, but we can just use it as is
36
- shard.update(delayed_jobs_shard_id: target_shard.try(:id) || target_shard, block_stranded: true)
36
+ target_shard = target_shard.try(:id) || target_shard
37
+ target_shards[target_shard] += [shard.id]
37
38
  end
38
39
 
40
+ # Do the updates in batches and then just clear redis instead of clearing them one at a time
41
+ target_shards.each do |target_shard, shards|
42
+ ::Switchman::Shard.where(id: shards).update_all(delayed_jobs_shard_id: target_shard, block_stranded: true)
43
+ end
44
+ clear_shard_cache
45
+
39
46
  # Wait a little over the 60 second in-process shard cache clearing
40
47
  # threshold to ensure that all new stranded jobs are now being
41
48
  # enqueued with next_in_strand: false
42
- Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
49
+ Rails.logger.debug('Waiting for caches to clear')
43
50
  sleep(65) unless @skip_cache_wait
44
51
 
45
- # TODO: 4 has been picked completely out of a hat. We should make it configurable or something
46
- Parallel.each(source_shards, in_processes: 4) do |s|
47
- # Ensure the child processes don't share connections with the parent
48
- Delayed::Pool.on_fork.call
49
- ActiveRecord::Base.clear_all_connections!
50
- s.activate(:delayed_jobs) { run }
52
+ ::Switchman::Shard.clear_cache
53
+ # rubocop:disable Style/CombinableLoops
54
+ # We first migrate strands so that we can stop blocking strands before we migrate unstranded jobs
55
+ source_shards.each do |s|
56
+ ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_strands }
57
+ end
58
+
59
+ source_shards.each do |s|
60
+ ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_everything }
51
61
  end
62
+ # rubocop:enable Style/CombinableLoops
63
+ end
64
+
65
+ def clear_shard_cache
66
+ ::Switchman.cache.clear
52
67
  end
53
68
 
54
69
  # This method expects that all relevant shards already have block_stranded: true
@@ -71,7 +86,7 @@ module SwitchmanInstJobs
71
86
  # those (= do nothing since it should already be true)
72
87
 
73
88
  source_shard = ::Switchman::Shard.current(:delayed_jobs)
74
- strand_scope = ::Delayed::Job.shard(source_shard).where('strand IS NOT NULL')
89
+ strand_scope = ::Delayed::Job.shard(source_shard).where.not(strand: nil)
75
90
  shard_map = build_shard_map(strand_scope, source_shard)
76
91
  shard_map.each do |(target_shard, source_shard_ids)|
77
92
  shard_scope = strand_scope.where(shard_id: source_shard_ids)
@@ -92,7 +107,7 @@ module SwitchmanInstJobs
92
107
  # We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
93
108
  # Since we only unlock it on the new jobs queue *after* deleting from the original
94
109
  # the lock ensures the blocker always gets unlocked
95
- first = this_strand_scope.where('locked_by IS NOT NULL').next_in_strand_order.lock.first
110
+ first = this_strand_scope.where.not(locked_by: nil).next_in_strand_order.lock.first
96
111
  if first
97
112
  first_job = ::Delayed::Job.create!(strand: strand, next_in_strand: false)
98
113
  first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, args: [0])
@@ -123,26 +138,37 @@ module SwitchmanInstJobs
123
138
  end
124
139
  end
125
140
 
126
- ::Switchman::Shard.find(source_shard_ids).each do |shard|
127
- shard.update(block_stranded: false)
141
+ updated = ::Switchman::Shard.where(id: source_shard_ids, block_stranded: true).
142
+ update_all(block_stranded: false)
143
+ # If this is being manually re-run for some reason to clean something up, don't wait for nothing to happen
144
+ unless updated.zero?
145
+ clear_shard_cache
146
+ # Wait a little over the 60 second in-process shard cache clearing
147
+ # threshold to ensure that all new stranded jobs are now being
148
+ # enqueued with next_in_strand: false
149
+ Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
150
+ # for spec usage only
151
+ sleep(65) unless @skip_cache_wait
128
152
  end
129
- # Wait a little over the 60 second in-process shard cache clearing
130
- # threshold to ensure that all new stranded jobs are now being
131
- # enqueued with next_in_strand: false
132
- Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
133
- # for spec usage only
134
- sleep(65) unless @skip_cache_wait
153
+ ::Switchman::Shard.clear_cache
135
154
  # At this time, let's unblock all the strands on the target shard that aren't being held by a blocker
136
155
  # but actually could have run and we just didn't know it because we didn't know if they had jobs
137
156
  # on the source shard
138
- # rubocop:disable Layout/LineLength
139
- strands_to_unblock = shard_scope.where.not(source: 'JobsMigrator::StrandBlocker').
140
- distinct.
141
- where("NOT EXISTS (SELECT 1 FROM #{::Delayed::Job.quoted_table_name} dj2 WHERE delayed_jobs.strand=dj2.strand AND next_in_strand)").
142
- pluck(:strand)
143
- # rubocop:enable Layout/LineLength
144
- strands_to_unblock.each do |strand|
145
- Delayed::Job.where(strand: strand).next_in_strand_order.first.update_attribute(:next_in_strand, true)
157
+ target_shard.activate(:delayed_jobs) do
158
+ loop do
159
+ # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
160
+ # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
161
+ # this target shard, so we shouldn't unlock them yet. We only ever unlock one job here to keep the
162
+ # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
163
+ # batches
164
+ break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
165
+ where.not(strand: nil).
166
+ where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
167
+ ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
168
+ where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
169
+ where('dj2.strand = delayed_jobs.strand').arel.exists.not
170
+ ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
171
+ end
146
172
  end
147
173
  end
148
174
  end
@@ -242,7 +268,7 @@ module SwitchmanInstJobs
242
268
 
243
269
  connection.execute "COPY #{::Delayed::Job.quoted_table_name} (#{quoted_keys}) FROM STDIN"
244
270
  records.map do |record|
245
- connection.raw_connection.put_copy_data(keys.map { |k| quote_text(record[k]) }.join("\t") + "\n")
271
+ connection.raw_connection.put_copy_data("#{keys.map { |k| quote_text(record[k]) }.join("\t")}\n")
246
272
  end
247
273
  connection.clear_query_cache
248
274
  connection.raw_connection.put_copy_end
@@ -6,7 +6,7 @@ module SwitchmanInstJobs
6
6
  @cached_at = Time.zone.now
7
7
  end
8
8
 
9
- def clear(force = false)
9
+ def clear(force: false)
10
10
  if force || @cached_at < @timeout.call
11
11
  @block.call
12
12
  @cached_at = Time.zone.now
@@ -1,3 +1,3 @@
1
1
  module SwitchmanInstJobs
2
- VERSION = '3.0.3'.freeze
2
+ VERSION = '3.0.4'.freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: switchman-inst-jobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.3
4
+ version: 3.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bryan Petty
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-11 00:00:00.000000000 Z
11
+ date: 2020-11-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: inst-jobs
@@ -210,28 +210,28 @@ dependencies:
210
210
  requirements:
211
211
  - - "~>"
212
212
  - !ruby/object:Gem::Version
213
- version: 0.79.0
213
+ version: 1.3.1
214
214
  type: :development
215
215
  prerelease: false
216
216
  version_requirements: !ruby/object:Gem::Requirement
217
217
  requirements:
218
218
  - - "~>"
219
219
  - !ruby/object:Gem::Version
220
- version: 0.79.0
220
+ version: 1.3.1
221
221
  - !ruby/object:Gem::Dependency
222
222
  name: rubocop-rails
223
223
  requirement: !ruby/object:Gem::Requirement
224
224
  requirements:
225
225
  - - "~>"
226
226
  - !ruby/object:Gem::Version
227
- version: 2.4.2
227
+ version: 2.8.1
228
228
  type: :development
229
229
  prerelease: false
230
230
  version_requirements: !ruby/object:Gem::Requirement
231
231
  requirements:
232
232
  - - "~>"
233
233
  - !ruby/object:Gem::Version
234
- version: 2.4.2
234
+ version: 2.8.1
235
235
  - !ruby/object:Gem::Dependency
236
236
  name: simplecov
237
237
  requirement: !ruby/object:Gem::Requirement