switchman-inst-jobs 4.3.2 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/switchman_inst_jobs/jobs_migrator.rb +0 -2
- data/lib/switchman_inst_jobs/switchman/default_shard.rb +2 -2
- data/lib/switchman_inst_jobs/switchman/shard.rb +83 -41
- data/lib/switchman_inst_jobs/timed_cache.rb +1 -1
- data/lib/switchman_inst_jobs/version.rb +1 -1
- data/lib/switchman_inst_jobs.rb +1 -3
- metadata +12 -36
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f308b462ca12380d7575235b5e462045d03bcdf3602265c8ce526700b1846b6b
|
|
4
|
+
data.tar.gz: ed883697bf559536dd84364e8a40b74465c790fa72b66268151d66812b7fda07
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2258e5e4aae65d9fac61d9037ec3ece167c5f18dae08050167d9175384e231137dddeed8774950646484e6f8d6b165c4287977ec6aa78f3261bf486944e268ca
|
|
7
|
+
data.tar.gz: c715f2e7cf3b31af9860c1be5db6cca0bb784327acb3a0647980ca3f760dda160398dfc51f68490309ddb71f56abab0fd430c8a99e3d7add6efc7a20688101aa
|
|
@@ -22,56 +22,82 @@ module SwitchmanInstJobs
|
|
|
22
22
|
# Adapted from hold/unhold methods in base delayed jobs base
|
|
23
23
|
# Wait is required to be able to safely move jobs
|
|
24
24
|
def hold_jobs!(wait: false)
|
|
25
|
-
self.
|
|
26
|
-
save! if changed?
|
|
27
|
-
delayed_jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
|
28
|
-
lock_jobs_for_hold
|
|
29
|
-
end
|
|
30
|
-
return unless wait
|
|
31
|
-
|
|
32
|
-
delayed_jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
|
33
|
-
while ::Delayed::Job.where(shard_id: id)
|
|
34
|
-
.where.not(locked_at: nil)
|
|
35
|
-
.where.not(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY).exists?
|
|
36
|
-
sleep 10
|
|
37
|
-
lock_jobs_for_hold
|
|
38
|
-
end
|
|
39
|
-
end
|
|
25
|
+
::Switchman::Shard.where(id: self).hold_jobs!(wait:)
|
|
40
26
|
end
|
|
41
27
|
|
|
42
28
|
def unhold_jobs!
|
|
43
|
-
self.
|
|
44
|
-
|
|
45
|
-
|
|
29
|
+
::Switchman::Shard.where(id: self).unhold_jobs!
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
module ClassMethods
|
|
33
|
+
# Adapted from hold/unhold methods in base delayed jobs base
|
|
34
|
+
# Wait is required to be able to safely move jobs
|
|
35
|
+
def hold_jobs!(wait: false)
|
|
36
|
+
shards = all.to_a
|
|
37
|
+
wait_for_caches = false
|
|
38
|
+
shards.each do |shard|
|
|
39
|
+
shard.jobs_held = true
|
|
40
|
+
if shard.changed?
|
|
41
|
+
shard.save!
|
|
42
|
+
wait_for_caches = true if wait
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
shards_by_jobs_shard(shards).each do |jobs_shard, shard_ids|
|
|
46
|
+
jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
|
47
|
+
lock_jobs_for_hold(shard_ids)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
return unless wait
|
|
51
|
+
|
|
46
52
|
# Wait a little over the 60 second in-process shard cache clearing
|
|
47
53
|
# threshold to ensure that all new jobs are now being enqueued
|
|
48
|
-
#
|
|
54
|
+
# locked
|
|
49
55
|
Rails.logger.debug("Waiting for caches to clear")
|
|
50
|
-
sleep(65)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
56
|
+
sleep(65) if wait && wait_for_caches
|
|
57
|
+
|
|
58
|
+
shards_by_jobs_shard(shards).each do |jobs_shard, shard_ids|
|
|
59
|
+
jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
|
60
|
+
while ::Delayed::Job.where(shard_id: shard_ids)
|
|
61
|
+
.where.not(locked_at: nil)
|
|
62
|
+
.where.not(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY).exists?
|
|
63
|
+
sleep 10
|
|
64
|
+
lock_jobs_for_hold(shard_ids)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
61
68
|
end
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
private
|
|
65
69
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
def unhold_jobs!
|
|
71
|
+
shards = all.to_a
|
|
72
|
+
waited = false
|
|
73
|
+
shards.each do |shard|
|
|
74
|
+
shard.jobs_held = false
|
|
75
|
+
next unless shard.changed?
|
|
76
|
+
|
|
77
|
+
shard.save!
|
|
78
|
+
next if waited
|
|
79
|
+
|
|
80
|
+
# Wait a little over the 60 second in-process shard cache clearing
|
|
81
|
+
# threshold to ensure that all new jobs are now being enqueued
|
|
82
|
+
# unlocked
|
|
83
|
+
Rails.logger.debug("Waiting for caches to clear")
|
|
84
|
+
sleep(65)
|
|
85
|
+
waited = true
|
|
86
|
+
end
|
|
87
|
+
shards_by_jobs_shard(shards).each do |jobs_shard, shard_ids|
|
|
88
|
+
jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
|
|
89
|
+
::Delayed::Job.where(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY, shard_id: shard_ids)
|
|
90
|
+
.in_batches(of: 10_000)
|
|
91
|
+
.update_all(
|
|
92
|
+
locked_by: nil,
|
|
93
|
+
locked_at: nil,
|
|
94
|
+
attempts: 0,
|
|
95
|
+
failed_at: nil
|
|
96
|
+
)
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
73
100
|
|
|
74
|
-
module ClassMethods
|
|
75
101
|
def clear_cache
|
|
76
102
|
super
|
|
77
103
|
remove_instance_variable(:@delayed_jobs_shards) if instance_variable_defined?(:@delayed_jobs_shards)
|
|
@@ -130,6 +156,22 @@ module SwitchmanInstJobs
|
|
|
130
156
|
|
|
131
157
|
::Switchman::Shard.merge(scope)
|
|
132
158
|
end
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
# Group the given shards by the shard their jobs live on, returning a
|
|
163
|
+
# hash of delayed_jobs_shard => [shard_id, ...]
|
|
164
|
+
def shards_by_jobs_shard(shards)
|
|
165
|
+
shards.group_by(&:delayed_jobs_shard).transform_values { |group| group.map(&:id) }
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def lock_jobs_for_hold(shard_ids)
|
|
169
|
+
::Delayed::Job.where(locked_at: nil, shard_id: shard_ids).in_batches(of: 10_000).update_all(
|
|
170
|
+
locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY,
|
|
171
|
+
locked_at: ::Delayed::Job.db_time_now,
|
|
172
|
+
attempts: ::Delayed::Backend::Base::ON_HOLD_COUNT
|
|
173
|
+
)
|
|
174
|
+
end
|
|
133
175
|
end
|
|
134
176
|
end
|
|
135
177
|
end
|
data/lib/switchman_inst_jobs.rb
CHANGED
|
@@ -7,9 +7,7 @@ module SwitchmanInstJobs
|
|
|
7
7
|
cattr_accessor :delayed_jobs_shard_fallback
|
|
8
8
|
|
|
9
9
|
def self.initialize_active_record
|
|
10
|
-
|
|
11
|
-
ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
|
|
12
|
-
)
|
|
10
|
+
ActiveSupport.on_load(:active_record_postgresqladapter) { prepend ActiveRecord::ConnectionAdapters::PostgreSQLAdapter }
|
|
13
11
|
end
|
|
14
12
|
|
|
15
13
|
def self.initialize_inst_jobs
|
metadata
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: switchman-inst-jobs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 4.
|
|
4
|
+
version: 4.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Bryan Petty
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
13
|
name: inst-jobs
|
|
@@ -30,61 +29,40 @@ dependencies:
|
|
|
30
29
|
- - "<"
|
|
31
30
|
- !ruby/object:Gem::Version
|
|
32
31
|
version: '4.0'
|
|
33
|
-
- !ruby/object:Gem::Dependency
|
|
34
|
-
name: parallel
|
|
35
|
-
requirement: !ruby/object:Gem::Requirement
|
|
36
|
-
requirements:
|
|
37
|
-
- - ">="
|
|
38
|
-
- !ruby/object:Gem::Version
|
|
39
|
-
version: '1.19'
|
|
40
|
-
type: :runtime
|
|
41
|
-
prerelease: false
|
|
42
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
43
|
-
requirements:
|
|
44
|
-
- - ">="
|
|
45
|
-
- !ruby/object:Gem::Version
|
|
46
|
-
version: '1.19'
|
|
47
32
|
- !ruby/object:Gem::Dependency
|
|
48
33
|
name: railties
|
|
49
34
|
requirement: !ruby/object:Gem::Requirement
|
|
50
35
|
requirements:
|
|
51
36
|
- - ">="
|
|
52
37
|
- !ruby/object:Gem::Version
|
|
53
|
-
version: '7.
|
|
38
|
+
version: '7.2'
|
|
54
39
|
- - "<"
|
|
55
40
|
- !ruby/object:Gem::Version
|
|
56
|
-
version: '8.
|
|
41
|
+
version: '8.2'
|
|
57
42
|
type: :runtime
|
|
58
43
|
prerelease: false
|
|
59
44
|
version_requirements: !ruby/object:Gem::Requirement
|
|
60
45
|
requirements:
|
|
61
46
|
- - ">="
|
|
62
47
|
- !ruby/object:Gem::Version
|
|
63
|
-
version: '7.
|
|
48
|
+
version: '7.2'
|
|
64
49
|
- - "<"
|
|
65
50
|
- !ruby/object:Gem::Version
|
|
66
|
-
version: '8.
|
|
51
|
+
version: '8.2'
|
|
67
52
|
- !ruby/object:Gem::Dependency
|
|
68
53
|
name: switchman
|
|
69
54
|
requirement: !ruby/object:Gem::Requirement
|
|
70
55
|
requirements:
|
|
71
|
-
- - "
|
|
72
|
-
- !ruby/object:Gem::Version
|
|
73
|
-
version: 3.5.14
|
|
74
|
-
- - "<"
|
|
56
|
+
- - "~>"
|
|
75
57
|
- !ruby/object:Gem::Version
|
|
76
|
-
version: '
|
|
58
|
+
version: '4.3'
|
|
77
59
|
type: :runtime
|
|
78
60
|
prerelease: false
|
|
79
61
|
version_requirements: !ruby/object:Gem::Requirement
|
|
80
62
|
requirements:
|
|
81
|
-
- - "
|
|
82
|
-
- !ruby/object:Gem::Version
|
|
83
|
-
version: 3.5.14
|
|
84
|
-
- - "<"
|
|
63
|
+
- - "~>"
|
|
85
64
|
- !ruby/object:Gem::Version
|
|
86
|
-
version: '
|
|
87
|
-
description:
|
|
65
|
+
version: '4.3'
|
|
88
66
|
email:
|
|
89
67
|
- bpetty@instructure.com
|
|
90
68
|
executables: []
|
|
@@ -164,7 +142,6 @@ licenses:
|
|
|
164
142
|
metadata:
|
|
165
143
|
allowed_push_host: https://rubygems.org
|
|
166
144
|
rubygems_mfa_required: 'true'
|
|
167
|
-
post_install_message:
|
|
168
145
|
rdoc_options: []
|
|
169
146
|
require_paths:
|
|
170
147
|
- lib
|
|
@@ -172,15 +149,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
172
149
|
requirements:
|
|
173
150
|
- - ">="
|
|
174
151
|
- !ruby/object:Gem::Version
|
|
175
|
-
version: '3.
|
|
152
|
+
version: '3.3'
|
|
176
153
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
177
154
|
requirements:
|
|
178
155
|
- - ">="
|
|
179
156
|
- !ruby/object:Gem::Version
|
|
180
157
|
version: '0'
|
|
181
158
|
requirements: []
|
|
182
|
-
rubygems_version: 3.
|
|
183
|
-
signing_key:
|
|
159
|
+
rubygems_version: 3.6.9
|
|
184
160
|
specification_version: 4
|
|
185
161
|
summary: Switchman and Instructure Jobs compatibility gem.
|
|
186
162
|
test_files: []
|