canvas_sync 0.21.1 → 0.22.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/canvas_sync/concerns/auto_relations.rb +11 -0
- data/lib/canvas_sync/config.rb +3 -5
- data/lib/canvas_sync/generators/templates/models/rubric.rb +2 -1
- data/lib/canvas_sync/job_batches/batch.rb +432 -402
- data/lib/canvas_sync/job_batches/callback.rb +100 -114
- data/lib/canvas_sync/job_batches/chain_builder.rb +194 -196
- data/lib/canvas_sync/job_batches/{active_job.rb → compat/active_job.rb} +2 -2
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/helpers.rb +1 -1
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web.rb +3 -3
- data/lib/canvas_sync/job_batches/{sidekiq.rb → compat/sidekiq.rb} +35 -22
- data/lib/canvas_sync/job_batches/compat.rb +20 -0
- data/lib/canvas_sync/job_batches/context_hash.rb +124 -126
- data/lib/canvas_sync/job_batches/jobs/base_job.rb +2 -4
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +125 -127
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/pool.rb +193 -195
- data/lib/canvas_sync/job_batches/redis_model.rb +50 -52
- data/lib/canvas_sync/job_batches/redis_script.rb +129 -131
- data/lib/canvas_sync/job_batches/status.rb +85 -87
- data/lib/canvas_sync/job_uniqueness/compat/active_job.rb +75 -0
- data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb +135 -0
- data/lib/canvas_sync/job_uniqueness/compat.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/configuration.rb +25 -0
- data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb +47 -0
- data/lib/canvas_sync/job_uniqueness/lock_context.rb +171 -0
- data/lib/canvas_sync/job_uniqueness/locksmith.rb +92 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/base.rb +32 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/log.rb +13 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/null_strategy.rb +9 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/raise.rb +11 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reject.rb +21 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reschedule.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict.rb +41 -0
- data/lib/canvas_sync/job_uniqueness/strategy/base.rb +104 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_and_while_executing.rb +35 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executed.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executing.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_expired.rb +16 -0
- data/lib/canvas_sync/job_uniqueness/strategy/while_executing.rb +26 -0
- data/lib/canvas_sync/job_uniqueness/strategy.rb +27 -0
- data/lib/canvas_sync/job_uniqueness/unique_job_common.rb +79 -0
- data/lib/canvas_sync/misc_helper.rb +1 -1
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +4 -3
- data/spec/dummy/app/models/rubric.rb +2 -1
- data/spec/dummy/config/environments/test.rb +1 -1
- data/spec/job_batching/batch_spec.rb +49 -7
- data/spec/job_batching/{active_job_spec.rb → compat/active_job_spec.rb} +2 -2
- data/spec/job_batching/{sidekiq_spec.rb → compat/sidekiq_spec.rb} +14 -12
- data/spec/job_batching/flow_spec.rb +1 -1
- data/spec/job_batching/integration_helper.rb +1 -1
- data/spec/job_batching/status_spec.rb +2 -2
- data/spec/job_uniqueness/compat/active_job_spec.rb +49 -0
- data/spec/job_uniqueness/compat/sidekiq_spec.rb +68 -0
- data/spec/job_uniqueness/lock_context_spec.rb +95 -0
- data/spec/job_uniqueness/on_conflict/log_spec.rb +11 -0
- data/spec/job_uniqueness/on_conflict/raise_spec.rb +10 -0
- data/spec/job_uniqueness/on_conflict/reschedule_spec.rb +24 -0
- data/spec/job_uniqueness/on_conflict_spec.rb +16 -0
- data/spec/job_uniqueness/spec_helper.rb +14 -0
- data/spec/job_uniqueness/strategy/base_spec.rb +100 -0
- data/spec/job_uniqueness/strategy/until_and_while_executing_spec.rb +48 -0
- data/spec/job_uniqueness/strategy/until_executed_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_executing_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_expired_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/while_executing_spec.rb +33 -0
- data/spec/job_uniqueness/support/lock_strategy.rb +28 -0
- data/spec/job_uniqueness/support/on_conflict.rb +24 -0
- data/spec/job_uniqueness/support/test_worker.rb +19 -0
- data/spec/job_uniqueness/unique_job_common_spec.rb +45 -0
- data/spec/spec_helper.rb +1 -1
- metadata +278 -204
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/css/styles.less +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/batch_tree.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/util.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batch_tree.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batches_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_common.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_jobs_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_pagination.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batch.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batches.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pool.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pools.erb +0 -0
@@ -1,249 +1,247 @@
|
|
1
|
-
module CanvasSync
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
@base_job = build_job_hash(base_type)
|
15
|
-
end
|
16
|
-
|
17
|
-
self.class.get_chain_parameter(base_job)
|
1
|
+
module CanvasSync::JobBatches
|
2
|
+
class ChainBuilder
|
3
|
+
VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
|
4
|
+
|
5
|
+
attr_reader :base_job
|
6
|
+
|
7
|
+
def initialize(base_type = SerialBatchJob)
|
8
|
+
if base_type.is_a?(Hash)
|
9
|
+
@base_job = base_type
|
10
|
+
@base_job[:args] ||= @base_job[:parameters] || []
|
11
|
+
@base_job[:kwargs] ||= {}
|
12
|
+
else
|
13
|
+
@base_job = build_job_hash(base_type)
|
18
14
|
end
|
19
15
|
|
20
|
-
|
21
|
-
|
22
|
-
self.class.enqueue_job(base_job)
|
23
|
-
end
|
16
|
+
self.class.get_chain_parameter(base_job)
|
17
|
+
end
|
24
18
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
# Legacy Support
|
30
|
-
key = :args if key == :parameters
|
19
|
+
def process!
|
20
|
+
normalize!
|
21
|
+
self.class.enqueue_job(base_job)
|
22
|
+
end
|
31
23
|
|
32
|
-
|
33
|
-
|
24
|
+
def [](key)
|
25
|
+
if key.is_a?(Class)
|
26
|
+
get_sub_chain(key)
|
27
|
+
else
|
28
|
+
# Legacy Support
|
29
|
+
key = :args if key == :parameters
|
30
|
+
|
31
|
+
@base_job[key]
|
34
32
|
end
|
33
|
+
end
|
35
34
|
|
36
|
-
|
37
|
-
|
35
|
+
def args; return self[:args]; end
|
36
|
+
def kwargs; return self[:kwargs]; end
|
38
37
|
|
39
|
-
|
40
|
-
|
38
|
+
def <<(new_job)
|
39
|
+
insert_at(-1, new_job)
|
40
|
+
end
|
41
|
+
|
42
|
+
def insert_at(position, new_jobs, *args, **kwargs, &blk)
|
43
|
+
chain = self.class.get_chain_parameter(base_job)
|
44
|
+
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
45
|
+
new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
|
46
|
+
elsif args.count > 0 || kwargs.count > 0
|
47
|
+
raise "Unexpected number of arguments"
|
41
48
|
end
|
49
|
+
new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
|
50
|
+
chain.insert(position, *new_jobs)
|
51
|
+
end
|
42
52
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
53
|
+
def insert(new_jobs, *args, **kwargs, &blk)
|
54
|
+
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
55
|
+
job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
|
56
|
+
new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
|
57
|
+
kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
|
58
|
+
else
|
59
|
+
invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
|
60
|
+
raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
|
61
|
+
raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
|
62
|
+
raise "Unexpected number of arguments" if args.length > 0
|
52
63
|
end
|
53
64
|
|
54
|
-
|
55
|
-
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
56
|
-
job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
|
57
|
-
new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
|
58
|
-
kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
|
59
|
-
else
|
60
|
-
invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
|
61
|
-
raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
|
62
|
-
raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
|
63
|
-
raise "Unexpected number of arguments" if args.length > 0
|
64
|
-
end
|
65
|
+
new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
|
65
66
|
|
66
|
-
|
67
|
+
if !kwargs.present?
|
68
|
+
insert_at(-1, new_jobs)
|
69
|
+
else
|
70
|
+
placement = kwargs.keys[0]
|
71
|
+
relative_to = kwargs.values[0]
|
67
72
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
placement = kwargs.keys[0]
|
72
|
-
relative_to = kwargs.values[0]
|
73
|
+
matching_jobs = find_matching_jobs(relative_to).to_a
|
74
|
+
raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
|
75
|
+
raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
|
73
76
|
|
74
|
-
|
75
|
-
|
76
|
-
raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
|
77
|
+
relative_job, parent_job, sub_index = matching_jobs[0]
|
78
|
+
needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
|
77
79
|
|
78
|
-
|
79
|
-
needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
|
80
|
+
chain = self.class.get_chain_parameter(parent_job)
|
80
81
|
|
82
|
+
if parent_job[:job] != needed_parent_type
|
83
|
+
old_job = chain[sub_index]
|
84
|
+
parent_job = chain[sub_index] = {
|
85
|
+
job: needed_parent_type,
|
86
|
+
parameters: [],
|
87
|
+
}
|
88
|
+
sub_index = 0
|
81
89
|
chain = self.class.get_chain_parameter(parent_job)
|
90
|
+
chain << old_job
|
91
|
+
end
|
82
92
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
}
|
89
|
-
sub_index = 0
|
90
|
-
chain = self.class.get_chain_parameter(parent_job)
|
91
|
-
chain << old_job
|
92
|
-
end
|
93
|
-
|
94
|
-
if placement == :with
|
95
|
-
chain.insert(-1, *new_jobs)
|
96
|
-
else
|
97
|
-
sub_index += 1 if placement == :after
|
98
|
-
chain.insert(sub_index, *new_jobs)
|
99
|
-
end
|
93
|
+
if placement == :with
|
94
|
+
chain.insert(-1, *new_jobs)
|
95
|
+
else
|
96
|
+
sub_index += 1 if placement == :after
|
97
|
+
chain.insert(sub_index, *new_jobs)
|
100
98
|
end
|
101
99
|
end
|
100
|
+
end
|
102
101
|
|
103
|
-
|
104
|
-
|
105
|
-
|
102
|
+
def empty?
|
103
|
+
self.class.get_chain_parameter(self).empty?
|
104
|
+
end
|
106
105
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
106
|
+
def get_sub_chain(sub_type)
|
107
|
+
matching_jobs = find_matching_jobs(sub_type).to_a
|
108
|
+
raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
|
109
|
+
return nil if matching_jobs.count == 0
|
111
110
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
111
|
+
job = matching_jobs[0][0]
|
112
|
+
job = self.class.new(job) unless job.is_a?(ChainBuilder)
|
113
|
+
job
|
114
|
+
end
|
116
115
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
end
|
125
|
-
job_def
|
116
|
+
def normalize!(job_def = self.base_job)
|
117
|
+
if job_def.is_a?(ChainBuilder)
|
118
|
+
job_def.normalize!
|
119
|
+
else
|
120
|
+
job_def[:job] = job_def[:job].to_s
|
121
|
+
if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
|
122
|
+
chain.map! { |sub_job| normalize!(sub_job) }
|
126
123
|
end
|
124
|
+
job_def
|
127
125
|
end
|
126
|
+
end
|
128
127
|
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
128
|
+
def apply_block(&blk)
|
129
|
+
return unless blk.present?
|
130
|
+
instance_exec(&blk)
|
131
|
+
end
|
133
132
|
|
134
|
-
|
133
|
+
private
|
135
134
|
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
135
|
+
def build_job_hash(job, args: [], kwargs: {}, &blk)
|
136
|
+
hsh = {
|
137
|
+
job: job,
|
138
|
+
args: args,
|
139
|
+
kwargs: kwargs,
|
140
|
+
}
|
141
|
+
self.class.new(hsh).apply_block(&blk) if blk.present?
|
142
|
+
hsh
|
143
|
+
end
|
145
144
|
|
146
|
-
|
147
|
-
|
145
|
+
def find_matching_jobs(search_job, parent_job = self.base_job)
|
146
|
+
return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
|
148
147
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
end
|
148
|
+
sub_jobs = self.class.get_chain_parameter(parent_job)
|
149
|
+
sub_jobs.each_with_index do |sub_job, i|
|
150
|
+
if sub_job[:job].to_s == search_job.to_s
|
151
|
+
yield [sub_job, parent_job, i]
|
152
|
+
elsif self.class._job_type_definitions[sub_job[:job].to_s]
|
153
|
+
find_matching_jobs(search_job, sub_job) { |item| yield item }
|
156
154
|
end
|
157
155
|
end
|
156
|
+
end
|
158
157
|
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
end
|
163
|
-
nil
|
158
|
+
def find_parent_job(job_def)
|
159
|
+
iterate_job_tree do |job, path|
|
160
|
+
return path[-1] if job == job_def
|
164
161
|
end
|
162
|
+
nil
|
163
|
+
end
|
165
164
|
|
166
|
-
|
167
|
-
|
165
|
+
def iterate_job_tree(root: self.base_job, path: [], &blk)
|
166
|
+
blk.call(root, path)
|
168
167
|
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
end
|
168
|
+
if self.class._job_type_definitions[root[:job]]
|
169
|
+
sub_jobs = self.class.get_chain_parameter(root)
|
170
|
+
sub_jobs.each_with_index do |sub_job, i|
|
171
|
+
iterate_job_tree(root: sub_job, path: [*path, root], &blk)
|
174
172
|
end
|
175
173
|
end
|
174
|
+
end
|
176
175
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
end
|
176
|
+
class << self
|
177
|
+
# Support builder syntaxt/DSL
|
178
|
+
# Chain.build(ConcurrentBatchJob) do
|
179
|
+
# insert(SomeJob, arg1, kwarg: 1)
|
180
|
+
# insert(SerialBatchJob) do
|
181
|
+
# insert(SomeJob, arg1, kwarg: 1)
|
182
|
+
# end
|
183
|
+
# end
|
184
|
+
def build(job, *args, **kwargs, &blk)
|
185
|
+
new(job).tap do |ch|
|
186
|
+
ch.base_job[:args] = args
|
187
|
+
ch.base_job[:kwargs] = kwargs
|
188
|
+
ch.apply_block(&blk)
|
191
189
|
end
|
190
|
+
end
|
191
|
+
|
192
|
+
def _job_type_definitions
|
193
|
+
@job_type_definitions ||= {}
|
194
|
+
end
|
195
|
+
|
196
|
+
def register_chain_job(job_class, chain_parameter, **options)
|
197
|
+
_job_type_definitions[job_class.to_s] = {
|
198
|
+
**options,
|
199
|
+
chain_parameter: chain_parameter,
|
200
|
+
}
|
201
|
+
end
|
192
202
|
|
193
|
-
|
194
|
-
|
203
|
+
def get_chain_parameter(job_def, raise_error: true)
|
204
|
+
unless _job_type_definitions[job_def[:job].to_s].present?
|
205
|
+
raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
|
206
|
+
return nil
|
195
207
|
end
|
196
208
|
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
209
|
+
key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
|
210
|
+
if key.is_a?(Numeric)
|
211
|
+
job_def[:args][key] ||= []
|
212
|
+
else
|
213
|
+
job_def[:kwargs][key] ||= []
|
202
214
|
end
|
215
|
+
end
|
203
216
|
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
217
|
+
# TODO: Add a Chain progress web View
|
218
|
+
# Augment Batch tree-view with Chain data
|
219
|
+
# > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
|
220
|
+
# > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
|
221
|
+
# > Chain-jobs will be supplied chain_id and chain_step_id metadata
|
222
|
+
# > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
|
223
|
+
# > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
|
224
|
+
# > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
|
225
|
+
def enqueue_job(job_def)
|
226
|
+
job_class = job_def[:job].constantize
|
227
|
+
job_args = job_def[:args] || job_def[:parameters] || []
|
228
|
+
job_kwargs = job_def[:kwargs] || {}
|
229
|
+
|
230
|
+
# Legacy Support
|
231
|
+
if job_def[:options]
|
232
|
+
job_args << {} unless job_args[-1].is_a?(Hash)
|
233
|
+
job_args[-1].merge!(job_def[:options])
|
216
234
|
end
|
217
235
|
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
# > Chain-jobs will be supplied chain_id and chain_step_id metadata
|
223
|
-
# > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
|
224
|
-
# > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
|
225
|
-
# > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
|
226
|
-
def enqueue_job(job_def)
|
227
|
-
job_class = job_def[:job].constantize
|
228
|
-
job_args = job_def[:args] || job_def[:parameters] || []
|
229
|
-
job_kwargs = job_def[:kwargs] || {}
|
230
|
-
|
231
|
-
# Legacy Support
|
232
|
-
if job_def[:options]
|
233
|
-
job_args << {} unless job_args[-1].is_a?(Hash)
|
234
|
-
job_args[-1].merge!(job_def[:options])
|
235
|
-
end
|
236
|
-
|
237
|
-
if job_class.respond_to? :perform_async
|
238
|
-
job_class.perform_async(*job_args, **job_kwargs)
|
239
|
-
else
|
240
|
-
job_class.perform_later(*job_args, **job_kwargs)
|
241
|
-
end
|
236
|
+
if job_class.respond_to? :perform_async
|
237
|
+
job_class.perform_async(*job_args, **job_kwargs)
|
238
|
+
else
|
239
|
+
job_class.perform_later(*job_args, **job_kwargs)
|
242
240
|
end
|
243
241
|
end
|
244
242
|
end
|
245
|
-
|
246
|
-
ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
|
247
|
-
ChainBuilder.register_chain_job(SerialBatchJob, 0)
|
248
243
|
end
|
244
|
+
|
245
|
+
ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
|
246
|
+
ChainBuilder.register_chain_job(SerialBatchJob, 0)
|
249
247
|
end
|
@@ -7,7 +7,7 @@ end
|
|
7
7
|
|
8
8
|
require_relative "web/helpers"
|
9
9
|
|
10
|
-
module CanvasSync::JobBatches::Sidekiq
|
10
|
+
module CanvasSync::JobBatches::Compat::Sidekiq
|
11
11
|
module Web
|
12
12
|
DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
|
13
13
|
Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
|
@@ -204,14 +204,14 @@ end
|
|
204
204
|
|
205
205
|
if defined?(::Sidekiq::Web)
|
206
206
|
rules = []
|
207
|
-
rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Sidekiq::Web::DEV_MODE
|
207
|
+
rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Compat::Sidekiq::Web::DEV_MODE
|
208
208
|
|
209
209
|
::Sidekiq::Web.use Rack::Static, urls: ["/batches_assets"],
|
210
210
|
root: File.expand_path("#{File.dirname(__FILE__)}/web"),
|
211
211
|
cascade: true,
|
212
212
|
header_rules: rules
|
213
213
|
|
214
|
-
::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
|
214
|
+
::Sidekiq::Web.register CanvasSync::JobBatches::Compat::Sidekiq::Web
|
215
215
|
::Sidekiq::Web.tabs["Batches"] = "batches"
|
216
216
|
::Sidekiq::Web.tabs["Pools"] = "pools"
|
217
217
|
::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
|
@@ -3,8 +3,8 @@ begin
|
|
3
3
|
rescue LoadError
|
4
4
|
end
|
5
5
|
|
6
|
-
module CanvasSync
|
7
|
-
module
|
6
|
+
module CanvasSync::JobBatches
|
7
|
+
module Compat
|
8
8
|
module Sidekiq
|
9
9
|
module WorkerExtension
|
10
10
|
def bid
|
@@ -41,6 +41,8 @@ module CanvasSync
|
|
41
41
|
end
|
42
42
|
|
43
43
|
class ClientMiddleware
|
44
|
+
include ::Sidekiq::ClientMiddleware if defined? ::Sidekiq::ClientMiddleware
|
45
|
+
|
44
46
|
def call(_worker, msg, _queue, _redis_pool = nil)
|
45
47
|
if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
|
46
48
|
batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
|
@@ -49,12 +51,14 @@ module CanvasSync
|
|
49
51
|
end
|
50
52
|
|
51
53
|
def should_handle_batch?(msg)
|
52
|
-
return false if JobBatches::Sidekiq.is_activejob_job?(msg)
|
54
|
+
return false if CanvasSync::JobBatches::Compat::Sidekiq.is_activejob_job?(msg)
|
53
55
|
true
|
54
56
|
end
|
55
57
|
end
|
56
58
|
|
57
59
|
class ServerMiddleware
|
60
|
+
include ::Sidekiq::ServerMiddleware if defined? ::Sidekiq::ServerMiddleware
|
61
|
+
|
58
62
|
def call(_worker, msg, _queue)
|
59
63
|
if (bid = msg['bid'])
|
60
64
|
prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
@@ -78,7 +82,7 @@ module CanvasSync
|
|
78
82
|
def self.is_activejob_job?(msg)
|
79
83
|
return false unless defined?(::ActiveJob)
|
80
84
|
|
81
|
-
msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < JobBatches::ActiveJob::BatchAwareJob
|
85
|
+
msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < CanvasSync::JobBatches::Compat::ActiveJob::BatchAwareJob
|
82
86
|
end
|
83
87
|
|
84
88
|
def self.switch_tenant(job)
|
@@ -91,42 +95,51 @@ module CanvasSync
|
|
91
95
|
end
|
92
96
|
end
|
93
97
|
|
98
|
+
def self.sidekiq_middleware(placement, &blk)
|
99
|
+
install_middleware = ->(config) do
|
100
|
+
config.send("#{placement}_middleware") do |chain|
|
101
|
+
blk.call(chain)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
::Sidekiq.configure_client(&install_middleware) if placement == :client
|
106
|
+
::Sidekiq.configure_server(&install_middleware)
|
107
|
+
end
|
108
|
+
|
94
109
|
def self.configure
|
95
|
-
if
|
110
|
+
return if @already_configured
|
111
|
+
@already_configured = true
|
112
|
+
|
113
|
+
if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != CanvasSync::JobBatches::Batch
|
96
114
|
print "WARNING: Detected Sidekiq Pro or sidekiq-batch. CanvasSync JobBatches may not be fully compatible!"
|
97
115
|
end
|
98
116
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
chain.add JobBatches::Sidekiq::ClientMiddleware
|
103
|
-
end
|
117
|
+
sidekiq_middleware(:client) do |chain|
|
118
|
+
chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
|
119
|
+
chain.add CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware
|
104
120
|
end
|
105
|
-
::Sidekiq.configure_server do |config|
|
106
|
-
config.client_middleware do |chain|
|
107
|
-
chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
|
108
|
-
chain.add JobBatches::Sidekiq::ClientMiddleware
|
109
|
-
end
|
110
121
|
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
122
|
+
sidekiq_middleware(:server) do |chain|
|
123
|
+
chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
|
124
|
+
chain.add CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware
|
125
|
+
end
|
115
126
|
|
127
|
+
::Sidekiq.configure_server do |config|
|
116
128
|
config.death_handlers << ->(job, ex) do
|
117
129
|
switch_tenant(job) do
|
118
130
|
if is_activejob_job?(job)
|
119
|
-
JobBatches::ActiveJob.handle_job_death(job["args"][0], ex)
|
131
|
+
CanvasSync::JobBatches::Compat::ActiveJob.handle_job_death(job["args"][0], ex)
|
120
132
|
elsif job['bid'].present?
|
121
133
|
::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
|
122
134
|
end
|
123
135
|
end
|
124
136
|
end
|
125
137
|
end
|
138
|
+
|
126
139
|
::Sidekiq.const_set(:Batch, CanvasSync::JobBatches::Batch)
|
127
140
|
# This alias helps apartment-sidekiq set itself up correctly
|
128
|
-
::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
|
129
|
-
::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
|
141
|
+
::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware)
|
142
|
+
::Sidekiq::Worker.send(:include, CanvasSync::JobBatches::Compat::Sidekiq::WorkerExtension)
|
130
143
|
Batch::Callback.worker_class = SidekiqCallbackWorker
|
131
144
|
end
|
132
145
|
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
|
2
|
+
module CanvasSync::JobBatches
|
3
|
+
module Compat
|
4
|
+
def self.load_compat(name)
|
5
|
+
name = name.to_s
|
6
|
+
begin
|
7
|
+
require name
|
8
|
+
rescue LoadError
|
9
|
+
end
|
10
|
+
|
11
|
+
if name.classify.safe_constantize
|
12
|
+
require_relative "./compat/#{name}"
|
13
|
+
"CanvasSync::JobBatches::Compat::#{name.classify}".constantize.configure
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
load_compat(:active_job)
|
18
|
+
load_compat(:sidekiq)
|
19
|
+
end
|
20
|
+
end
|