canvas_sync 0.21.1 → 0.22.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/canvas_sync/concerns/auto_relations.rb +11 -0
- data/lib/canvas_sync/config.rb +3 -5
- data/lib/canvas_sync/generators/templates/models/rubric.rb +2 -1
- data/lib/canvas_sync/job_batches/batch.rb +432 -402
- data/lib/canvas_sync/job_batches/callback.rb +100 -114
- data/lib/canvas_sync/job_batches/chain_builder.rb +194 -196
- data/lib/canvas_sync/job_batches/{active_job.rb → compat/active_job.rb} +2 -2
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/helpers.rb +1 -1
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web.rb +3 -3
- data/lib/canvas_sync/job_batches/{sidekiq.rb → compat/sidekiq.rb} +35 -22
- data/lib/canvas_sync/job_batches/compat.rb +20 -0
- data/lib/canvas_sync/job_batches/context_hash.rb +124 -126
- data/lib/canvas_sync/job_batches/jobs/base_job.rb +2 -4
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +125 -127
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/pool.rb +193 -195
- data/lib/canvas_sync/job_batches/redis_model.rb +50 -52
- data/lib/canvas_sync/job_batches/redis_script.rb +129 -131
- data/lib/canvas_sync/job_batches/status.rb +85 -87
- data/lib/canvas_sync/job_uniqueness/compat/active_job.rb +75 -0
- data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb +135 -0
- data/lib/canvas_sync/job_uniqueness/compat.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/configuration.rb +25 -0
- data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb +47 -0
- data/lib/canvas_sync/job_uniqueness/lock_context.rb +171 -0
- data/lib/canvas_sync/job_uniqueness/locksmith.rb +92 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/base.rb +32 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/log.rb +13 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/null_strategy.rb +9 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/raise.rb +11 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reject.rb +21 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reschedule.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict.rb +41 -0
- data/lib/canvas_sync/job_uniqueness/strategy/base.rb +104 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_and_while_executing.rb +35 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executed.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executing.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_expired.rb +16 -0
- data/lib/canvas_sync/job_uniqueness/strategy/while_executing.rb +26 -0
- data/lib/canvas_sync/job_uniqueness/strategy.rb +27 -0
- data/lib/canvas_sync/job_uniqueness/unique_job_common.rb +79 -0
- data/lib/canvas_sync/misc_helper.rb +1 -1
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +4 -3
- data/spec/dummy/app/models/rubric.rb +2 -1
- data/spec/dummy/config/environments/test.rb +1 -1
- data/spec/job_batching/batch_spec.rb +49 -7
- data/spec/job_batching/{active_job_spec.rb → compat/active_job_spec.rb} +2 -2
- data/spec/job_batching/{sidekiq_spec.rb → compat/sidekiq_spec.rb} +14 -12
- data/spec/job_batching/flow_spec.rb +1 -1
- data/spec/job_batching/integration_helper.rb +1 -1
- data/spec/job_batching/status_spec.rb +2 -2
- data/spec/job_uniqueness/compat/active_job_spec.rb +49 -0
- data/spec/job_uniqueness/compat/sidekiq_spec.rb +68 -0
- data/spec/job_uniqueness/lock_context_spec.rb +95 -0
- data/spec/job_uniqueness/on_conflict/log_spec.rb +11 -0
- data/spec/job_uniqueness/on_conflict/raise_spec.rb +10 -0
- data/spec/job_uniqueness/on_conflict/reschedule_spec.rb +24 -0
- data/spec/job_uniqueness/on_conflict_spec.rb +16 -0
- data/spec/job_uniqueness/spec_helper.rb +14 -0
- data/spec/job_uniqueness/strategy/base_spec.rb +100 -0
- data/spec/job_uniqueness/strategy/until_and_while_executing_spec.rb +48 -0
- data/spec/job_uniqueness/strategy/until_executed_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_executing_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_expired_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/while_executing_spec.rb +33 -0
- data/spec/job_uniqueness/support/lock_strategy.rb +28 -0
- data/spec/job_uniqueness/support/on_conflict.rb +24 -0
- data/spec/job_uniqueness/support/test_worker.rb +19 -0
- data/spec/job_uniqueness/unique_job_common_spec.rb +45 -0
- data/spec/spec_helper.rb +1 -1
- metadata +278 -204
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/css/styles.less +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/batch_tree.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/util.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batch_tree.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batches_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_common.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_jobs_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_pagination.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batch.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batches.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pool.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pools.erb +0 -0
@@ -1,249 +1,247 @@
|
|
1
|
-
module CanvasSync
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
@base_job = build_job_hash(base_type)
|
15
|
-
end
|
16
|
-
|
17
|
-
self.class.get_chain_parameter(base_job)
|
1
|
+
module CanvasSync::JobBatches
|
2
|
+
class ChainBuilder
|
3
|
+
VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
|
4
|
+
|
5
|
+
attr_reader :base_job
|
6
|
+
|
7
|
+
def initialize(base_type = SerialBatchJob)
|
8
|
+
if base_type.is_a?(Hash)
|
9
|
+
@base_job = base_type
|
10
|
+
@base_job[:args] ||= @base_job[:parameters] || []
|
11
|
+
@base_job[:kwargs] ||= {}
|
12
|
+
else
|
13
|
+
@base_job = build_job_hash(base_type)
|
18
14
|
end
|
19
15
|
|
20
|
-
|
21
|
-
|
22
|
-
self.class.enqueue_job(base_job)
|
23
|
-
end
|
16
|
+
self.class.get_chain_parameter(base_job)
|
17
|
+
end
|
24
18
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
# Legacy Support
|
30
|
-
key = :args if key == :parameters
|
19
|
+
def process!
|
20
|
+
normalize!
|
21
|
+
self.class.enqueue_job(base_job)
|
22
|
+
end
|
31
23
|
|
32
|
-
|
33
|
-
|
24
|
+
def [](key)
|
25
|
+
if key.is_a?(Class)
|
26
|
+
get_sub_chain(key)
|
27
|
+
else
|
28
|
+
# Legacy Support
|
29
|
+
key = :args if key == :parameters
|
30
|
+
|
31
|
+
@base_job[key]
|
34
32
|
end
|
33
|
+
end
|
35
34
|
|
36
|
-
|
37
|
-
|
35
|
+
def args; return self[:args]; end
|
36
|
+
def kwargs; return self[:kwargs]; end
|
38
37
|
|
39
|
-
|
40
|
-
|
38
|
+
def <<(new_job)
|
39
|
+
insert_at(-1, new_job)
|
40
|
+
end
|
41
|
+
|
42
|
+
def insert_at(position, new_jobs, *args, **kwargs, &blk)
|
43
|
+
chain = self.class.get_chain_parameter(base_job)
|
44
|
+
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
45
|
+
new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
|
46
|
+
elsif args.count > 0 || kwargs.count > 0
|
47
|
+
raise "Unexpected number of arguments"
|
41
48
|
end
|
49
|
+
new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
|
50
|
+
chain.insert(position, *new_jobs)
|
51
|
+
end
|
42
52
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
53
|
+
def insert(new_jobs, *args, **kwargs, &blk)
|
54
|
+
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
55
|
+
job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
|
56
|
+
new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
|
57
|
+
kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
|
58
|
+
else
|
59
|
+
invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
|
60
|
+
raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
|
61
|
+
raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
|
62
|
+
raise "Unexpected number of arguments" if args.length > 0
|
52
63
|
end
|
53
64
|
|
54
|
-
|
55
|
-
if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
|
56
|
-
job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
|
57
|
-
new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
|
58
|
-
kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
|
59
|
-
else
|
60
|
-
invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
|
61
|
-
raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
|
62
|
-
raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
|
63
|
-
raise "Unexpected number of arguments" if args.length > 0
|
64
|
-
end
|
65
|
+
new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
|
65
66
|
|
66
|
-
|
67
|
+
if !kwargs.present?
|
68
|
+
insert_at(-1, new_jobs)
|
69
|
+
else
|
70
|
+
placement = kwargs.keys[0]
|
71
|
+
relative_to = kwargs.values[0]
|
67
72
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
placement = kwargs.keys[0]
|
72
|
-
relative_to = kwargs.values[0]
|
73
|
+
matching_jobs = find_matching_jobs(relative_to).to_a
|
74
|
+
raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
|
75
|
+
raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
|
73
76
|
|
74
|
-
|
75
|
-
|
76
|
-
raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
|
77
|
+
relative_job, parent_job, sub_index = matching_jobs[0]
|
78
|
+
needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
|
77
79
|
|
78
|
-
|
79
|
-
needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
|
80
|
+
chain = self.class.get_chain_parameter(parent_job)
|
80
81
|
|
82
|
+
if parent_job[:job] != needed_parent_type
|
83
|
+
old_job = chain[sub_index]
|
84
|
+
parent_job = chain[sub_index] = {
|
85
|
+
job: needed_parent_type,
|
86
|
+
parameters: [],
|
87
|
+
}
|
88
|
+
sub_index = 0
|
81
89
|
chain = self.class.get_chain_parameter(parent_job)
|
90
|
+
chain << old_job
|
91
|
+
end
|
82
92
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
}
|
89
|
-
sub_index = 0
|
90
|
-
chain = self.class.get_chain_parameter(parent_job)
|
91
|
-
chain << old_job
|
92
|
-
end
|
93
|
-
|
94
|
-
if placement == :with
|
95
|
-
chain.insert(-1, *new_jobs)
|
96
|
-
else
|
97
|
-
sub_index += 1 if placement == :after
|
98
|
-
chain.insert(sub_index, *new_jobs)
|
99
|
-
end
|
93
|
+
if placement == :with
|
94
|
+
chain.insert(-1, *new_jobs)
|
95
|
+
else
|
96
|
+
sub_index += 1 if placement == :after
|
97
|
+
chain.insert(sub_index, *new_jobs)
|
100
98
|
end
|
101
99
|
end
|
100
|
+
end
|
102
101
|
|
103
|
-
|
104
|
-
|
105
|
-
|
102
|
+
def empty?
|
103
|
+
self.class.get_chain_parameter(self).empty?
|
104
|
+
end
|
106
105
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
106
|
+
def get_sub_chain(sub_type)
|
107
|
+
matching_jobs = find_matching_jobs(sub_type).to_a
|
108
|
+
raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
|
109
|
+
return nil if matching_jobs.count == 0
|
111
110
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
111
|
+
job = matching_jobs[0][0]
|
112
|
+
job = self.class.new(job) unless job.is_a?(ChainBuilder)
|
113
|
+
job
|
114
|
+
end
|
116
115
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
end
|
125
|
-
job_def
|
116
|
+
def normalize!(job_def = self.base_job)
|
117
|
+
if job_def.is_a?(ChainBuilder)
|
118
|
+
job_def.normalize!
|
119
|
+
else
|
120
|
+
job_def[:job] = job_def[:job].to_s
|
121
|
+
if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
|
122
|
+
chain.map! { |sub_job| normalize!(sub_job) }
|
126
123
|
end
|
124
|
+
job_def
|
127
125
|
end
|
126
|
+
end
|
128
127
|
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
128
|
+
def apply_block(&blk)
|
129
|
+
return unless blk.present?
|
130
|
+
instance_exec(&blk)
|
131
|
+
end
|
133
132
|
|
134
|
-
|
133
|
+
private
|
135
134
|
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
135
|
+
def build_job_hash(job, args: [], kwargs: {}, &blk)
|
136
|
+
hsh = {
|
137
|
+
job: job,
|
138
|
+
args: args,
|
139
|
+
kwargs: kwargs,
|
140
|
+
}
|
141
|
+
self.class.new(hsh).apply_block(&blk) if blk.present?
|
142
|
+
hsh
|
143
|
+
end
|
145
144
|
|
146
|
-
|
147
|
-
|
145
|
+
def find_matching_jobs(search_job, parent_job = self.base_job)
|
146
|
+
return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
|
148
147
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
end
|
148
|
+
sub_jobs = self.class.get_chain_parameter(parent_job)
|
149
|
+
sub_jobs.each_with_index do |sub_job, i|
|
150
|
+
if sub_job[:job].to_s == search_job.to_s
|
151
|
+
yield [sub_job, parent_job, i]
|
152
|
+
elsif self.class._job_type_definitions[sub_job[:job].to_s]
|
153
|
+
find_matching_jobs(search_job, sub_job) { |item| yield item }
|
156
154
|
end
|
157
155
|
end
|
156
|
+
end
|
158
157
|
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
end
|
163
|
-
nil
|
158
|
+
def find_parent_job(job_def)
|
159
|
+
iterate_job_tree do |job, path|
|
160
|
+
return path[-1] if job == job_def
|
164
161
|
end
|
162
|
+
nil
|
163
|
+
end
|
165
164
|
|
166
|
-
|
167
|
-
|
165
|
+
def iterate_job_tree(root: self.base_job, path: [], &blk)
|
166
|
+
blk.call(root, path)
|
168
167
|
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
end
|
168
|
+
if self.class._job_type_definitions[root[:job]]
|
169
|
+
sub_jobs = self.class.get_chain_parameter(root)
|
170
|
+
sub_jobs.each_with_index do |sub_job, i|
|
171
|
+
iterate_job_tree(root: sub_job, path: [*path, root], &blk)
|
174
172
|
end
|
175
173
|
end
|
174
|
+
end
|
176
175
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
end
|
176
|
+
class << self
|
177
|
+
# Support builder syntaxt/DSL
|
178
|
+
# Chain.build(ConcurrentBatchJob) do
|
179
|
+
# insert(SomeJob, arg1, kwarg: 1)
|
180
|
+
# insert(SerialBatchJob) do
|
181
|
+
# insert(SomeJob, arg1, kwarg: 1)
|
182
|
+
# end
|
183
|
+
# end
|
184
|
+
def build(job, *args, **kwargs, &blk)
|
185
|
+
new(job).tap do |ch|
|
186
|
+
ch.base_job[:args] = args
|
187
|
+
ch.base_job[:kwargs] = kwargs
|
188
|
+
ch.apply_block(&blk)
|
191
189
|
end
|
190
|
+
end
|
191
|
+
|
192
|
+
def _job_type_definitions
|
193
|
+
@job_type_definitions ||= {}
|
194
|
+
end
|
195
|
+
|
196
|
+
def register_chain_job(job_class, chain_parameter, **options)
|
197
|
+
_job_type_definitions[job_class.to_s] = {
|
198
|
+
**options,
|
199
|
+
chain_parameter: chain_parameter,
|
200
|
+
}
|
201
|
+
end
|
192
202
|
|
193
|
-
|
194
|
-
|
203
|
+
def get_chain_parameter(job_def, raise_error: true)
|
204
|
+
unless _job_type_definitions[job_def[:job].to_s].present?
|
205
|
+
raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
|
206
|
+
return nil
|
195
207
|
end
|
196
208
|
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
209
|
+
key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
|
210
|
+
if key.is_a?(Numeric)
|
211
|
+
job_def[:args][key] ||= []
|
212
|
+
else
|
213
|
+
job_def[:kwargs][key] ||= []
|
202
214
|
end
|
215
|
+
end
|
203
216
|
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
217
|
+
# TODO: Add a Chain progress web View
|
218
|
+
# Augment Batch tree-view with Chain data
|
219
|
+
# > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
|
220
|
+
# > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
|
221
|
+
# > Chain-jobs will be supplied chain_id and chain_step_id metadata
|
222
|
+
# > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
|
223
|
+
# > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
|
224
|
+
# > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
|
225
|
+
def enqueue_job(job_def)
|
226
|
+
job_class = job_def[:job].constantize
|
227
|
+
job_args = job_def[:args] || job_def[:parameters] || []
|
228
|
+
job_kwargs = job_def[:kwargs] || {}
|
229
|
+
|
230
|
+
# Legacy Support
|
231
|
+
if job_def[:options]
|
232
|
+
job_args << {} unless job_args[-1].is_a?(Hash)
|
233
|
+
job_args[-1].merge!(job_def[:options])
|
216
234
|
end
|
217
235
|
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
# > Chain-jobs will be supplied chain_id and chain_step_id metadata
|
223
|
-
# > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
|
224
|
-
# > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
|
225
|
-
# > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
|
226
|
-
def enqueue_job(job_def)
|
227
|
-
job_class = job_def[:job].constantize
|
228
|
-
job_args = job_def[:args] || job_def[:parameters] || []
|
229
|
-
job_kwargs = job_def[:kwargs] || {}
|
230
|
-
|
231
|
-
# Legacy Support
|
232
|
-
if job_def[:options]
|
233
|
-
job_args << {} unless job_args[-1].is_a?(Hash)
|
234
|
-
job_args[-1].merge!(job_def[:options])
|
235
|
-
end
|
236
|
-
|
237
|
-
if job_class.respond_to? :perform_async
|
238
|
-
job_class.perform_async(*job_args, **job_kwargs)
|
239
|
-
else
|
240
|
-
job_class.perform_later(*job_args, **job_kwargs)
|
241
|
-
end
|
236
|
+
if job_class.respond_to? :perform_async
|
237
|
+
job_class.perform_async(*job_args, **job_kwargs)
|
238
|
+
else
|
239
|
+
job_class.perform_later(*job_args, **job_kwargs)
|
242
240
|
end
|
243
241
|
end
|
244
242
|
end
|
245
|
-
|
246
|
-
ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
|
247
|
-
ChainBuilder.register_chain_job(SerialBatchJob, 0)
|
248
243
|
end
|
244
|
+
|
245
|
+
ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
|
246
|
+
ChainBuilder.register_chain_job(SerialBatchJob, 0)
|
249
247
|
end
|
@@ -7,7 +7,7 @@ end
|
|
7
7
|
|
8
8
|
require_relative "web/helpers"
|
9
9
|
|
10
|
-
module CanvasSync::JobBatches::Sidekiq
|
10
|
+
module CanvasSync::JobBatches::Compat::Sidekiq
|
11
11
|
module Web
|
12
12
|
DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
|
13
13
|
Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
|
@@ -204,14 +204,14 @@ end
|
|
204
204
|
|
205
205
|
if defined?(::Sidekiq::Web)
|
206
206
|
rules = []
|
207
|
-
rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Sidekiq::Web::DEV_MODE
|
207
|
+
rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Compat::Sidekiq::Web::DEV_MODE
|
208
208
|
|
209
209
|
::Sidekiq::Web.use Rack::Static, urls: ["/batches_assets"],
|
210
210
|
root: File.expand_path("#{File.dirname(__FILE__)}/web"),
|
211
211
|
cascade: true,
|
212
212
|
header_rules: rules
|
213
213
|
|
214
|
-
::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
|
214
|
+
::Sidekiq::Web.register CanvasSync::JobBatches::Compat::Sidekiq::Web
|
215
215
|
::Sidekiq::Web.tabs["Batches"] = "batches"
|
216
216
|
::Sidekiq::Web.tabs["Pools"] = "pools"
|
217
217
|
::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
|
@@ -3,8 +3,8 @@ begin
|
|
3
3
|
rescue LoadError
|
4
4
|
end
|
5
5
|
|
6
|
-
module CanvasSync
|
7
|
-
module
|
6
|
+
module CanvasSync::JobBatches
|
7
|
+
module Compat
|
8
8
|
module Sidekiq
|
9
9
|
module WorkerExtension
|
10
10
|
def bid
|
@@ -41,6 +41,8 @@ module CanvasSync
|
|
41
41
|
end
|
42
42
|
|
43
43
|
class ClientMiddleware
|
44
|
+
include ::Sidekiq::ClientMiddleware if defined? ::Sidekiq::ClientMiddleware
|
45
|
+
|
44
46
|
def call(_worker, msg, _queue, _redis_pool = nil)
|
45
47
|
if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
|
46
48
|
batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
|
@@ -49,12 +51,14 @@ module CanvasSync
|
|
49
51
|
end
|
50
52
|
|
51
53
|
def should_handle_batch?(msg)
|
52
|
-
return false if JobBatches::Sidekiq.is_activejob_job?(msg)
|
54
|
+
return false if CanvasSync::JobBatches::Compat::Sidekiq.is_activejob_job?(msg)
|
53
55
|
true
|
54
56
|
end
|
55
57
|
end
|
56
58
|
|
57
59
|
class ServerMiddleware
|
60
|
+
include ::Sidekiq::ServerMiddleware if defined? ::Sidekiq::ServerMiddleware
|
61
|
+
|
58
62
|
def call(_worker, msg, _queue)
|
59
63
|
if (bid = msg['bid'])
|
60
64
|
prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
|
@@ -78,7 +82,7 @@ module CanvasSync
|
|
78
82
|
def self.is_activejob_job?(msg)
|
79
83
|
return false unless defined?(::ActiveJob)
|
80
84
|
|
81
|
-
msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < JobBatches::ActiveJob::BatchAwareJob
|
85
|
+
msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < CanvasSync::JobBatches::Compat::ActiveJob::BatchAwareJob
|
82
86
|
end
|
83
87
|
|
84
88
|
def self.switch_tenant(job)
|
@@ -91,42 +95,51 @@ module CanvasSync
|
|
91
95
|
end
|
92
96
|
end
|
93
97
|
|
98
|
+
def self.sidekiq_middleware(placement, &blk)
|
99
|
+
install_middleware = ->(config) do
|
100
|
+
config.send("#{placement}_middleware") do |chain|
|
101
|
+
blk.call(chain)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
::Sidekiq.configure_client(&install_middleware) if placement == :client
|
106
|
+
::Sidekiq.configure_server(&install_middleware)
|
107
|
+
end
|
108
|
+
|
94
109
|
def self.configure
|
95
|
-
if
|
110
|
+
return if @already_configured
|
111
|
+
@already_configured = true
|
112
|
+
|
113
|
+
if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != CanvasSync::JobBatches::Batch
|
96
114
|
print "WARNING: Detected Sidekiq Pro or sidekiq-batch. CanvasSync JobBatches may not be fully compatible!"
|
97
115
|
end
|
98
116
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
chain.add JobBatches::Sidekiq::ClientMiddleware
|
103
|
-
end
|
117
|
+
sidekiq_middleware(:client) do |chain|
|
118
|
+
chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
|
119
|
+
chain.add CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware
|
104
120
|
end
|
105
|
-
::Sidekiq.configure_server do |config|
|
106
|
-
config.client_middleware do |chain|
|
107
|
-
chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
|
108
|
-
chain.add JobBatches::Sidekiq::ClientMiddleware
|
109
|
-
end
|
110
121
|
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
122
|
+
sidekiq_middleware(:server) do |chain|
|
123
|
+
chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
|
124
|
+
chain.add CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware
|
125
|
+
end
|
115
126
|
|
127
|
+
::Sidekiq.configure_server do |config|
|
116
128
|
config.death_handlers << ->(job, ex) do
|
117
129
|
switch_tenant(job) do
|
118
130
|
if is_activejob_job?(job)
|
119
|
-
JobBatches::ActiveJob.handle_job_death(job["args"][0], ex)
|
131
|
+
CanvasSync::JobBatches::Compat::ActiveJob.handle_job_death(job["args"][0], ex)
|
120
132
|
elsif job['bid'].present?
|
121
133
|
::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
|
122
134
|
end
|
123
135
|
end
|
124
136
|
end
|
125
137
|
end
|
138
|
+
|
126
139
|
::Sidekiq.const_set(:Batch, CanvasSync::JobBatches::Batch)
|
127
140
|
# This alias helps apartment-sidekiq set itself up correctly
|
128
|
-
::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
|
129
|
-
::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
|
141
|
+
::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware)
|
142
|
+
::Sidekiq::Worker.send(:include, CanvasSync::JobBatches::Compat::Sidekiq::WorkerExtension)
|
130
143
|
Batch::Callback.worker_class = SidekiqCallbackWorker
|
131
144
|
end
|
132
145
|
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
|
2
|
+
module CanvasSync::JobBatches
|
3
|
+
module Compat
|
4
|
+
def self.load_compat(name)
|
5
|
+
name = name.to_s
|
6
|
+
begin
|
7
|
+
require name
|
8
|
+
rescue LoadError
|
9
|
+
end
|
10
|
+
|
11
|
+
if name.classify.safe_constantize
|
12
|
+
require_relative "./compat/#{name}"
|
13
|
+
"CanvasSync::JobBatches::Compat::#{name.classify}".constantize.configure
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
load_compat(:active_job)
|
18
|
+
load_compat(:sidekiq)
|
19
|
+
end
|
20
|
+
end
|