canvas_sync 0.21.1 → 0.22.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/lib/canvas_sync/concerns/auto_relations.rb +11 -0
  3. data/lib/canvas_sync/config.rb +3 -5
  4. data/lib/canvas_sync/generators/templates/models/rubric.rb +2 -1
  5. data/lib/canvas_sync/job_batches/batch.rb +432 -402
  6. data/lib/canvas_sync/job_batches/callback.rb +100 -114
  7. data/lib/canvas_sync/job_batches/chain_builder.rb +194 -196
  8. data/lib/canvas_sync/job_batches/{active_job.rb → compat/active_job.rb} +2 -2
  9. data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/helpers.rb +1 -1
  10. data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web.rb +3 -3
  11. data/lib/canvas_sync/job_batches/{sidekiq.rb → compat/sidekiq.rb} +35 -22
  12. data/lib/canvas_sync/job_batches/compat.rb +20 -0
  13. data/lib/canvas_sync/job_batches/context_hash.rb +124 -126
  14. data/lib/canvas_sync/job_batches/jobs/base_job.rb +2 -4
  15. data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +14 -16
  16. data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +125 -127
  17. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +14 -16
  18. data/lib/canvas_sync/job_batches/pool.rb +193 -195
  19. data/lib/canvas_sync/job_batches/redis_model.rb +50 -52
  20. data/lib/canvas_sync/job_batches/redis_script.rb +129 -131
  21. data/lib/canvas_sync/job_batches/status.rb +85 -87
  22. data/lib/canvas_sync/job_uniqueness/compat/active_job.rb +75 -0
  23. data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb +135 -0
  24. data/lib/canvas_sync/job_uniqueness/compat.rb +20 -0
  25. data/lib/canvas_sync/job_uniqueness/configuration.rb +25 -0
  26. data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb +47 -0
  27. data/lib/canvas_sync/job_uniqueness/lock_context.rb +171 -0
  28. data/lib/canvas_sync/job_uniqueness/locksmith.rb +92 -0
  29. data/lib/canvas_sync/job_uniqueness/on_conflict/base.rb +32 -0
  30. data/lib/canvas_sync/job_uniqueness/on_conflict/log.rb +13 -0
  31. data/lib/canvas_sync/job_uniqueness/on_conflict/null_strategy.rb +9 -0
  32. data/lib/canvas_sync/job_uniqueness/on_conflict/raise.rb +11 -0
  33. data/lib/canvas_sync/job_uniqueness/on_conflict/reject.rb +21 -0
  34. data/lib/canvas_sync/job_uniqueness/on_conflict/reschedule.rb +20 -0
  35. data/lib/canvas_sync/job_uniqueness/on_conflict.rb +41 -0
  36. data/lib/canvas_sync/job_uniqueness/strategy/base.rb +104 -0
  37. data/lib/canvas_sync/job_uniqueness/strategy/until_and_while_executing.rb +35 -0
  38. data/lib/canvas_sync/job_uniqueness/strategy/until_executed.rb +20 -0
  39. data/lib/canvas_sync/job_uniqueness/strategy/until_executing.rb +20 -0
  40. data/lib/canvas_sync/job_uniqueness/strategy/until_expired.rb +16 -0
  41. data/lib/canvas_sync/job_uniqueness/strategy/while_executing.rb +26 -0
  42. data/lib/canvas_sync/job_uniqueness/strategy.rb +27 -0
  43. data/lib/canvas_sync/job_uniqueness/unique_job_common.rb +79 -0
  44. data/lib/canvas_sync/misc_helper.rb +1 -1
  45. data/lib/canvas_sync/version.rb +1 -1
  46. data/lib/canvas_sync.rb +4 -3
  47. data/spec/dummy/app/models/rubric.rb +2 -1
  48. data/spec/dummy/config/environments/test.rb +1 -1
  49. data/spec/job_batching/batch_spec.rb +49 -7
  50. data/spec/job_batching/{active_job_spec.rb → compat/active_job_spec.rb} +2 -2
  51. data/spec/job_batching/{sidekiq_spec.rb → compat/sidekiq_spec.rb} +14 -12
  52. data/spec/job_batching/flow_spec.rb +1 -1
  53. data/spec/job_batching/integration_helper.rb +1 -1
  54. data/spec/job_batching/status_spec.rb +2 -2
  55. data/spec/job_uniqueness/compat/active_job_spec.rb +49 -0
  56. data/spec/job_uniqueness/compat/sidekiq_spec.rb +68 -0
  57. data/spec/job_uniqueness/lock_context_spec.rb +95 -0
  58. data/spec/job_uniqueness/on_conflict/log_spec.rb +11 -0
  59. data/spec/job_uniqueness/on_conflict/raise_spec.rb +10 -0
  60. data/spec/job_uniqueness/on_conflict/reschedule_spec.rb +24 -0
  61. data/spec/job_uniqueness/on_conflict_spec.rb +16 -0
  62. data/spec/job_uniqueness/spec_helper.rb +14 -0
  63. data/spec/job_uniqueness/strategy/base_spec.rb +100 -0
  64. data/spec/job_uniqueness/strategy/until_and_while_executing_spec.rb +48 -0
  65. data/spec/job_uniqueness/strategy/until_executed_spec.rb +23 -0
  66. data/spec/job_uniqueness/strategy/until_executing_spec.rb +23 -0
  67. data/spec/job_uniqueness/strategy/until_expired_spec.rb +23 -0
  68. data/spec/job_uniqueness/strategy/while_executing_spec.rb +33 -0
  69. data/spec/job_uniqueness/support/lock_strategy.rb +28 -0
  70. data/spec/job_uniqueness/support/on_conflict.rb +24 -0
  71. data/spec/job_uniqueness/support/test_worker.rb +19 -0
  72. data/spec/job_uniqueness/unique_job_common_spec.rb +45 -0
  73. data/spec/spec_helper.rb +1 -1
  74. metadata +278 -204
  75. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/css/styles.less +0 -0
  76. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/batch_tree.js +0 -0
  77. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/util.js +0 -0
  78. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batch_tree.erb +0 -0
  79. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batches_table.erb +0 -0
  80. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_common.erb +0 -0
  81. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_jobs_table.erb +0 -0
  82. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_pagination.erb +0 -0
  83. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batch.erb +0 -0
  84. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batches.erb +0 -0
  85. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pool.erb +0 -0
  86. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pools.erb +0 -0
@@ -1,249 +1,247 @@
1
- module CanvasSync
2
- module JobBatches
3
- class ChainBuilder
4
- VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
5
-
6
- attr_reader :base_job
7
-
8
- def initialize(base_type = SerialBatchJob)
9
- if base_type.is_a?(Hash)
10
- @base_job = base_type
11
- @base_job[:args] ||= @base_job[:parameters] || []
12
- @base_job[:kwargs] ||= {}
13
- else
14
- @base_job = build_job_hash(base_type)
15
- end
16
-
17
- self.class.get_chain_parameter(base_job)
1
+ module CanvasSync::JobBatches
2
+ class ChainBuilder
3
+ VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
4
+
5
+ attr_reader :base_job
6
+
7
+ def initialize(base_type = SerialBatchJob)
8
+ if base_type.is_a?(Hash)
9
+ @base_job = base_type
10
+ @base_job[:args] ||= @base_job[:parameters] || []
11
+ @base_job[:kwargs] ||= {}
12
+ else
13
+ @base_job = build_job_hash(base_type)
18
14
  end
19
15
 
20
- def process!
21
- normalize!
22
- self.class.enqueue_job(base_job)
23
- end
16
+ self.class.get_chain_parameter(base_job)
17
+ end
24
18
 
25
- def [](key)
26
- if key.is_a?(Class)
27
- get_sub_chain(key)
28
- else
29
- # Legacy Support
30
- key = :args if key == :parameters
19
+ def process!
20
+ normalize!
21
+ self.class.enqueue_job(base_job)
22
+ end
31
23
 
32
- @base_job[key]
33
- end
24
+ def [](key)
25
+ if key.is_a?(Class)
26
+ get_sub_chain(key)
27
+ else
28
+ # Legacy Support
29
+ key = :args if key == :parameters
30
+
31
+ @base_job[key]
34
32
  end
33
+ end
35
34
 
36
- def args; return self[:args]; end
37
- def kwargs; return self[:kwargs]; end
35
+ def args; return self[:args]; end
36
+ def kwargs; return self[:kwargs]; end
38
37
 
39
- def <<(new_job)
40
- insert_at(-1, new_job)
38
+ def <<(new_job)
39
+ insert_at(-1, new_job)
40
+ end
41
+
42
+ def insert_at(position, new_jobs, *args, **kwargs, &blk)
43
+ chain = self.class.get_chain_parameter(base_job)
44
+ if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
45
+ new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
46
+ elsif args.count > 0 || kwargs.count > 0
47
+ raise "Unexpected number of arguments"
41
48
  end
49
+ new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
50
+ chain.insert(position, *new_jobs)
51
+ end
42
52
 
43
- def insert_at(position, new_jobs, *args, **kwargs, &blk)
44
- chain = self.class.get_chain_parameter(base_job)
45
- if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
46
- new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
47
- elsif args.count > 0 || kwargs.count > 0
48
- raise "Unexpected number of arguments"
49
- end
50
- new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
51
- chain.insert(position, *new_jobs)
53
+ def insert(new_jobs, *args, **kwargs, &blk)
54
+ if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
55
+ job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
56
+ new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
57
+ kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
58
+ else
59
+ invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
60
+ raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
61
+ raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
62
+ raise "Unexpected number of arguments" if args.length > 0
52
63
  end
53
64
 
54
- def insert(new_jobs, *args, **kwargs, &blk)
55
- if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
56
- job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
57
- new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
58
- kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
59
- else
60
- invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
61
- raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
62
- raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
63
- raise "Unexpected number of arguments" if args.length > 0
64
- end
65
+ new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
65
66
 
66
- new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
67
+ if !kwargs.present?
68
+ insert_at(-1, new_jobs)
69
+ else
70
+ placement = kwargs.keys[0]
71
+ relative_to = kwargs.values[0]
67
72
 
68
- if !kwargs.present?
69
- insert_at(-1, new_jobs)
70
- else
71
- placement = kwargs.keys[0]
72
- relative_to = kwargs.values[0]
73
+ matching_jobs = find_matching_jobs(relative_to).to_a
74
+ raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
75
+ raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
73
76
 
74
- matching_jobs = find_matching_jobs(relative_to).to_a
75
- raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
76
- raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
77
+ relative_job, parent_job, sub_index = matching_jobs[0]
78
+ needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
77
79
 
78
- relative_job, parent_job, sub_index = matching_jobs[0]
79
- needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
80
+ chain = self.class.get_chain_parameter(parent_job)
80
81
 
82
+ if parent_job[:job] != needed_parent_type
83
+ old_job = chain[sub_index]
84
+ parent_job = chain[sub_index] = {
85
+ job: needed_parent_type,
86
+ parameters: [],
87
+ }
88
+ sub_index = 0
81
89
  chain = self.class.get_chain_parameter(parent_job)
90
+ chain << old_job
91
+ end
82
92
 
83
- if parent_job[:job] != needed_parent_type
84
- old_job = chain[sub_index]
85
- parent_job = chain[sub_index] = {
86
- job: needed_parent_type,
87
- parameters: [],
88
- }
89
- sub_index = 0
90
- chain = self.class.get_chain_parameter(parent_job)
91
- chain << old_job
92
- end
93
-
94
- if placement == :with
95
- chain.insert(-1, *new_jobs)
96
- else
97
- sub_index += 1 if placement == :after
98
- chain.insert(sub_index, *new_jobs)
99
- end
93
+ if placement == :with
94
+ chain.insert(-1, *new_jobs)
95
+ else
96
+ sub_index += 1 if placement == :after
97
+ chain.insert(sub_index, *new_jobs)
100
98
  end
101
99
  end
100
+ end
102
101
 
103
- def empty?
104
- self.class.get_chain_parameter(self).empty?
105
- end
102
+ def empty?
103
+ self.class.get_chain_parameter(self).empty?
104
+ end
106
105
 
107
- def get_sub_chain(sub_type)
108
- matching_jobs = find_matching_jobs(sub_type).to_a
109
- raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
110
- return nil if matching_jobs.count == 0
106
+ def get_sub_chain(sub_type)
107
+ matching_jobs = find_matching_jobs(sub_type).to_a
108
+ raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
109
+ return nil if matching_jobs.count == 0
111
110
 
112
- job = matching_jobs[0][0]
113
- job = self.class.new(job) unless job.is_a?(ChainBuilder)
114
- job
115
- end
111
+ job = matching_jobs[0][0]
112
+ job = self.class.new(job) unless job.is_a?(ChainBuilder)
113
+ job
114
+ end
116
115
 
117
- def normalize!(job_def = self.base_job)
118
- if job_def.is_a?(ChainBuilder)
119
- job_def.normalize!
120
- else
121
- job_def[:job] = job_def[:job].to_s
122
- if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
123
- chain.map! { |sub_job| normalize!(sub_job) }
124
- end
125
- job_def
116
+ def normalize!(job_def = self.base_job)
117
+ if job_def.is_a?(ChainBuilder)
118
+ job_def.normalize!
119
+ else
120
+ job_def[:job] = job_def[:job].to_s
121
+ if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
122
+ chain.map! { |sub_job| normalize!(sub_job) }
126
123
  end
124
+ job_def
127
125
  end
126
+ end
128
127
 
129
- def apply_block(&blk)
130
- return unless blk.present?
131
- instance_exec(&blk)
132
- end
128
+ def apply_block(&blk)
129
+ return unless blk.present?
130
+ instance_exec(&blk)
131
+ end
133
132
 
134
- private
133
+ private
135
134
 
136
- def build_job_hash(job, args: [], kwargs: {}, &blk)
137
- hsh = {
138
- job: job,
139
- args: args,
140
- kwargs: kwargs,
141
- }
142
- self.class.new(hsh).apply_block(&blk) if blk.present?
143
- hsh
144
- end
135
+ def build_job_hash(job, args: [], kwargs: {}, &blk)
136
+ hsh = {
137
+ job: job,
138
+ args: args,
139
+ kwargs: kwargs,
140
+ }
141
+ self.class.new(hsh).apply_block(&blk) if blk.present?
142
+ hsh
143
+ end
145
144
 
146
- def find_matching_jobs(search_job, parent_job = self.base_job)
147
- return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
145
+ def find_matching_jobs(search_job, parent_job = self.base_job)
146
+ return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
148
147
 
149
- sub_jobs = self.class.get_chain_parameter(parent_job)
150
- sub_jobs.each_with_index do |sub_job, i|
151
- if sub_job[:job].to_s == search_job.to_s
152
- yield [sub_job, parent_job, i]
153
- elsif self.class._job_type_definitions[sub_job[:job].to_s]
154
- find_matching_jobs(search_job, sub_job) { |item| yield item }
155
- end
148
+ sub_jobs = self.class.get_chain_parameter(parent_job)
149
+ sub_jobs.each_with_index do |sub_job, i|
150
+ if sub_job[:job].to_s == search_job.to_s
151
+ yield [sub_job, parent_job, i]
152
+ elsif self.class._job_type_definitions[sub_job[:job].to_s]
153
+ find_matching_jobs(search_job, sub_job) { |item| yield item }
156
154
  end
157
155
  end
156
+ end
158
157
 
159
- def find_parent_job(job_def)
160
- iterate_job_tree do |job, path|
161
- return path[-1] if job == job_def
162
- end
163
- nil
158
+ def find_parent_job(job_def)
159
+ iterate_job_tree do |job, path|
160
+ return path[-1] if job == job_def
164
161
  end
162
+ nil
163
+ end
165
164
 
166
- def iterate_job_tree(root: self.base_job, path: [], &blk)
167
- blk.call(root, path)
165
+ def iterate_job_tree(root: self.base_job, path: [], &blk)
166
+ blk.call(root, path)
168
167
 
169
- if self.class._job_type_definitions[root[:job]]
170
- sub_jobs = self.class.get_chain_parameter(root)
171
- sub_jobs.each_with_index do |sub_job, i|
172
- iterate_job_tree(root: sub_job, path: [*path, root], &blk)
173
- end
168
+ if self.class._job_type_definitions[root[:job]]
169
+ sub_jobs = self.class.get_chain_parameter(root)
170
+ sub_jobs.each_with_index do |sub_job, i|
171
+ iterate_job_tree(root: sub_job, path: [*path, root], &blk)
174
172
  end
175
173
  end
174
+ end
176
175
 
177
- class << self
178
- # Support builder syntaxt/DSL
179
- # Chain.build(ConcurrentBatchJob) do
180
- # insert(SomeJob, arg1, kwarg: 1)
181
- # insert(SerialBatchJob) do
182
- # insert(SomeJob, arg1, kwarg: 1)
183
- # end
184
- # end
185
- def build(job, *args, **kwargs, &blk)
186
- new(job).tap do |ch|
187
- ch.base_job[:args] = args
188
- ch.base_job[:kwargs] = kwargs
189
- ch.apply_block(&blk)
190
- end
176
+ class << self
177
+ # Support builder syntaxt/DSL
178
+ # Chain.build(ConcurrentBatchJob) do
179
+ # insert(SomeJob, arg1, kwarg: 1)
180
+ # insert(SerialBatchJob) do
181
+ # insert(SomeJob, arg1, kwarg: 1)
182
+ # end
183
+ # end
184
+ def build(job, *args, **kwargs, &blk)
185
+ new(job).tap do |ch|
186
+ ch.base_job[:args] = args
187
+ ch.base_job[:kwargs] = kwargs
188
+ ch.apply_block(&blk)
191
189
  end
190
+ end
191
+
192
+ def _job_type_definitions
193
+ @job_type_definitions ||= {}
194
+ end
195
+
196
+ def register_chain_job(job_class, chain_parameter, **options)
197
+ _job_type_definitions[job_class.to_s] = {
198
+ **options,
199
+ chain_parameter: chain_parameter,
200
+ }
201
+ end
192
202
 
193
- def _job_type_definitions
194
- @job_type_definitions ||= {}
203
+ def get_chain_parameter(job_def, raise_error: true)
204
+ unless _job_type_definitions[job_def[:job].to_s].present?
205
+ raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
206
+ return nil
195
207
  end
196
208
 
197
- def register_chain_job(job_class, chain_parameter, **options)
198
- _job_type_definitions[job_class.to_s] = {
199
- **options,
200
- chain_parameter: chain_parameter,
201
- }
209
+ key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
210
+ if key.is_a?(Numeric)
211
+ job_def[:args][key] ||= []
212
+ else
213
+ job_def[:kwargs][key] ||= []
202
214
  end
215
+ end
203
216
 
204
- def get_chain_parameter(job_def, raise_error: true)
205
- unless _job_type_definitions[job_def[:job].to_s].present?
206
- raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
207
- return nil
208
- end
209
-
210
- key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
211
- if key.is_a?(Numeric)
212
- job_def[:args][key] ||= []
213
- else
214
- job_def[:kwargs][key] ||= []
215
- end
217
+ # TODO: Add a Chain progress web View
218
+ # Augment Batch tree-view with Chain data
219
+ # > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
220
+ # > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
221
+ # > Chain-jobs will be supplied chain_id and chain_step_id metadata
222
+ # > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
223
+ # > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
224
+ # > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
225
+ def enqueue_job(job_def)
226
+ job_class = job_def[:job].constantize
227
+ job_args = job_def[:args] || job_def[:parameters] || []
228
+ job_kwargs = job_def[:kwargs] || {}
229
+
230
+ # Legacy Support
231
+ if job_def[:options]
232
+ job_args << {} unless job_args[-1].is_a?(Hash)
233
+ job_args[-1].merge!(job_def[:options])
216
234
  end
217
235
 
218
- # TODO: Add a Chain progress web View
219
- # Augment Batch tree-view with Chain data
220
- # > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
221
- # > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
222
- # > Chain-jobs will be supplied chain_id and chain_step_id metadata
223
- # > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
224
- # > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
225
- # > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
226
- def enqueue_job(job_def)
227
- job_class = job_def[:job].constantize
228
- job_args = job_def[:args] || job_def[:parameters] || []
229
- job_kwargs = job_def[:kwargs] || {}
230
-
231
- # Legacy Support
232
- if job_def[:options]
233
- job_args << {} unless job_args[-1].is_a?(Hash)
234
- job_args[-1].merge!(job_def[:options])
235
- end
236
-
237
- if job_class.respond_to? :perform_async
238
- job_class.perform_async(*job_args, **job_kwargs)
239
- else
240
- job_class.perform_later(*job_args, **job_kwargs)
241
- end
236
+ if job_class.respond_to? :perform_async
237
+ job_class.perform_async(*job_args, **job_kwargs)
238
+ else
239
+ job_class.perform_later(*job_args, **job_kwargs)
242
240
  end
243
241
  end
244
242
  end
245
-
246
- ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
247
- ChainBuilder.register_chain_job(SerialBatchJob, 0)
248
243
  end
244
+
245
+ ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
246
+ ChainBuilder.register_chain_job(SerialBatchJob, 0)
249
247
  end
@@ -1,6 +1,6 @@
1
1
 
2
- module CanvasSync
3
- module JobBatches
2
+ module CanvasSync::JobBatches
3
+ module Compat
4
4
  module ActiveJob
5
5
  module BatchAwareJob
6
6
  extend ActiveSupport::Concern
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module CanvasSync::JobBatches::Sidekiq
3
+ module CanvasSync::JobBatches::Compat::Sidekiq
4
4
  module Web
5
5
  module Helpers
6
6
  VIEW_PATH = File.expand_path("../web/views", __dir__)
@@ -7,7 +7,7 @@ end
7
7
 
8
8
  require_relative "web/helpers"
9
9
 
10
- module CanvasSync::JobBatches::Sidekiq
10
+ module CanvasSync::JobBatches::Compat::Sidekiq
11
11
  module Web
12
12
  DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
13
13
  Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
@@ -204,14 +204,14 @@ end
204
204
 
205
205
  if defined?(::Sidekiq::Web)
206
206
  rules = []
207
- rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Sidekiq::Web::DEV_MODE
207
+ rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Compat::Sidekiq::Web::DEV_MODE
208
208
 
209
209
  ::Sidekiq::Web.use Rack::Static, urls: ["/batches_assets"],
210
210
  root: File.expand_path("#{File.dirname(__FILE__)}/web"),
211
211
  cascade: true,
212
212
  header_rules: rules
213
213
 
214
- ::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
214
+ ::Sidekiq::Web.register CanvasSync::JobBatches::Compat::Sidekiq::Web
215
215
  ::Sidekiq::Web.tabs["Batches"] = "batches"
216
216
  ::Sidekiq::Web.tabs["Pools"] = "pools"
217
217
  ::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
@@ -3,8 +3,8 @@ begin
3
3
  rescue LoadError
4
4
  end
5
5
 
6
- module CanvasSync
7
- module JobBatches
6
+ module CanvasSync::JobBatches
7
+ module Compat
8
8
  module Sidekiq
9
9
  module WorkerExtension
10
10
  def bid
@@ -41,6 +41,8 @@ module CanvasSync
41
41
  end
42
42
 
43
43
  class ClientMiddleware
44
+ include ::Sidekiq::ClientMiddleware if defined? ::Sidekiq::ClientMiddleware
45
+
44
46
  def call(_worker, msg, _queue, _redis_pool = nil)
45
47
  if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
46
48
  batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
@@ -49,12 +51,14 @@ module CanvasSync
49
51
  end
50
52
 
51
53
  def should_handle_batch?(msg)
52
- return false if JobBatches::Sidekiq.is_activejob_job?(msg)
54
+ return false if CanvasSync::JobBatches::Compat::Sidekiq.is_activejob_job?(msg)
53
55
  true
54
56
  end
55
57
  end
56
58
 
57
59
  class ServerMiddleware
60
+ include ::Sidekiq::ServerMiddleware if defined? ::Sidekiq::ServerMiddleware
61
+
58
62
  def call(_worker, msg, _queue)
59
63
  if (bid = msg['bid'])
60
64
  prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
@@ -78,7 +82,7 @@ module CanvasSync
78
82
  def self.is_activejob_job?(msg)
79
83
  return false unless defined?(::ActiveJob)
80
84
 
81
- msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < JobBatches::ActiveJob::BatchAwareJob
85
+ msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < CanvasSync::JobBatches::Compat::ActiveJob::BatchAwareJob
82
86
  end
83
87
 
84
88
  def self.switch_tenant(job)
@@ -91,42 +95,51 @@ module CanvasSync
91
95
  end
92
96
  end
93
97
 
98
+ def self.sidekiq_middleware(placement, &blk)
99
+ install_middleware = ->(config) do
100
+ config.send("#{placement}_middleware") do |chain|
101
+ blk.call(chain)
102
+ end
103
+ end
104
+
105
+ ::Sidekiq.configure_client(&install_middleware) if placement == :client
106
+ ::Sidekiq.configure_server(&install_middleware)
107
+ end
108
+
94
109
  def self.configure
95
- if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != JobBatches::Batch
110
+ return if @already_configured
111
+ @already_configured = true
112
+
113
+ if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != CanvasSync::JobBatches::Batch
96
114
  print "WARNING: Detected Sidekiq Pro or sidekiq-batch. CanvasSync JobBatches may not be fully compatible!"
97
115
  end
98
116
 
99
- ::Sidekiq.configure_client do |config|
100
- config.client_middleware do |chain|
101
- chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
102
- chain.add JobBatches::Sidekiq::ClientMiddleware
103
- end
117
+ sidekiq_middleware(:client) do |chain|
118
+ chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
119
+ chain.add CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware
104
120
  end
105
- ::Sidekiq.configure_server do |config|
106
- config.client_middleware do |chain|
107
- chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
108
- chain.add JobBatches::Sidekiq::ClientMiddleware
109
- end
110
121
 
111
- config.server_middleware do |chain|
112
- chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
113
- chain.add JobBatches::Sidekiq::ServerMiddleware
114
- end
122
+ sidekiq_middleware(:server) do |chain|
123
+ chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
124
+ chain.add CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware
125
+ end
115
126
 
127
+ ::Sidekiq.configure_server do |config|
116
128
  config.death_handlers << ->(job, ex) do
117
129
  switch_tenant(job) do
118
130
  if is_activejob_job?(job)
119
- JobBatches::ActiveJob.handle_job_death(job["args"][0], ex)
131
+ CanvasSync::JobBatches::Compat::ActiveJob.handle_job_death(job["args"][0], ex)
120
132
  elsif job['bid'].present?
121
133
  ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
122
134
  end
123
135
  end
124
136
  end
125
137
  end
138
+
126
139
  ::Sidekiq.const_set(:Batch, CanvasSync::JobBatches::Batch)
127
140
  # This alias helps apartment-sidekiq set itself up correctly
128
- ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
129
- ::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
141
+ ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware)
142
+ ::Sidekiq::Worker.send(:include, CanvasSync::JobBatches::Compat::Sidekiq::WorkerExtension)
130
143
  Batch::Callback.worker_class = SidekiqCallbackWorker
131
144
  end
132
145
  end
@@ -0,0 +1,20 @@
1
+
2
+ module CanvasSync::JobBatches
3
+ module Compat
4
+ def self.load_compat(name)
5
+ name = name.to_s
6
+ begin
7
+ require name
8
+ rescue LoadError
9
+ end
10
+
11
+ if name.classify.safe_constantize
12
+ require_relative "./compat/#{name}"
13
+ "CanvasSync::JobBatches::Compat::#{name.classify}".constantize.configure
14
+ end
15
+ end
16
+
17
+ load_compat(:active_job)
18
+ load_compat(:sidekiq)
19
+ end
20
+ end