canvas_sync 0.21.1 → 0.22.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/lib/canvas_sync/concerns/auto_relations.rb +11 -0
  3. data/lib/canvas_sync/config.rb +3 -5
  4. data/lib/canvas_sync/generators/templates/models/rubric.rb +2 -1
  5. data/lib/canvas_sync/job_batches/batch.rb +432 -402
  6. data/lib/canvas_sync/job_batches/callback.rb +100 -114
  7. data/lib/canvas_sync/job_batches/chain_builder.rb +194 -196
  8. data/lib/canvas_sync/job_batches/{active_job.rb → compat/active_job.rb} +2 -2
  9. data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/helpers.rb +1 -1
  10. data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web.rb +3 -3
  11. data/lib/canvas_sync/job_batches/{sidekiq.rb → compat/sidekiq.rb} +35 -22
  12. data/lib/canvas_sync/job_batches/compat.rb +20 -0
  13. data/lib/canvas_sync/job_batches/context_hash.rb +124 -126
  14. data/lib/canvas_sync/job_batches/jobs/base_job.rb +2 -4
  15. data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +14 -16
  16. data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +125 -127
  17. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +14 -16
  18. data/lib/canvas_sync/job_batches/pool.rb +193 -195
  19. data/lib/canvas_sync/job_batches/redis_model.rb +50 -52
  20. data/lib/canvas_sync/job_batches/redis_script.rb +129 -131
  21. data/lib/canvas_sync/job_batches/status.rb +85 -87
  22. data/lib/canvas_sync/job_uniqueness/compat/active_job.rb +75 -0
  23. data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb +135 -0
  24. data/lib/canvas_sync/job_uniqueness/compat.rb +20 -0
  25. data/lib/canvas_sync/job_uniqueness/configuration.rb +25 -0
  26. data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb +47 -0
  27. data/lib/canvas_sync/job_uniqueness/lock_context.rb +171 -0
  28. data/lib/canvas_sync/job_uniqueness/locksmith.rb +92 -0
  29. data/lib/canvas_sync/job_uniqueness/on_conflict/base.rb +32 -0
  30. data/lib/canvas_sync/job_uniqueness/on_conflict/log.rb +13 -0
  31. data/lib/canvas_sync/job_uniqueness/on_conflict/null_strategy.rb +9 -0
  32. data/lib/canvas_sync/job_uniqueness/on_conflict/raise.rb +11 -0
  33. data/lib/canvas_sync/job_uniqueness/on_conflict/reject.rb +21 -0
  34. data/lib/canvas_sync/job_uniqueness/on_conflict/reschedule.rb +20 -0
  35. data/lib/canvas_sync/job_uniqueness/on_conflict.rb +41 -0
  36. data/lib/canvas_sync/job_uniqueness/strategy/base.rb +104 -0
  37. data/lib/canvas_sync/job_uniqueness/strategy/until_and_while_executing.rb +35 -0
  38. data/lib/canvas_sync/job_uniqueness/strategy/until_executed.rb +20 -0
  39. data/lib/canvas_sync/job_uniqueness/strategy/until_executing.rb +20 -0
  40. data/lib/canvas_sync/job_uniqueness/strategy/until_expired.rb +16 -0
  41. data/lib/canvas_sync/job_uniqueness/strategy/while_executing.rb +26 -0
  42. data/lib/canvas_sync/job_uniqueness/strategy.rb +27 -0
  43. data/lib/canvas_sync/job_uniqueness/unique_job_common.rb +79 -0
  44. data/lib/canvas_sync/misc_helper.rb +1 -1
  45. data/lib/canvas_sync/version.rb +1 -1
  46. data/lib/canvas_sync.rb +4 -3
  47. data/spec/dummy/app/models/rubric.rb +2 -1
  48. data/spec/dummy/config/environments/test.rb +1 -1
  49. data/spec/job_batching/batch_spec.rb +49 -7
  50. data/spec/job_batching/{active_job_spec.rb → compat/active_job_spec.rb} +2 -2
  51. data/spec/job_batching/{sidekiq_spec.rb → compat/sidekiq_spec.rb} +14 -12
  52. data/spec/job_batching/flow_spec.rb +1 -1
  53. data/spec/job_batching/integration_helper.rb +1 -1
  54. data/spec/job_batching/status_spec.rb +2 -2
  55. data/spec/job_uniqueness/compat/active_job_spec.rb +49 -0
  56. data/spec/job_uniqueness/compat/sidekiq_spec.rb +68 -0
  57. data/spec/job_uniqueness/lock_context_spec.rb +95 -0
  58. data/spec/job_uniqueness/on_conflict/log_spec.rb +11 -0
  59. data/spec/job_uniqueness/on_conflict/raise_spec.rb +10 -0
  60. data/spec/job_uniqueness/on_conflict/reschedule_spec.rb +24 -0
  61. data/spec/job_uniqueness/on_conflict_spec.rb +16 -0
  62. data/spec/job_uniqueness/spec_helper.rb +14 -0
  63. data/spec/job_uniqueness/strategy/base_spec.rb +100 -0
  64. data/spec/job_uniqueness/strategy/until_and_while_executing_spec.rb +48 -0
  65. data/spec/job_uniqueness/strategy/until_executed_spec.rb +23 -0
  66. data/spec/job_uniqueness/strategy/until_executing_spec.rb +23 -0
  67. data/spec/job_uniqueness/strategy/until_expired_spec.rb +23 -0
  68. data/spec/job_uniqueness/strategy/while_executing_spec.rb +33 -0
  69. data/spec/job_uniqueness/support/lock_strategy.rb +28 -0
  70. data/spec/job_uniqueness/support/on_conflict.rb +24 -0
  71. data/spec/job_uniqueness/support/test_worker.rb +19 -0
  72. data/spec/job_uniqueness/unique_job_common_spec.rb +45 -0
  73. data/spec/spec_helper.rb +1 -1
  74. metadata +278 -204
  75. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/css/styles.less +0 -0
  76. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/batch_tree.js +0 -0
  77. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/util.js +0 -0
  78. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batch_tree.erb +0 -0
  79. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batches_table.erb +0 -0
  80. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_common.erb +0 -0
  81. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_jobs_table.erb +0 -0
  82. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_pagination.erb +0 -0
  83. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batch.erb +0 -0
  84. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batches.erb +0 -0
  85. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pool.erb +0 -0
  86. /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pools.erb +0 -0
@@ -1,249 +1,247 @@
1
- module CanvasSync
2
- module JobBatches
3
- class ChainBuilder
4
- VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
5
-
6
- attr_reader :base_job
7
-
8
- def initialize(base_type = SerialBatchJob)
9
- if base_type.is_a?(Hash)
10
- @base_job = base_type
11
- @base_job[:args] ||= @base_job[:parameters] || []
12
- @base_job[:kwargs] ||= {}
13
- else
14
- @base_job = build_job_hash(base_type)
15
- end
16
-
17
- self.class.get_chain_parameter(base_job)
1
+ module CanvasSync::JobBatches
2
+ class ChainBuilder
3
+ VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
4
+
5
+ attr_reader :base_job
6
+
7
+ def initialize(base_type = SerialBatchJob)
8
+ if base_type.is_a?(Hash)
9
+ @base_job = base_type
10
+ @base_job[:args] ||= @base_job[:parameters] || []
11
+ @base_job[:kwargs] ||= {}
12
+ else
13
+ @base_job = build_job_hash(base_type)
18
14
  end
19
15
 
20
- def process!
21
- normalize!
22
- self.class.enqueue_job(base_job)
23
- end
16
+ self.class.get_chain_parameter(base_job)
17
+ end
24
18
 
25
- def [](key)
26
- if key.is_a?(Class)
27
- get_sub_chain(key)
28
- else
29
- # Legacy Support
30
- key = :args if key == :parameters
19
+ def process!
20
+ normalize!
21
+ self.class.enqueue_job(base_job)
22
+ end
31
23
 
32
- @base_job[key]
33
- end
24
+ def [](key)
25
+ if key.is_a?(Class)
26
+ get_sub_chain(key)
27
+ else
28
+ # Legacy Support
29
+ key = :args if key == :parameters
30
+
31
+ @base_job[key]
34
32
  end
33
+ end
35
34
 
36
- def args; return self[:args]; end
37
- def kwargs; return self[:kwargs]; end
35
+ def args; return self[:args]; end
36
+ def kwargs; return self[:kwargs]; end
38
37
 
39
- def <<(new_job)
40
- insert_at(-1, new_job)
38
+ def <<(new_job)
39
+ insert_at(-1, new_job)
40
+ end
41
+
42
+ def insert_at(position, new_jobs, *args, **kwargs, &blk)
43
+ chain = self.class.get_chain_parameter(base_job)
44
+ if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
45
+ new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
46
+ elsif args.count > 0 || kwargs.count > 0
47
+ raise "Unexpected number of arguments"
41
48
  end
49
+ new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
50
+ chain.insert(position, *new_jobs)
51
+ end
42
52
 
43
- def insert_at(position, new_jobs, *args, **kwargs, &blk)
44
- chain = self.class.get_chain_parameter(base_job)
45
- if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
46
- new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
47
- elsif args.count > 0 || kwargs.count > 0
48
- raise "Unexpected number of arguments"
49
- end
50
- new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
51
- chain.insert(position, *new_jobs)
53
+ def insert(new_jobs, *args, **kwargs, &blk)
54
+ if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
55
+ job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
56
+ new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
57
+ kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
58
+ else
59
+ invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
60
+ raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
61
+ raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
62
+ raise "Unexpected number of arguments" if args.length > 0
52
63
  end
53
64
 
54
- def insert(new_jobs, *args, **kwargs, &blk)
55
- if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
56
- job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
57
- new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
58
- kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
59
- else
60
- invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
61
- raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
62
- raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
63
- raise "Unexpected number of arguments" if args.length > 0
64
- end
65
+ new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
65
66
 
66
- new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
67
+ if !kwargs.present?
68
+ insert_at(-1, new_jobs)
69
+ else
70
+ placement = kwargs.keys[0]
71
+ relative_to = kwargs.values[0]
67
72
 
68
- if !kwargs.present?
69
- insert_at(-1, new_jobs)
70
- else
71
- placement = kwargs.keys[0]
72
- relative_to = kwargs.values[0]
73
+ matching_jobs = find_matching_jobs(relative_to).to_a
74
+ raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
75
+ raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
73
76
 
74
- matching_jobs = find_matching_jobs(relative_to).to_a
75
- raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
76
- raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
77
+ relative_job, parent_job, sub_index = matching_jobs[0]
78
+ needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
77
79
 
78
- relative_job, parent_job, sub_index = matching_jobs[0]
79
- needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
80
+ chain = self.class.get_chain_parameter(parent_job)
80
81
 
82
+ if parent_job[:job] != needed_parent_type
83
+ old_job = chain[sub_index]
84
+ parent_job = chain[sub_index] = {
85
+ job: needed_parent_type,
86
+ parameters: [],
87
+ }
88
+ sub_index = 0
81
89
  chain = self.class.get_chain_parameter(parent_job)
90
+ chain << old_job
91
+ end
82
92
 
83
- if parent_job[:job] != needed_parent_type
84
- old_job = chain[sub_index]
85
- parent_job = chain[sub_index] = {
86
- job: needed_parent_type,
87
- parameters: [],
88
- }
89
- sub_index = 0
90
- chain = self.class.get_chain_parameter(parent_job)
91
- chain << old_job
92
- end
93
-
94
- if placement == :with
95
- chain.insert(-1, *new_jobs)
96
- else
97
- sub_index += 1 if placement == :after
98
- chain.insert(sub_index, *new_jobs)
99
- end
93
+ if placement == :with
94
+ chain.insert(-1, *new_jobs)
95
+ else
96
+ sub_index += 1 if placement == :after
97
+ chain.insert(sub_index, *new_jobs)
100
98
  end
101
99
  end
100
+ end
102
101
 
103
- def empty?
104
- self.class.get_chain_parameter(self).empty?
105
- end
102
+ def empty?
103
+ self.class.get_chain_parameter(self).empty?
104
+ end
106
105
 
107
- def get_sub_chain(sub_type)
108
- matching_jobs = find_matching_jobs(sub_type).to_a
109
- raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
110
- return nil if matching_jobs.count == 0
106
+ def get_sub_chain(sub_type)
107
+ matching_jobs = find_matching_jobs(sub_type).to_a
108
+ raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
109
+ return nil if matching_jobs.count == 0
111
110
 
112
- job = matching_jobs[0][0]
113
- job = self.class.new(job) unless job.is_a?(ChainBuilder)
114
- job
115
- end
111
+ job = matching_jobs[0][0]
112
+ job = self.class.new(job) unless job.is_a?(ChainBuilder)
113
+ job
114
+ end
116
115
 
117
- def normalize!(job_def = self.base_job)
118
- if job_def.is_a?(ChainBuilder)
119
- job_def.normalize!
120
- else
121
- job_def[:job] = job_def[:job].to_s
122
- if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
123
- chain.map! { |sub_job| normalize!(sub_job) }
124
- end
125
- job_def
116
+ def normalize!(job_def = self.base_job)
117
+ if job_def.is_a?(ChainBuilder)
118
+ job_def.normalize!
119
+ else
120
+ job_def[:job] = job_def[:job].to_s
121
+ if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
122
+ chain.map! { |sub_job| normalize!(sub_job) }
126
123
  end
124
+ job_def
127
125
  end
126
+ end
128
127
 
129
- def apply_block(&blk)
130
- return unless blk.present?
131
- instance_exec(&blk)
132
- end
128
+ def apply_block(&blk)
129
+ return unless blk.present?
130
+ instance_exec(&blk)
131
+ end
133
132
 
134
- private
133
+ private
135
134
 
136
- def build_job_hash(job, args: [], kwargs: {}, &blk)
137
- hsh = {
138
- job: job,
139
- args: args,
140
- kwargs: kwargs,
141
- }
142
- self.class.new(hsh).apply_block(&blk) if blk.present?
143
- hsh
144
- end
135
+ def build_job_hash(job, args: [], kwargs: {}, &blk)
136
+ hsh = {
137
+ job: job,
138
+ args: args,
139
+ kwargs: kwargs,
140
+ }
141
+ self.class.new(hsh).apply_block(&blk) if blk.present?
142
+ hsh
143
+ end
145
144
 
146
- def find_matching_jobs(search_job, parent_job = self.base_job)
147
- return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
145
+ def find_matching_jobs(search_job, parent_job = self.base_job)
146
+ return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
148
147
 
149
- sub_jobs = self.class.get_chain_parameter(parent_job)
150
- sub_jobs.each_with_index do |sub_job, i|
151
- if sub_job[:job].to_s == search_job.to_s
152
- yield [sub_job, parent_job, i]
153
- elsif self.class._job_type_definitions[sub_job[:job].to_s]
154
- find_matching_jobs(search_job, sub_job) { |item| yield item }
155
- end
148
+ sub_jobs = self.class.get_chain_parameter(parent_job)
149
+ sub_jobs.each_with_index do |sub_job, i|
150
+ if sub_job[:job].to_s == search_job.to_s
151
+ yield [sub_job, parent_job, i]
152
+ elsif self.class._job_type_definitions[sub_job[:job].to_s]
153
+ find_matching_jobs(search_job, sub_job) { |item| yield item }
156
154
  end
157
155
  end
156
+ end
158
157
 
159
- def find_parent_job(job_def)
160
- iterate_job_tree do |job, path|
161
- return path[-1] if job == job_def
162
- end
163
- nil
158
+ def find_parent_job(job_def)
159
+ iterate_job_tree do |job, path|
160
+ return path[-1] if job == job_def
164
161
  end
162
+ nil
163
+ end
165
164
 
166
- def iterate_job_tree(root: self.base_job, path: [], &blk)
167
- blk.call(root, path)
165
+ def iterate_job_tree(root: self.base_job, path: [], &blk)
166
+ blk.call(root, path)
168
167
 
169
- if self.class._job_type_definitions[root[:job]]
170
- sub_jobs = self.class.get_chain_parameter(root)
171
- sub_jobs.each_with_index do |sub_job, i|
172
- iterate_job_tree(root: sub_job, path: [*path, root], &blk)
173
- end
168
+ if self.class._job_type_definitions[root[:job]]
169
+ sub_jobs = self.class.get_chain_parameter(root)
170
+ sub_jobs.each_with_index do |sub_job, i|
171
+ iterate_job_tree(root: sub_job, path: [*path, root], &blk)
174
172
  end
175
173
  end
174
+ end
176
175
 
177
- class << self
178
- # Support builder syntaxt/DSL
179
- # Chain.build(ConcurrentBatchJob) do
180
- # insert(SomeJob, arg1, kwarg: 1)
181
- # insert(SerialBatchJob) do
182
- # insert(SomeJob, arg1, kwarg: 1)
183
- # end
184
- # end
185
- def build(job, *args, **kwargs, &blk)
186
- new(job).tap do |ch|
187
- ch.base_job[:args] = args
188
- ch.base_job[:kwargs] = kwargs
189
- ch.apply_block(&blk)
190
- end
176
+ class << self
177
+ # Support builder syntaxt/DSL
178
+ # Chain.build(ConcurrentBatchJob) do
179
+ # insert(SomeJob, arg1, kwarg: 1)
180
+ # insert(SerialBatchJob) do
181
+ # insert(SomeJob, arg1, kwarg: 1)
182
+ # end
183
+ # end
184
+ def build(job, *args, **kwargs, &blk)
185
+ new(job).tap do |ch|
186
+ ch.base_job[:args] = args
187
+ ch.base_job[:kwargs] = kwargs
188
+ ch.apply_block(&blk)
191
189
  end
190
+ end
191
+
192
+ def _job_type_definitions
193
+ @job_type_definitions ||= {}
194
+ end
195
+
196
+ def register_chain_job(job_class, chain_parameter, **options)
197
+ _job_type_definitions[job_class.to_s] = {
198
+ **options,
199
+ chain_parameter: chain_parameter,
200
+ }
201
+ end
192
202
 
193
- def _job_type_definitions
194
- @job_type_definitions ||= {}
203
+ def get_chain_parameter(job_def, raise_error: true)
204
+ unless _job_type_definitions[job_def[:job].to_s].present?
205
+ raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
206
+ return nil
195
207
  end
196
208
 
197
- def register_chain_job(job_class, chain_parameter, **options)
198
- _job_type_definitions[job_class.to_s] = {
199
- **options,
200
- chain_parameter: chain_parameter,
201
- }
209
+ key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
210
+ if key.is_a?(Numeric)
211
+ job_def[:args][key] ||= []
212
+ else
213
+ job_def[:kwargs][key] ||= []
202
214
  end
215
+ end
203
216
 
204
- def get_chain_parameter(job_def, raise_error: true)
205
- unless _job_type_definitions[job_def[:job].to_s].present?
206
- raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
207
- return nil
208
- end
209
-
210
- key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
211
- if key.is_a?(Numeric)
212
- job_def[:args][key] ||= []
213
- else
214
- job_def[:kwargs][key] ||= []
215
- end
217
+ # TODO: Add a Chain progress web View
218
+ # Augment Batch tree-view with Chain data
219
+ # > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
220
+ # > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
221
+ # > Chain-jobs will be supplied chain_id and chain_step_id metadata
222
+ # > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
223
+ # > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
224
+ # > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
225
+ def enqueue_job(job_def)
226
+ job_class = job_def[:job].constantize
227
+ job_args = job_def[:args] || job_def[:parameters] || []
228
+ job_kwargs = job_def[:kwargs] || {}
229
+
230
+ # Legacy Support
231
+ if job_def[:options]
232
+ job_args << {} unless job_args[-1].is_a?(Hash)
233
+ job_args[-1].merge!(job_def[:options])
216
234
  end
217
235
 
218
- # TODO: Add a Chain progress web View
219
- # Augment Batch tree-view with Chain data
220
- # > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
221
- # > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
222
- # > Chain-jobs will be supplied chain_id and chain_step_id metadata
223
- # > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
224
- # > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
225
- # > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
226
- def enqueue_job(job_def)
227
- job_class = job_def[:job].constantize
228
- job_args = job_def[:args] || job_def[:parameters] || []
229
- job_kwargs = job_def[:kwargs] || {}
230
-
231
- # Legacy Support
232
- if job_def[:options]
233
- job_args << {} unless job_args[-1].is_a?(Hash)
234
- job_args[-1].merge!(job_def[:options])
235
- end
236
-
237
- if job_class.respond_to? :perform_async
238
- job_class.perform_async(*job_args, **job_kwargs)
239
- else
240
- job_class.perform_later(*job_args, **job_kwargs)
241
- end
236
+ if job_class.respond_to? :perform_async
237
+ job_class.perform_async(*job_args, **job_kwargs)
238
+ else
239
+ job_class.perform_later(*job_args, **job_kwargs)
242
240
  end
243
241
  end
244
242
  end
245
-
246
- ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
247
- ChainBuilder.register_chain_job(SerialBatchJob, 0)
248
243
  end
244
+
245
+ ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
246
+ ChainBuilder.register_chain_job(SerialBatchJob, 0)
249
247
  end
@@ -1,6 +1,6 @@
1
1
 
2
- module CanvasSync
3
- module JobBatches
2
+ module CanvasSync::JobBatches
3
+ module Compat
4
4
  module ActiveJob
5
5
  module BatchAwareJob
6
6
  extend ActiveSupport::Concern
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module CanvasSync::JobBatches::Sidekiq
3
+ module CanvasSync::JobBatches::Compat::Sidekiq
4
4
  module Web
5
5
  module Helpers
6
6
  VIEW_PATH = File.expand_path("../web/views", __dir__)
@@ -7,7 +7,7 @@ end
7
7
 
8
8
  require_relative "web/helpers"
9
9
 
10
- module CanvasSync::JobBatches::Sidekiq
10
+ module CanvasSync::JobBatches::Compat::Sidekiq
11
11
  module Web
12
12
  DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
13
13
  Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
@@ -204,14 +204,14 @@ end
204
204
 
205
205
  if defined?(::Sidekiq::Web)
206
206
  rules = []
207
- rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Sidekiq::Web::DEV_MODE
207
+ rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless CanvasSync::JobBatches::Compat::Sidekiq::Web::DEV_MODE
208
208
 
209
209
  ::Sidekiq::Web.use Rack::Static, urls: ["/batches_assets"],
210
210
  root: File.expand_path("#{File.dirname(__FILE__)}/web"),
211
211
  cascade: true,
212
212
  header_rules: rules
213
213
 
214
- ::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
214
+ ::Sidekiq::Web.register CanvasSync::JobBatches::Compat::Sidekiq::Web
215
215
  ::Sidekiq::Web.tabs["Batches"] = "batches"
216
216
  ::Sidekiq::Web.tabs["Pools"] = "pools"
217
217
  ::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
@@ -3,8 +3,8 @@ begin
3
3
  rescue LoadError
4
4
  end
5
5
 
6
- module CanvasSync
7
- module JobBatches
6
+ module CanvasSync::JobBatches
7
+ module Compat
8
8
  module Sidekiq
9
9
  module WorkerExtension
10
10
  def bid
@@ -41,6 +41,8 @@ module CanvasSync
41
41
  end
42
42
 
43
43
  class ClientMiddleware
44
+ include ::Sidekiq::ClientMiddleware if defined? ::Sidekiq::ClientMiddleware
45
+
44
46
  def call(_worker, msg, _queue, _redis_pool = nil)
45
47
  if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY]) && should_handle_batch?(msg)
46
48
  batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
@@ -49,12 +51,14 @@ module CanvasSync
49
51
  end
50
52
 
51
53
  def should_handle_batch?(msg)
52
- return false if JobBatches::Sidekiq.is_activejob_job?(msg)
54
+ return false if CanvasSync::JobBatches::Compat::Sidekiq.is_activejob_job?(msg)
53
55
  true
54
56
  end
55
57
  end
56
58
 
57
59
  class ServerMiddleware
60
+ include ::Sidekiq::ServerMiddleware if defined? ::Sidekiq::ServerMiddleware
61
+
58
62
  def call(_worker, msg, _queue)
59
63
  if (bid = msg['bid'])
60
64
  prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
@@ -78,7 +82,7 @@ module CanvasSync
78
82
  def self.is_activejob_job?(msg)
79
83
  return false unless defined?(::ActiveJob)
80
84
 
81
- msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < JobBatches::ActiveJob::BatchAwareJob
85
+ msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && (msg['wrapped'].to_s).constantize < CanvasSync::JobBatches::Compat::ActiveJob::BatchAwareJob
82
86
  end
83
87
 
84
88
  def self.switch_tenant(job)
@@ -91,42 +95,51 @@ module CanvasSync
91
95
  end
92
96
  end
93
97
 
98
+ def self.sidekiq_middleware(placement, &blk)
99
+ install_middleware = ->(config) do
100
+ config.send("#{placement}_middleware") do |chain|
101
+ blk.call(chain)
102
+ end
103
+ end
104
+
105
+ ::Sidekiq.configure_client(&install_middleware) if placement == :client
106
+ ::Sidekiq.configure_server(&install_middleware)
107
+ end
108
+
94
109
  def self.configure
95
- if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != JobBatches::Batch
110
+ return if @already_configured
111
+ @already_configured = true
112
+
113
+ if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != CanvasSync::JobBatches::Batch
96
114
  print "WARNING: Detected Sidekiq Pro or sidekiq-batch. CanvasSync JobBatches may not be fully compatible!"
97
115
  end
98
116
 
99
- ::Sidekiq.configure_client do |config|
100
- config.client_middleware do |chain|
101
- chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
102
- chain.add JobBatches::Sidekiq::ClientMiddleware
103
- end
117
+ sidekiq_middleware(:client) do |chain|
118
+ chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
119
+ chain.add CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware
104
120
  end
105
- ::Sidekiq.configure_server do |config|
106
- config.client_middleware do |chain|
107
- chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
108
- chain.add JobBatches::Sidekiq::ClientMiddleware
109
- end
110
121
 
111
- config.server_middleware do |chain|
112
- chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
113
- chain.add JobBatches::Sidekiq::ServerMiddleware
114
- end
122
+ sidekiq_middleware(:server) do |chain|
123
+ chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
124
+ chain.add CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware
125
+ end
115
126
 
127
+ ::Sidekiq.configure_server do |config|
116
128
  config.death_handlers << ->(job, ex) do
117
129
  switch_tenant(job) do
118
130
  if is_activejob_job?(job)
119
- JobBatches::ActiveJob.handle_job_death(job["args"][0], ex)
131
+ CanvasSync::JobBatches::Compat::ActiveJob.handle_job_death(job["args"][0], ex)
120
132
  elsif job['bid'].present?
121
133
  ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
122
134
  end
123
135
  end
124
136
  end
125
137
  end
138
+
126
139
  ::Sidekiq.const_set(:Batch, CanvasSync::JobBatches::Batch)
127
140
  # This alias helps apartment-sidekiq set itself up correctly
128
- ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
129
- ::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
141
+ ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware)
142
+ ::Sidekiq::Worker.send(:include, CanvasSync::JobBatches::Compat::Sidekiq::WorkerExtension)
130
143
  Batch::Callback.worker_class = SidekiqCallbackWorker
131
144
  end
132
145
  end
@@ -0,0 +1,20 @@
1
+
2
+ module CanvasSync::JobBatches
3
+ module Compat
4
+ def self.load_compat(name)
5
+ name = name.to_s
6
+ begin
7
+ require name
8
+ rescue LoadError
9
+ end
10
+
11
+ if name.classify.safe_constantize
12
+ require_relative "./compat/#{name}"
13
+ "CanvasSync::JobBatches::Compat::#{name.classify}".constantize.configure
14
+ end
15
+ end
16
+
17
+ load_compat(:active_job)
18
+ load_compat(:sidekiq)
19
+ end
20
+ end