canvas_sync 0.17.2 → 0.17.5.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +58 -0
  3. data/lib/canvas_sync/job_batches/batch.rb +103 -115
  4. data/lib/canvas_sync/job_batches/batch_aware_job.rb +5 -1
  5. data/lib/canvas_sync/job_batches/callback.rb +29 -34
  6. data/lib/canvas_sync/job_batches/context_hash.rb +13 -5
  7. data/lib/canvas_sync/job_batches/hincr_max.lua +5 -0
  8. data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +99 -0
  9. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +6 -65
  10. data/lib/canvas_sync/job_batches/pool.rb +213 -0
  11. data/lib/canvas_sync/job_batches/redis_model.rb +69 -0
  12. data/lib/canvas_sync/job_batches/redis_script.rb +163 -0
  13. data/lib/canvas_sync/job_batches/sidekiq.rb +31 -3
  14. data/lib/canvas_sync/job_batches/sidekiq/web.rb +114 -0
  15. data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb +41 -0
  16. data/lib/canvas_sync/job_batches/sidekiq/web/views/_batches_table.erb +42 -0
  17. data/lib/canvas_sync/job_batches/sidekiq/web/views/_pagination.erb +26 -0
  18. data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb +138 -0
  19. data/lib/canvas_sync/job_batches/sidekiq/web/views/batches.erb +23 -0
  20. data/lib/canvas_sync/job_batches/sidekiq/web/views/pool.erb +85 -0
  21. data/lib/canvas_sync/job_batches/sidekiq/web/views/pools.erb +47 -0
  22. data/lib/canvas_sync/job_batches/status.rb +9 -5
  23. data/lib/canvas_sync/version.rb +1 -1
  24. data/spec/dummy/log/test.log +144212 -0
  25. data/spec/job_batching/batch_aware_job_spec.rb +1 -0
  26. data/spec/job_batching/batch_spec.rb +72 -16
  27. data/spec/job_batching/callback_spec.rb +1 -1
  28. data/spec/job_batching/context_hash_spec.rb +54 -0
  29. data/spec/job_batching/flow_spec.rb +5 -11
  30. data/spec/job_batching/integration/fail_then_succeed.rb +42 -0
  31. data/spec/job_batching/integration_helper.rb +6 -4
  32. data/spec/job_batching/sidekiq_spec.rb +1 -0
  33. data/spec/job_batching/status_spec.rb +4 -20
  34. data/spec/spec_helper.rb +3 -7
  35. metadata +21 -18
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e7e920904fe92285bd67cbf48a88e8aa6c03761dbb543eb9ee477058f3086c2a
4
- data.tar.gz: d2221d37737fde81de62a0f9d6d304cf059c30cc78f3be814c01849365257680
3
+ metadata.gz: 5291764ad05bd9687d1e6b4533dfa883be3a394934aba61707e0e1fd77076c35
4
+ data.tar.gz: 191f1e112916749fbc81797c5af3826ebf65eac94d293584bab424cefd12506d
5
5
  SHA512:
6
- metadata.gz: 3a988e7678bc3f0e6e33325171f7665cc016cf30d07aa299c69268339587433e69d7c11036d797358c4f563d21956b74a48bb7bd5cc656d86ba01edde5d85a3f
7
- data.tar.gz: 74f9f9ba4a6a6a59e1a7dbb2dba351cee464d12d74ef526784fa63d81e319656163ccb52fbca1f71a7af664148f6712e77513bbf4346e432bc9394fede53ed3f
6
+ metadata.gz: eb42503d2afe8ec5a0b6f667c056fb038e3999580ca82c52ff0cac8d410e981f5f6a5c9be4bcee6ad9878d54b4a54cc15eafca0e5c202dadea0315bfe9e2b824
7
+ data.tar.gz: 4e912435b6110c8b552570112db9b4f3fce4fc34e851907e09326290e1988ca11333ee0c619ad2dab8b5d077cd275727bd603f8d1438962cea88937a1076d4e9
data/README.md CHANGED
@@ -246,6 +246,64 @@ A batch can be created using `Sidekiq::Batch` or `CanvasSync::JobBatching::Batch
246
246
 
247
247
  Also see `canvas_sync/jobs/begin_sync_chain_job`, `canvas_sync/Job_batches/jobs/serial_batch_job`, or `canvas_sync/Job_batches/jobs/concurrent_batch_job` for example usage.
248
248
 
249
+ Example:
250
+ ```ruby
251
+ batch = CanvasSync::JobBatches::Batch.new
252
+ batch.description = "Some Batch" # Optional, but can be useful for debugging
253
+
254
+ batch.on(:complete, "SomeClass.on_complete", kw_param: 1)
255
+ batch.on(:success, "SomeClass.on_success", some_param: 'foo')
256
+
257
+ # Add context to the batch. Can be accessed as batch_context on any jobs within the batch.
258
+ # Nested Batches will have their contexts merged
259
+ batch.context = {
260
+ some_value: 'blah',
261
+ }
262
+
263
+ batch.jobs do
264
+ # Enqueue jobs like normal
265
+ end
266
+ ```
267
+
268
+ #### Job Pools
269
+ A job pool is like a custom Sidekiq Queue. You can add jobs to it and it will empty itself out into one of the actual queues.
270
+ However, it adds some options for tweaking the logic:
271
+ - `concurrency` (default: `nil`) - Define how many jobs from the pool can run at once.
272
+ - `order` (default: `fifo`) - Define how the pool will empty itself
273
+ - `fifo` - First-In First-Out, a traditional queue
274
+ - `lifo` - Last-In First-Out
275
+ - `random` - Pluck and run jobs in random order
276
+ - `priority` - Execute jobs in a priority order (NB: Due to Redis, this priority-random, meaning that items with the same priority will be run in random order, not fifo)
277
+ - `clean_when_empty` (default: `true`) - Automatically clean the pool when it is empty
278
+ - `on_failed_job` (default `:wait`) - If a Job fails, should the pool `:continue` and still enqueue the next job or `:wait` for the job to succeed
279
+
280
+ Example:
281
+ ```ruby
282
+ pool = CanvasSync::JobBatches::Pool.new(concurrency: 4, order: :priority, clean_when_empty: false)
283
+ pool_id = pool.pid
284
+
285
+ # Add a job to the pool
286
+ pool << {
287
+ job: SomeJob, # The Class of a ActiveJob Job or Sidekiq Worker
288
+ parameters: [1, 2, 3], # Array of params to pass th e Job
289
+ priority: 100, # Only effective if order=:priority, higher is higher
290
+ }
291
+
292
+ # Add many jobs to the pool
293
+ pool.add_jobs([
294
+ {
295
+ job: SomeJob, # The Class of a ActiveJob Job or Sidekiq Worker
296
+ parameters: [1, 2, 3], # Array of params to pass th e Job
297
+ priority: 100, # Only effective if order=:priority, higher is higher
298
+ },
299
+ # ...
300
+ ])
301
+
302
+ # ...Later
303
+ CanvasSync::JobBatches::Pool.from_pid(pool_id).cleanup_redis
304
+
305
+ ```
306
+
249
307
  ## Legacy Support
250
308
 
251
309
  ### Legacy Mappings
@@ -4,10 +4,13 @@ begin
4
4
  rescue LoadError
5
5
  end
6
6
 
7
+ require_relative './redis_model'
8
+ require_relative './redis_script'
7
9
  require_relative './batch_aware_job'
8
10
  require_relative "./callback"
9
11
  require_relative "./context_hash"
10
12
  require_relative "./status"
13
+ require_relative "./pool"
11
14
  Dir[File.dirname(__FILE__) + "/jobs/*.rb"].each { |file| require file }
12
15
  require_relative "./chain_builder"
13
16
 
@@ -16,25 +19,12 @@ require_relative "./chain_builder"
16
19
 
17
20
  module CanvasSync
18
21
  module JobBatches
19
- class Batch
20
- class NoBlockGivenError < StandardError; end
22
+ class SuccessfulFailure < RuntimeError; end
21
23
 
22
- def self.batch_attr(key, read_only: true)
23
- class_eval <<-RUBY, __FILE__, __LINE__ + 1
24
- def #{key}=(value)
25
- raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
26
- @#{key} = value
27
- persist_bid_attr('#{key}', value)
28
- end
24
+ class Batch
25
+ include RedisModel
29
26
 
30
- def #{key}
31
- return @#{key} if defined?(@#{key})
32
- if (@initialized || @existing)
33
- @#{key} = read_bid_attr('#{key}')
34
- end
35
- end
36
- RUBY
37
- end
27
+ class NoBlockGivenError < StandardError; end
38
28
 
39
29
  delegate :redis, to: :class
40
30
 
@@ -47,16 +37,14 @@ module CanvasSync
47
37
  @existing = !(!existing_bid || existing_bid.empty?) # Basically existing_bid.present?
48
38
  @initialized = false
49
39
  @bidkey = "BID-" + @bid.to_s
50
- @pending_attrs = {}
51
- @ready_to_queue = nil
52
40
  self.created_at = Time.now.utc.to_f unless @existing
53
41
  end
54
42
 
55
- batch_attr :description
56
- batch_attr :created_at
57
- batch_attr :callback_queue, read_only: false
58
- batch_attr :callback_batch, read_only: false
59
- batch_attr :allow_context_changes
43
+ redis_attr :description
44
+ redis_attr :created_at
45
+ redis_attr :callback_queue, read_only: false
46
+ redis_attr :callback_params, :json
47
+ redis_attr :allow_context_changes
60
48
 
61
49
  def context
62
50
  return @context if defined?(@context)
@@ -113,7 +101,7 @@ module CanvasSync
113
101
  if parent_bid
114
102
  r.hincrby("BID-#{parent_bid}", "children", 1)
115
103
  r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
116
- r.sadd("BID-#{parent_bid}-bids", bid)
104
+ r.zadd("BID-#{parent_bid}-bids", created_at, bid)
117
105
  end
118
106
  end
119
107
  end
@@ -122,29 +110,24 @@ module CanvasSync
122
110
  @context&.save!
123
111
 
124
112
  @initialized = true
113
+ else
114
+ assert_batch_is_open
125
115
  end
126
116
 
127
- job_queue = @ready_to_queue = []
128
-
129
117
  begin
130
118
  parent = Thread.current[:batch]
131
119
  Thread.current[:batch] = self
132
120
  yield
133
121
  ensure
134
- @ready_to_queue = nil
135
- append_jobs(job_queue, parent_bid)
136
122
  Thread.current[:batch] = parent
137
123
  end
138
124
 
139
- job_queue
125
+ nil
140
126
  end
141
127
 
142
128
  def increment_job_queue(jid)
143
- if @ready_to_queue
144
- @ready_to_queue << jid
145
- else
146
- append_jobs([jid])
147
- end
129
+ assert_batch_is_open
130
+ append_jobs([jid])
148
131
  end
149
132
 
150
133
  def invalidate_all
@@ -170,58 +153,56 @@ module CanvasSync
170
153
  batch.parent ? valid && valid?(batch.parent) : valid
171
154
  end
172
155
 
173
- # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
174
- def self.without_batch
156
+ def self.with_batch(batch)
157
+ batch = self.new(batch) if batch.is_a?(String)
175
158
  parent = Thread.current[:batch]
176
- Thread.current[:batch] = nil
159
+ Thread.current[:batch] = batch
177
160
  yield
178
161
  ensure
179
162
  Thread.current[:batch] = parent
180
163
  end
181
164
 
182
- private
165
+ # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
166
+ def self.without_batch(&blk)
167
+ with_batch(nil, &blk)
168
+ end
183
169
 
184
- def persist_bid_attr(attribute, value)
185
- if @initialized || @existing
186
- redis do |r|
187
- r.multi do
188
- r.hset(@bidkey, attribute, value)
189
- r.expire(@bidkey, BID_EXPIRE_TTL)
190
- end
191
- end
192
- else
193
- @pending_attrs[attribute] = value
194
- end
170
+ protected
171
+
172
+ def redis_key
173
+ @bidkey
195
174
  end
196
175
 
197
- def read_bid_attr(attribute)
176
+ def flush_pending_attrs
177
+ super
198
178
  redis do |r|
199
- r.hget(@bidkey, attribute)
179
+ r.zadd("batches", created_at, bid)
200
180
  end
201
181
  end
202
182
 
203
- def flush_pending_attrs
204
- redis do |r|
205
- r.mapped_hmset(@bidkey, @pending_attrs)
183
+ private
184
+
185
+ def assert_batch_is_open
186
+ unless defined?(@closed)
187
+ redis do |r|
188
+ @closed = r.hget(@bidkey, 'success') == 'true'
189
+ end
206
190
  end
207
- @pending_attrs = {}
191
+ raise "Cannot add jobs to Batch #{} bid - it has already entered the callback-stage" if @closed
208
192
  end
209
193
 
210
- def append_jobs(jids, parent_bid = self.parent_bid)
194
+ def append_jobs(jids)
195
+ jids = jids.uniq
196
+ return unless jids.size > 0
197
+
211
198
  redis do |r|
199
+ tme = Time.now.utc.to_f
200
+ added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
212
201
  r.multi do
213
- if parent_bid
214
- r.hincrby("BID-#{parent_bid}", "total", jids.size)
215
- end
216
-
217
- r.hincrby(@bidkey, "pending", jids.size)
218
- r.hincrby(@bidkey, "total", jids.size)
202
+ r.hincrby(@bidkey, "pending", added)
203
+ r.hincrby(@bidkey, "job_count", added)
219
204
  r.expire(@bidkey, BID_EXPIRE_TTL)
220
-
221
- if jids.size > 0
222
- r.sadd(@bidkey + "-jids", jids)
223
- r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
224
- end
205
+ r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
225
206
  end
226
207
  end
227
208
  end
@@ -229,6 +210,8 @@ module CanvasSync
229
210
  class << self
230
211
  def process_failed_job(bid, jid)
231
212
  _, pending, failed, children, complete, parent_bid = redis do |r|
213
+ return unless r.exists?("BID-#{bid}")
214
+
232
215
  r.multi do
233
216
  r.sadd("BID-#{bid}-failed", jid)
234
217
 
@@ -242,17 +225,6 @@ module CanvasSync
242
225
  end
243
226
  end
244
227
 
245
- # if the batch failed, and has a parent, update the parent to show one pending and failed job
246
- if parent_bid
247
- redis do |r|
248
- r.multi do
249
- r.hincrby("BID-#{parent_bid}", "pending", 1)
250
- r.sadd("BID-#{parent_bid}-failed", jid)
251
- r.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL)
252
- end
253
- end
254
- end
255
-
256
228
  if pending.to_i == failed.to_i && children == complete
257
229
  enqueue_callbacks(:complete, bid)
258
230
  end
@@ -260,6 +232,8 @@ module CanvasSync
260
232
 
261
233
  def process_dead_job(bid, jid)
262
234
  _, failed, children, complete, parent_bid = redis do |r|
235
+ return unless r.exists?("BID-#{bid}")
236
+
263
237
  r.multi do
264
238
  r.sadd("BID-#{bid}-dead", jid)
265
239
 
@@ -285,7 +259,9 @@ module CanvasSync
285
259
  end
286
260
 
287
261
  def process_successful_job(bid, jid)
288
- _, failed, pending, children, complete, success, total, parent_bid = redis do |r|
262
+ _, failed, pending, children, complete, success, parent_bid = redis do |r|
263
+ return unless r.exists?("BID-#{bid}")
264
+
289
265
  r.multi do
290
266
  r.srem("BID-#{bid}-failed", jid)
291
267
 
@@ -294,10 +270,10 @@ module CanvasSync
294
270
  r.hincrby("BID-#{bid}", "children", 0)
295
271
  r.scard("BID-#{bid}-batches-complete")
296
272
  r.scard("BID-#{bid}-batches-success")
297
- r.hget("BID-#{bid}", "total")
298
273
  r.hget("BID-#{bid}", "parent_bid")
299
274
 
300
- r.srem("BID-#{bid}-jids", jid)
275
+ r.hincrby("BID-#{bid}", "successful-jobs", 1)
276
+ r.zrem("BID-#{bid}-jids", jid)
301
277
  r.expire("BID-#{bid}", BID_EXPIRE_TTL)
302
278
  end
303
279
  end
@@ -313,14 +289,15 @@ module CanvasSync
313
289
  def enqueue_callbacks(event, bid)
314
290
  batch_key = "BID-#{bid}"
315
291
  callback_key = "#{batch_key}-callbacks-#{event}"
316
- already_processed, _, callbacks, queue, parent_bid, callback_batch = redis do |r|
292
+ already_processed, _, callbacks, queue, parent_bid, callback_params = redis do |r|
293
+ return unless r.exists?(batch_key)
317
294
  r.multi do
318
295
  r.hget(batch_key, event)
319
296
  r.hset(batch_key, event, true)
320
297
  r.smembers(callback_key)
321
298
  r.hget(batch_key, "callback_queue")
322
299
  r.hget(batch_key, "parent_bid")
323
- r.hget(batch_key, "callback_batch")
300
+ r.hget(batch_key, "callback_params")
324
301
  end
325
302
  end
326
303
 
@@ -328,6 +305,7 @@ module CanvasSync
328
305
 
329
306
  queue ||= "default"
330
307
  parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
308
+ callback_params = JSON.parse(callback_params) if callback_params.present?
331
309
  callback_args = callbacks.reduce([]) do |memo, jcb|
332
310
  cb = JSON.load(jcb)
333
311
  memo << [cb['callback'], event.to_s, cb['opts'], bid, parent_bid]
@@ -335,44 +313,44 @@ module CanvasSync
335
313
 
336
314
  opts = {"bid" => bid, "event" => event}
337
315
 
338
- # Run callback batch finalize synchronously
339
- if callback_batch
340
- # Extract opts from cb_args or use current
341
- # Pass in stored event as callback finalize is processed on complete event
342
- cb_opts = callback_args.first&.at(2) || opts
316
+ if callback_args.present? && !callback_params.present?
317
+ logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
343
318
 
344
- logger.debug {"Run callback batch bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
345
- # Finalize now
346
- finalizer = Batch::Callback::Finalize.new
347
- status = Status.new bid
348
- finalizer.dispatch(status, cb_opts)
319
+ redis do |r|
320
+ r.sadd("#{batch_key}-pending_callbacks", event)
321
+ r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
322
+ end
349
323
 
350
- return
324
+ with_batch(parent_bid) do
325
+ cb_batch = self.new
326
+ cb_batch.callback_params = {
327
+ for_bid: bid,
328
+ event: event,
329
+ }
330
+ opts['callback_bid'] = cb_batch.bid
331
+
332
+ logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
333
+ cb_batch.jobs do
334
+ push_callbacks callback_args, queue
335
+ end
336
+ end
351
337
  end
352
338
 
353
- logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
354
-
355
- if callback_args.empty?
356
- # Finalize now
357
- finalizer = Batch::Callback::Finalize.new
358
- status = Status.new bid
359
- finalizer.dispatch(status, opts)
360
- else
361
- # Otherwise finalize in sub batch complete callback
362
- cb_batch = self.new
363
- cb_batch.callback_batch = true
364
- logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
365
- cb_batch.on(:complete, "#{Batch::Callback::Finalize.to_s}#dispatch", opts)
366
- cb_batch.jobs do
367
- push_callbacks callback_args, queue
368
- end
339
+ if callback_params.present?
340
+ opts['origin'] = callback_params
369
341
  end
342
+
343
+ logger.debug {"Run batch finalizer bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
344
+ finalizer = Batch::Callback::Finalize.new
345
+ status = Status.new bid
346
+ finalizer.dispatch(status, opts)
370
347
  end
371
348
 
372
349
  def cleanup_redis(bid)
373
350
  logger.debug {"Cleaning redis of batch #{bid}"}
374
351
  redis do |r|
375
- r.del(
352
+ r.zrem("batches", bid)
353
+ r.unlink(
376
354
  "BID-#{bid}",
377
355
  "BID-#{bid}-callbacks-complete",
378
356
  "BID-#{bid}-callbacks-success",
@@ -381,11 +359,23 @@ module CanvasSync
381
359
  "BID-#{bid}-batches-success",
382
360
  "BID-#{bid}-batches-complete",
383
361
  "BID-#{bid}-batches-failed",
362
+ "BID-#{bid}-bids",
384
363
  "BID-#{bid}-jids",
364
+ "BID-#{bid}-pending_callbacks",
385
365
  )
386
366
  end
387
367
  end
388
368
 
369
+ def delete_prematurely!(bid)
370
+ child_bids = redis do |r|
371
+ r.zrange("BID-#{bid}-bids", 0, -1)
372
+ end
373
+ child_bids.each do |cbid|
374
+ delete_prematurely!(cbid)
375
+ end
376
+ cleanup_redis(bid)
377
+ end
378
+
389
379
  def redis(*args, &blk)
390
380
  defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
391
381
  end
@@ -394,10 +384,8 @@ module CanvasSync
394
384
  defined?(::Sidekiq) ? ::Sidekiq.logger : Rails.logger
395
385
  end
396
386
 
397
- private
398
-
399
387
  def push_callbacks(args, queue)
400
- Batch::Callback::Worker.enqueue_all(args, queue)
388
+ Batch::Callback::worker_class.enqueue_all(args, queue)
401
389
  end
402
390
  end
403
391
  end