canvas_sync 0.17.0 → 0.17.3.beta3

Sign up to get free protection for your applications and to get access to all the features.
@@ -69,6 +69,7 @@ RSpec.describe CanvasSync::JobBatches::BatchAwareJob do
69
69
  Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid)
70
70
  Thread.current[:batch].instance_variable_set(:@open, true)
71
71
  end
72
+ after { Thread.current[:batch] = nil }
72
73
 
73
74
  it 'yields' do
74
75
  expect {
@@ -193,6 +193,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
193
193
 
194
194
  context 'success' do
195
195
  before { batch.on(:complete, Object) }
196
+
196
197
  it 'tries to call complete callback' do
197
198
  expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
198
199
  expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
@@ -208,6 +209,61 @@ RSpec.describe CanvasSync::JobBatches::Batch do
208
209
  CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
209
210
  end
210
211
 
212
+ it 'triggers callbacks as expected' do
213
+ ActiveJob::Base.queue_adapter = :sidekiq
214
+ CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
215
+
216
+ callback_instance = double('SampleCallback')
217
+ expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
218
+ expect(callback_instance).to receive(:on_complete)
219
+ expect(callback_instance).to receive(:on_success)
220
+
221
+ batch.on(:complete, SampleCallback)
222
+ batch.on(:success, SampleCallback)
223
+
224
+ Sidekiq::Testing.inline! do
225
+ CanvasSync::JobBatches::Batch.process_failed_job(bid, jid)
226
+ CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
227
+ end
228
+ end
229
+
230
+ it 'triggers callbacks as expected' do
231
+ ActiveJob::Base.queue_adapter = :sidekiq
232
+ CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 0) }
233
+
234
+ class RetryingJob < BatchTestJobBase
235
+ @@failed = false
236
+
237
+ def perform
238
+ unless @@failed
239
+ @@failed = true
240
+ raise "A Failure"
241
+ end
242
+ end
243
+ end
244
+
245
+ callback_instance = double('SampleCallback')
246
+ expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
247
+ expect(callback_instance).to receive(:on_complete)
248
+ expect(callback_instance).to receive(:on_success)
249
+
250
+ batch.on(:complete, SampleCallback)
251
+ batch.on(:success, SampleCallback)
252
+
253
+ batch.jobs do
254
+ RetryingJob.perform_later
255
+ end
256
+
257
+ job_def = Sidekiq::Worker.jobs[0]
258
+ int_job_class = job_def["class"].constantize
259
+
260
+ begin
261
+ int_job_class.process_job(job_def)
262
+ rescue
263
+ end
264
+ Sidekiq::Worker.drain_all
265
+ end
266
+
211
267
  it 'cleanups redis key' do
212
268
  CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
213
269
  expect(CanvasSync::JobBatches::Batch.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0)
@@ -224,12 +280,6 @@ RSpec.describe CanvasSync::JobBatches::Batch do
224
280
  pending = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'pending') }
225
281
  expect(pending).to eq('1')
226
282
  end
227
-
228
- it 'increments total' do
229
- batch.jobs do TestWorker.perform_async end
230
- total = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'total') }
231
- expect(total).to eq('1')
232
- end
233
283
  end
234
284
 
235
285
  describe '#enqueue_callbacks' do
@@ -241,6 +291,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
241
291
  context 'when no callbacks are defined' do
242
292
  it 'clears redis keys' do
243
293
  batch = CanvasSync::JobBatches::Batch.new
294
+ batch.jobs {}
244
295
  expect(CanvasSync::JobBatches::Batch).to receive(:cleanup_redis).with(batch.bid)
245
296
  CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
246
297
  end
@@ -260,8 +311,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
260
311
 
261
312
  context 'With ActiveJob Adapter' do
262
313
  around(:all) do |block|
263
- CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
264
- CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::ActiveJobCallbackWorker)
314
+ CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Batch::Callback::ActiveJobCallbackWorker
265
315
  block.run
266
316
  end
267
317
 
@@ -279,12 +329,15 @@ RSpec.describe CanvasSync::JobBatches::Batch do
279
329
  let(:opts) { { 'a' => 'b' } }
280
330
 
281
331
  it 'calls it passing options' do
332
+ ActiveJob::Base.queue_adapter = :test
333
+
282
334
  batch = CanvasSync::JobBatches::Batch.new
283
335
  batch.on(event, SampleCallback, opts)
336
+ batch.jobs {}
284
337
 
285
338
  CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
286
339
 
287
- expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
340
+ expect(CanvasSync::JobBatches::Batch::Callback.worker_class).to have_been_enqueued.with(
288
341
  'SampleCallback', event.to_s, opts, batch.bid, nil
289
342
  )
290
343
  end
@@ -295,15 +348,18 @@ RSpec.describe CanvasSync::JobBatches::Batch do
295
348
  let(:opts2) { { 'b' => 'a' } }
296
349
 
297
350
  it 'enqueues each callback passing their options' do
351
+ ActiveJob::Base.queue_adapter = :test
352
+
298
353
  batch = CanvasSync::JobBatches::Batch.new
299
354
  batch.on(event, SampleCallback, opts)
300
355
  batch.on(event, SampleCallback2, opts2)
356
+ batch.jobs{}
301
357
 
302
358
  CanvasSync::JobBatches::Batch.enqueue_callbacks(event, batch.bid)
303
- expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
359
+ expect(CanvasSync::JobBatches::Batch::Callback.worker_class).to have_been_enqueued.with(
304
360
  'SampleCallback2', event.to_s, opts2, batch.bid, nil
305
361
  )
306
- expect(CanvasSync::JobBatches::Batch::Callback::Worker).to have_been_enqueued.with(
362
+ expect(CanvasSync::JobBatches::Batch::Callback.worker_class).to have_been_enqueued.with(
307
363
  'SampleCallback', event.to_s, opts, batch.bid, nil
308
364
  )
309
365
  end
@@ -313,8 +369,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
313
369
 
314
370
  context 'With Sidekiq Adapter' do
315
371
  around(:all) do |block|
316
- CanvasSync::JobBatches::Batch::Callback.send(:remove_const, :Worker)
317
- CanvasSync::JobBatches::Batch::Callback.const_set(:Worker, CanvasSync::JobBatches::Batch::Callback::SidekiqCallbackWorker)
372
+ CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
318
373
  block.run
319
374
  end
320
375
 
@@ -334,9 +389,10 @@ RSpec.describe CanvasSync::JobBatches::Batch do
334
389
  it 'calls it passing options' do
335
390
  batch = CanvasSync::JobBatches::Batch.new
336
391
  batch.on(event, SampleCallback, opts)
392
+ batch.jobs{}
337
393
 
338
394
  expect(Sidekiq::Client).to receive(:push_bulk).with(
339
- 'class' => Sidekiq::Batch::Callback::Worker,
395
+ 'class' => Sidekiq::Batch::Callback.worker_class,
340
396
  'args' => [['SampleCallback', event.to_s, opts, batch.bid, nil]],
341
397
  'queue' => 'default'
342
398
  )
@@ -353,9 +409,10 @@ RSpec.describe CanvasSync::JobBatches::Batch do
353
409
  batch = CanvasSync::JobBatches::Batch.new
354
410
  batch.on(event, SampleCallback, opts)
355
411
  batch.on(event, SampleCallback2, opts2)
412
+ batch.jobs{}
356
413
 
357
414
  expect(Sidekiq::Client).to receive(:push_bulk).with(
358
- 'class' => Sidekiq::Batch::Callback::Worker,
415
+ 'class' => Sidekiq::Batch::Callback.worker_class,
359
416
  'args' => [
360
417
  ['SampleCallback2', event.to_s, opts2, batch.bid, nil],
361
418
  ['SampleCallback', event.to_s, opts, batch.bid, nil]
@@ -1,6 +1,6 @@
1
1
  require 'spec_helper'
2
2
 
3
- RSpec.describe CanvasSync::JobBatches::Batch::Callback::Worker do
3
+ RSpec.describe CanvasSync::JobBatches::Batch::Callback.worker_class do
4
4
  describe '#perform' do
5
5
  it 'does not do anything if it cannot find the callback class' do
6
6
  subject.perform('SampleCallback', 'complete', {}, 'ABCD', 'EFGH')
@@ -21,30 +21,24 @@ RSpec.describe 'Batch flow' do
21
21
  before { batch.on(:complete, SampleCallback, :id => 42) }
22
22
  before { batch.description = 'describing the batch' }
23
23
  let(:status) { CanvasSync::JobBatches::Batch::Status.new(batch.bid) }
24
- let(:jids) { batch.jobs do 3.times do TestWorker.perform_async end end }
25
24
  let(:queue) { Sidekiq::Queue.new }
26
25
 
27
26
  it 'correctly initializes' do
28
- expect(jids.size).to eq(3)
29
-
30
27
  expect(batch.bid).not_to be_nil
31
28
  expect(batch.description).to eq('describing the batch')
32
29
 
33
- batch.jobs {}
30
+ batch.jobs do
31
+ 3.times do
32
+ TestWorker.perform_async
33
+ end
34
+ end
34
35
 
35
- expect(status.total).to eq(3)
36
36
  expect(status.pending).to eq(3)
37
37
  expect(status.failures).to eq(0)
38
38
  expect(status.complete?).to be false
39
39
  expect(status.created_at).not_to be_nil
40
40
  expect(status.bid).to eq(batch.bid)
41
41
  end
42
-
43
- it 'handles an empty batch' do
44
- batch = CanvasSync::JobBatches::Batch.new
45
- jids = batch.jobs do nil end
46
- expect(jids.size).to eq(0)
47
- end
48
42
  end
49
43
 
50
44
  context 'when handling a nested batch' do
@@ -0,0 +1,42 @@
1
+ require_relative '../integration_helper'
2
+
3
+ # Workflow when a Job fails, retries, and then succeeds
4
+
5
+ class Worker1
6
+ include Sidekiq::Worker
7
+ sidekiq_options retry: 5
8
+
9
+ @@failed = false
10
+
11
+ def perform
12
+ Sidekiq.logger.info "Work 1"
13
+
14
+ unless @@failed
15
+ @@failed = true
16
+ raise "One Failure"
17
+ end
18
+ end
19
+ end
20
+
21
+ class MyCallback
22
+ def on_success(status, options)
23
+ Sidekiq.logger.info "Overall Success #{options} #{status.data}"
24
+ end
25
+ alias_method :multi, :on_success
26
+
27
+ def on_complete(status, options)
28
+ Sidekiq.logger.info "Overall Complete #{options} #{status.data}"
29
+ end
30
+ end
31
+
32
+ overall = CanvasSync::JobBatches::Batch.new
33
+ overall.on(:success, MyCallback, to: 'success@gmail.com')
34
+ overall.on(:complete, MyCallback, to: 'success@gmail.com')
35
+ overall.jobs do
36
+ Worker1.perform_async
37
+ end
38
+
39
+ puts "Overall bid #{overall.bid}"
40
+
41
+ output, keys = process_tests
42
+ overall_tests(output, keys, file: __FILE__)
@@ -1,9 +1,8 @@
1
1
  require 'spec_helper'
2
- require 'sidekiq/batch'
3
2
  require 'sidekiq/testing'
4
3
 
5
4
  Sidekiq::Testing.server_middleware do |chain|
6
- chain.add CanvasSync::JobBatches::Batch::Middleware::ServerMiddleware
5
+ chain.add CanvasSync::JobBatches::Sidekiq::ServerMiddleware
7
6
  end
8
7
 
9
8
  Sidekiq.redis { |r| r.flushdb }
@@ -31,8 +30,11 @@ def process_tests
31
30
  [output, keys]
32
31
  end
33
32
 
34
- def overall_tests output, keys
35
- describe "sidekiq batch" do
33
+ def overall_tests(output, keys, file: nil)
34
+ test_name = "Batch Integration Test"
35
+ test_name = File.basename(file, ".*") if file
36
+
37
+ Rspec.describe test_name do
36
38
  it "runs overall complete callback" do
37
39
  expect(output).to include "Overall Complete"
38
40
  end
@@ -58,6 +58,7 @@ RSpec.describe CanvasSync::JobBatches::Sidekiq do
58
58
  let(:bid) { 'SAMPLEBID' }
59
59
  let(:jid) { 'SAMPLEJID' }
60
60
  before { Thread.current[:batch] = CanvasSync::JobBatches::Batch.new(bid) }
61
+ after { Thread.current[:batch] = nil }
61
62
 
62
63
  it 'yields' do
63
64
  yielded = false
@@ -59,25 +59,9 @@ RSpec.describe CanvasSync::JobBatches::Batch::Status do
59
59
  end
60
60
  end
61
61
 
62
- describe '#total' do
63
- context 'when not initalized' do
64
- it 'returns 0 failed jobs' do
65
- expect(subject.total).to eq(0)
66
- end
67
- end
68
-
69
- context 'when more than 0' do
70
- before { batch.jobs do TestWorker.perform_async end }
71
-
72
- it 'returns failed jobs' do
73
- expect(subject.total).to eq(1)
74
- end
75
- end
76
- end
77
-
78
62
  describe '#data' do
79
63
  it 'returns batch description' do
80
- expect(subject.data).to include(total: 0, failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
64
+ expect(subject.data).to include(failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
81
65
  end
82
66
  end
83
67
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: canvas_sync
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.17.0
4
+ version: 0.17.3.beta3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nate Collings
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-05 00:00:00.000000000 Z
11
+ date: 2020-11-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -427,9 +427,14 @@ files:
427
427
  - lib/canvas_sync/job_batches/callback.rb
428
428
  - lib/canvas_sync/job_batches/chain_builder.rb
429
429
  - lib/canvas_sync/job_batches/context_hash.rb
430
+ - lib/canvas_sync/job_batches/hincr_max.lua
430
431
  - lib/canvas_sync/job_batches/jobs/base_job.rb
431
432
  - lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb
433
+ - lib/canvas_sync/job_batches/jobs/managed_batch_job.rb
432
434
  - lib/canvas_sync/job_batches/jobs/serial_batch_job.rb
435
+ - lib/canvas_sync/job_batches/pool.rb
436
+ - lib/canvas_sync/job_batches/redis_model.rb
437
+ - lib/canvas_sync/job_batches/redis_script.rb
433
438
  - lib/canvas_sync/job_batches/sidekiq.rb
434
439
  - lib/canvas_sync/job_batches/status.rb
435
440
  - lib/canvas_sync/jobs/begin_sync_chain_job.rb
@@ -598,6 +603,7 @@ files:
598
603
  - spec/job_batching/batch_spec.rb
599
604
  - spec/job_batching/callback_spec.rb
600
605
  - spec/job_batching/flow_spec.rb
606
+ - spec/job_batching/integration/fail_then_succeed.rb
601
607
  - spec/job_batching/integration/integration.rb
602
608
  - spec/job_batching/integration/nested.rb
603
609
  - spec/job_batching/integration/simple.rb
@@ -643,9 +649,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
643
649
  version: '0'
644
650
  required_rubygems_version: !ruby/object:Gem::Requirement
645
651
  requirements:
646
- - - ">="
652
+ - - ">"
647
653
  - !ruby/object:Gem::Version
648
- version: '0'
654
+ version: 1.3.1
649
655
  requirements: []
650
656
  rubygems_version: 3.0.3
651
657
  signing_key:
@@ -790,6 +796,7 @@ test_files:
790
796
  - spec/job_batching/batch_spec.rb
791
797
  - spec/job_batching/callback_spec.rb
792
798
  - spec/job_batching/flow_spec.rb
799
+ - spec/job_batching/integration/fail_then_succeed.rb
793
800
  - spec/job_batching/integration/integration.rb
794
801
  - spec/job_batching/integration/nested.rb
795
802
  - spec/job_batching/integration/simple.rb