canvas_sync 0.17.0.beta3 → 0.17.0.beta8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d7f0019fb32e5940971c4845d47a317bd4f451dec546a96a505e3162c8a120d4
4
- data.tar.gz: 563d6fb7375f3576f8d2eac14752e14ebedbd210cc63b277c311d0e1c846da7e
3
+ metadata.gz: 5b9dfa016822ff745b64793d0e25e00a9080ba783af50f126c49271cc9a0d021
4
+ data.tar.gz: eb7120d981dcf6402afac72c2e3d5785f3da59d20ce151952f43de6d8b829ef7
5
5
  SHA512:
6
- metadata.gz: ce3550e2f0aa94df6c6b712776d605c16cac568f2a70d49ee668ce5ebdc0978654f01d63619d38bdb1a7befe670518210dbc1e847497fababa4428f779bfd775
7
- data.tar.gz: 48a0e195407552a88b46ee93d7e8706585bd4729c618cdb807971a8694405c9e442b361da21472cf9c9d301bbc4dddb0c4c28b7faa5a8f3f4daf1a013195d6f3
6
+ metadata.gz: 98e434b79e7d96fc64db88a879df7f2b3d6e754bafaa9a8b1adba9f3fb3fad6fd4ca44c5b7b0307155065354738ef4a4f0101907f5b6dbf1d2a96fa74a040802
7
+ data.tar.gz: 3f051aeb858f550588a67ae83d812e8a08d304b3dbe448221c0ffbeb7a6618a1bdd13f1b7bd619344d66b4dcd0d4a9d69162847fbbc36dfb770509bd84339468
@@ -118,40 +118,29 @@ module CanvasSync
118
118
  @initialized = true
119
119
  end
120
120
 
121
- @ready_to_queue = []
121
+ job_queue = @ready_to_queue = []
122
+
123
+ puts "Beginning Batch #{@bidkey}"
122
124
 
123
125
  begin
124
126
  parent = Thread.current[:batch]
125
127
  Thread.current[:batch] = self
126
128
  yield
127
129
  ensure
130
+ @ready_to_queue = nil
131
+ append_jobs(job_queue, parent_bid)
128
132
  Thread.current[:batch] = parent
129
133
  end
130
134
 
131
- redis do |r|
132
- r.multi do
133
- if parent_bid
134
- r.hincrby("BID-#{parent_bid}", "children", 1)
135
- r.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
136
- r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
137
- end
138
-
139
- r.hincrby(@bidkey, "pending", @ready_to_queue.size)
140
- r.hincrby(@bidkey, "total", @ready_to_queue.size)
141
- r.expire(@bidkey, BID_EXPIRE_TTL)
142
-
143
- if @ready_to_queue.size > 0
144
- r.sadd(@bidkey + "-jids", @ready_to_queue)
145
- r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
146
- end
147
- end
148
- end
149
-
150
- @ready_to_queue
135
+ job_queue
151
136
  end
152
137
 
153
138
  def increment_job_queue(jid)
154
- @ready_to_queue << jid
139
+ if @ready_to_queue
140
+ @ready_to_queue << jid
141
+ else
142
+ append_jobs([jid])
143
+ end
155
144
  end
156
145
 
157
146
  def invalidate_all
@@ -214,6 +203,27 @@ module CanvasSync
214
203
  @pending_attrs = {}
215
204
  end
216
205
 
206
+ def append_jobs(jids, parent_bid = self.parent_bid)
207
+ redis do |r|
208
+ r.multi do
209
+ if parent_bid
210
+ r.hincrby("BID-#{parent_bid}", "children", 1)
211
+ r.hincrby("BID-#{parent_bid}", "total", jids.size)
212
+ r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
213
+ end
214
+
215
+ r.hincrby(@bidkey, "pending", jids.size)
216
+ r.hincrby(@bidkey, "total", jids.size)
217
+ r.expire(@bidkey, BID_EXPIRE_TTL)
218
+
219
+ if jids.size > 0
220
+ r.sadd(@bidkey + "-jids", jids)
221
+ r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
222
+ end
223
+ end
224
+ end
225
+ end
226
+
217
227
  class << self
218
228
  def process_failed_job(bid, jid)
219
229
  _, pending, failed, children, complete, parent_bid = redis do |r|
@@ -223,7 +233,7 @@ module CanvasSync
223
233
  r.hincrby("BID-#{bid}", "pending", 0)
224
234
  r.scard("BID-#{bid}-failed")
225
235
  r.hincrby("BID-#{bid}", "children", 0)
226
- r.scard("BID-#{bid}-complete")
236
+ r.scard("BID-#{bid}-batches-complete")
227
237
  r.hget("BID-#{bid}", "parent_bid")
228
238
 
229
239
  r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
@@ -253,7 +263,7 @@ module CanvasSync
253
263
 
254
264
  r.scard("BID-#{bid}-dead")
255
265
  r.hincrby("BID-#{bid}", "children", 0)
256
- r.scard("BID-#{bid}-complete")
266
+ r.scard("BID-#{bid}-batches-complete")
257
267
  r.hget("BID-#{bid}", "parent_bid")
258
268
 
259
269
  r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
@@ -273,17 +283,18 @@ module CanvasSync
273
283
  end
274
284
 
275
285
  def process_successful_job(bid, jid)
276
- failed, pending, children, complete, success, total, parent_bid = redis do |r|
286
+ _, failed, pending, children, complete, success, total, parent_bid = redis do |r|
277
287
  r.multi do
288
+ r.srem("BID-#{bid}-failed", jid)
289
+
278
290
  r.scard("BID-#{bid}-failed")
279
291
  r.hincrby("BID-#{bid}", "pending", -1)
280
292
  r.hincrby("BID-#{bid}", "children", 0)
281
- r.scard("BID-#{bid}-complete")
282
- r.scard("BID-#{bid}-success")
293
+ r.scard("BID-#{bid}-batches-complete")
294
+ r.scard("BID-#{bid}-batches-success")
283
295
  r.hget("BID-#{bid}", "total")
284
296
  r.hget("BID-#{bid}", "parent_bid")
285
297
 
286
- r.srem("BID-#{bid}-failed", jid)
287
298
  r.srem("BID-#{bid}-jids", jid)
288
299
  r.expire("BID-#{bid}", BID_EXPIRE_TTL)
289
300
  end
@@ -358,18 +369,19 @@ module CanvasSync
358
369
 
359
370
  def cleanup_redis(bid)
360
371
  logger.debug {"Cleaning redis of batch #{bid}"}
361
- redis do |r|
362
- r.del(
363
- "BID-#{bid}",
364
- "BID-#{bid}-callbacks-complete",
365
- "BID-#{bid}-callbacks-success",
366
- "BID-#{bid}-failed",
367
-
368
- "BID-#{bid}-success",
369
- "BID-#{bid}-complete",
370
- "BID-#{bid}-jids",
371
- )
372
- end
372
+ # redis do |r|
373
+ # r.del(
374
+ # "BID-#{bid}",
375
+ # "BID-#{bid}-callbacks-complete",
376
+ # "BID-#{bid}-callbacks-success",
377
+ # "BID-#{bid}-failed",
378
+
379
+ # "BID-#{bid}-batches-success",
380
+ # "BID-#{bid}-batches-complete",
381
+ # "BID-#{bid}-batches-failed",
382
+ # "BID-#{bid}-jids",
383
+ # )
384
+ # end
373
385
  end
374
386
 
375
387
  def redis(*args, &blk)
@@ -87,13 +87,16 @@ module CanvasSync
87
87
  def success(bid, status, parent_bid)
88
88
  return unless parent_bid
89
89
 
90
- _, _, success, _, complete, pending, children, failure = Batch.redis do |r|
90
+ _, _, success, _, _, complete, pending, children, failure = Batch.redis do |r|
91
91
  r.multi do
92
- r.sadd("BID-#{parent_bid}-success", bid)
93
- r.expire("BID-#{parent_bid}-success", Batch::BID_EXPIRE_TTL)
94
- r.scard("BID-#{parent_bid}-success")
95
- r.sadd("BID-#{parent_bid}-complete", bid)
96
- r.scard("BID-#{parent_bid}-complete")
92
+ r.sadd("BID-#{parent_bid}-batches-success", bid)
93
+ r.expire("BID-#{parent_bid}-batches-success", Batch::BID_EXPIRE_TTL)
94
+ r.scard("BID-#{parent_bid}-batches-success")
95
+
96
+ r.srem("BID-#{parent_bid}-batches-failed", bid)
97
+ r.sadd("BID-#{parent_bid}-batches-complete", bid)
98
+ r.scard("BID-#{parent_bid}-batches-complete")
99
+
97
100
  r.hincrby("BID-#{parent_bid}", "pending", 0)
98
101
  r.hincrby("BID-#{parent_bid}", "children", 0)
99
102
  r.scard("BID-#{parent_bid}-failed")
@@ -112,7 +115,7 @@ module CanvasSync
112
115
  r.multi do
113
116
  r.hincrby("BID-#{bid}", "pending", 0)
114
117
  r.hincrby("BID-#{bid}", "children", 0)
115
- r.scard("BID-#{bid}-success")
118
+ r.scard("BID-#{bid}-batches-success")
116
119
  end
117
120
  end
118
121
 
@@ -127,10 +130,11 @@ module CanvasSync
127
130
  # callback may add more jobs to the parent batch
128
131
 
129
132
  Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
130
- _, complete, pending, children, failure = Batch.redis do |r|
133
+ _, _, complete, pending, children, failure = Batch.redis do |r|
131
134
  r.multi do
132
- r.sadd("BID-#{parent_bid}-complete", bid)
133
- r.scard("BID-#{parent_bid}-complete")
135
+ r.sadd("BID-#{parent_bid}-batches-complete", bid)
136
+ r.sadd("BID-#{parent_bid}-batches-failed", bid)
137
+ r.scard("BID-#{parent_bid}-batches-complete")
134
138
  r.hincrby("BID-#{parent_bid}", "pending", 0)
135
139
  r.hincrby("BID-#{parent_bid}", "children", 0)
136
140
  r.scard("BID-#{parent_bid}-failed")
@@ -141,11 +145,6 @@ module CanvasSync
141
145
  end
142
146
  end
143
147
  end
144
-
145
- def cleanup_redis bid, callback_bid=nil
146
- Batch.cleanup_redis bid
147
- Batch.cleanup_redis callback_bid if callback_bid
148
- end
149
148
  end
150
149
  end
151
150
  end
@@ -104,6 +104,16 @@ module CanvasSync
104
104
  end
105
105
  end
106
106
 
107
+ # Legacy Support
108
+ def merge_options(job, options)
109
+ matching_jobs = find_matching_jobs(job)
110
+
111
+ matching_jobs.each do |j|
112
+ j[:options] ||= {}
113
+ j[:options].deep_merge!(options)
114
+ end
115
+ end
116
+
107
117
  private
108
118
 
109
119
  def find_matching_jobs(search_job, parent_job = self.base_job)
@@ -5,6 +5,7 @@ module CanvasSync
5
5
  class ConcurrentBatchJob < BaseJob
6
6
  def perform(sub_jobs, context: nil)
7
7
  Batch.new.tap do |b|
8
+ b.description = "Concurrent Batch Root"
8
9
  b.context = context
9
10
  b.jobs do
10
11
  sub_jobs.each do |j|
@@ -21,6 +21,7 @@ module CanvasSync
21
21
  end
22
22
  end
23
23
 
24
+ root_batch.description = "Serial Batch Root (#{serial_id})"
24
25
  root_batch.allow_context_changes = true
25
26
  root_batch.context = context
26
27
  root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", serial_batch_id: serial_id)
@@ -61,7 +62,8 @@ module CanvasSync
61
62
 
62
63
  Batch.new(root_bid).jobs do
63
64
  Batch.new.tap do |batch|
64
- batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
65
+ batch.description = "Serial Batch Fiber (#{serial_id})"
66
+ batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
65
67
  batch.jobs do
66
68
  ChainBuilder.enqueue_job(next_job)
67
69
  end
@@ -40,10 +40,26 @@ module CanvasSync
40
40
  'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'complete') }
41
41
  end
42
42
 
43
+ def success?
44
+ 'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
45
+ end
46
+
43
47
  def child_count
44
48
  Batch.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
45
49
  end
46
50
 
51
+ def completed_children_count
52
+ Batch.redis { |r| r.scard("BID-#{bid}-batches-complete") }.to_i
53
+ end
54
+
55
+ def successful_children_count
56
+ Batch.redis { |r| r.scard("BID-#{bid}-batches-success") }.to_i
57
+ end
58
+
59
+ def failed_children_count
60
+ Batch.redis { |r| r.scard("BID-#{bid}-batches-failed") }.to_i
61
+ end
62
+
47
63
  def data
48
64
  {
49
65
  bid: bid,
@@ -52,9 +68,13 @@ module CanvasSync
52
68
  pending: pending,
53
69
  created_at: created_at,
54
70
  complete: complete?,
71
+ success: success?,
55
72
  failure_info: failure_info,
56
73
  parent_bid: parent_bid,
57
- child_count: child_count
74
+ child_count: child_count,
75
+ completed_children_count: completed_children_count,
76
+ successful_children_count: successful_children_count,
77
+ failed_children_count: failed_children_count,
58
78
  }
59
79
  end
60
80
  end
@@ -15,6 +15,7 @@ module CanvasSync
15
15
  JobBatches::Batch.new.tap do |b|
16
16
  b.description = "CanvasSync Root Batch"
17
17
  b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
18
+ b.on(:success, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
18
19
  b.context = globals
19
20
  b.jobs do
20
21
  JobBatches::SerialBatchJob.perform_now(chain_definition)
@@ -25,7 +26,7 @@ module CanvasSync
25
26
  def self.batch_completed(status, options)
26
27
  sbatch = SyncBatch.find(options['sync_batch_id'])
27
28
  sbatch.update!(
28
- status: status.failures.positive? ? 'failed' : 'completed',
29
+ status: status.success? ? 'completed' : 'failed',
29
30
  completed_at: DateTime.now,
30
31
  )
31
32
  end
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.17.0.beta3".freeze
2
+ VERSION = "0.17.0.beta8".freeze
3
3
  end
@@ -182,12 +182,12 @@ RSpec.describe CanvasSync::JobBatches::Batch do
182
182
  context 'complete' do
183
183
  before { batch.on(:complete, Object) }
184
184
  # before { batch.increment_job_queue(bid) }
185
- before { batch.jobs do TestWorker.perform_async end }
186
- before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id') }
185
+ # before { batch.jobs do TestWorker.perform_async end }
186
+ # before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id') }
187
187
 
188
188
  it 'tries to call complete callback' do
189
189
  expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid)
190
- CanvasSync::JobBatches::Batch.process_successful_job(bid, 'failed-job-id')
190
+ CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id')
191
191
  end
192
192
  end
193
193
 
@@ -199,6 +199,15 @@ RSpec.describe CanvasSync::JobBatches::Batch do
199
199
  CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
200
200
  end
201
201
 
202
+ it 'tries to call success callback after a previous failure' do
203
+ expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
204
+ CanvasSync::JobBatches::Batch.process_failed_job(bid, jid)
205
+
206
+ expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered
207
+ expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered
208
+ CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
209
+ end
210
+
202
211
  it 'cleanups redis key' do
203
212
  CanvasSync::JobBatches::Batch.process_successful_job(bid, jid)
204
213
  expect(CanvasSync::JobBatches::Batch.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0)
@@ -2,19 +2,16 @@ require 'spec_helper'
2
2
 
3
3
  class WorkerA < BatchTestJobBase
4
4
  def perform
5
- puts 'A'
6
5
  end
7
6
  end
8
7
 
9
8
  class WorkerB < BatchTestJobBase
10
9
  def perform
11
- puts 'B'
12
10
  end
13
11
  end
14
12
 
15
13
  class WorkerC < BatchTestJobBase
16
14
  def perform
17
- puts 'C'
18
15
  end
19
16
  end
20
17
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: canvas_sync
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.17.0.beta3
4
+ version: 0.17.0.beta8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nate Collings
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-10-26 00:00:00.000000000 Z
11
+ date: 2020-10-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler