canvas_sync 0.17.6 → 0.17.8.beta3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 06f6a868c0dd531a98b7999e7c132f75edb211856dd84ef5519e47b4634a0001
4
- data.tar.gz: e305790ec6923cea6eabc085b67105b3ac6f66ea3f5eaf35083d3bfb552a5558
3
+ metadata.gz: 47913d27a72537ba9c33f5b282efd811dc0fb23d0114dabfdec52efa3eb062d0
4
+ data.tar.gz: ee5a090cfd2226868fc88324e767d7c61a6256d80ed0af35dd04e7967d76b602
5
5
  SHA512:
6
- metadata.gz: fd34b80603ecaf90040446cb998e4ce7f33e4ed28c29436aee0ad5a017119d1187b0dbc7961f4300a542a97cef7af051428ba30ced733c68c085b609fcb4c225
7
- data.tar.gz: e8227d369bbf5fab0f4bf8b4fcb90310a197986fa1a8f85c56689ab521ec9abfcc59ff6b4601b66a50742873aa1f98e969038f62ca1dca19a568e081a8f3e14a
6
+ metadata.gz: 46fcee4d05468452502ffb1660962e520e34f458f4d374c4d9e974c67364db574a7b3577f65016ddd77727e3131a97232466eda98b8855db11c164517df42bbb
7
+ data.tar.gz: 2fb317efd646549443680d8e9df41c2c2e228d4b8ea3a9d6e65f9be270b918d5423995571aaec72cb4616e083cc510cf5b0a63b0f3637d96e18c26ee7f7dd294
@@ -27,6 +27,7 @@ module CanvasSync
27
27
  delegate :redis, to: :class
28
28
 
29
29
  BID_EXPIRE_TTL = 2_592_000
30
+ SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
30
31
 
31
32
  attr_reader :bid
32
33
 
@@ -289,7 +290,7 @@ module CanvasSync
289
290
  end
290
291
  end
291
292
 
292
- enqueue_callbacks(:dead, bid)
293
+ enqueue_callbacks(:death, bid)
293
294
  end
294
295
 
295
296
  def process_successful_job(bid, jid)
@@ -324,12 +325,12 @@ module CanvasSync
324
325
  def enqueue_callbacks(event, bid)
325
326
  batch_key = "BID-#{bid}"
326
327
  callback_key = "#{batch_key}-callbacks-#{event}"
327
- already_processed, _, callbacks, queue, parent_bid, callback_params = redis do |r|
328
+
329
+ callbacks, queue, parent_bid, callback_params = redis do |r|
328
330
  return unless r.exists?(batch_key)
329
331
  return if r.hget(batch_key, 'keep_open') == 'true'
332
+
330
333
  r.multi do
331
- r.hget(batch_key, event)
332
- r.hset(batch_key, event, true)
333
334
  r.smembers(callback_key)
334
335
  r.hget(batch_key, "callback_queue")
335
336
  r.hget(batch_key, "parent_bid")
@@ -337,8 +338,6 @@ module CanvasSync
337
338
  end
338
339
  end
339
340
 
340
- return if already_processed == 'true'
341
-
342
341
  queue ||= "default"
343
342
  parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
344
343
  callback_params = JSON.parse(callback_params) if callback_params.present?
@@ -348,14 +347,15 @@ module CanvasSync
348
347
  end
349
348
 
350
349
  opts = {"bid" => bid, "event" => event}
350
+ should_schedule_batch = callback_args.present? && !callback_params.present?
351
+ already_processed = redis do |r|
352
+ SCHEDULE_CALLBACK.call(r, [batch_key], [event.to_s, should_schedule_batch, BID_EXPIRE_TTL])
353
+ end
351
354
 
352
- if callback_args.present? && !callback_params.present?
353
- logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
355
+ return if already_processed == 'true'
354
356
 
355
- redis do |r|
356
- r.sadd("#{batch_key}-pending_callbacks", event)
357
- r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
358
- end
357
+ if should_schedule_batch
358
+ logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
359
359
 
360
360
  with_batch(parent_bid) do
361
361
  cb_batch = self.new
@@ -367,7 +367,7 @@ module CanvasSync
367
367
 
368
368
  logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
369
369
  cb_batch.jobs do
370
- push_callbacks callback_args, queue
370
+ push_callbacks(callback_args, queue)
371
371
  end
372
372
  end
373
373
  end
@@ -4,7 +4,7 @@ module CanvasSync
4
4
  module Callback
5
5
  mattr_accessor :worker_class
6
6
 
7
- VALID_CALLBACKS = %w[success complete dead].freeze
7
+ VALID_CALLBACKS = %w[success complete death].freeze
8
8
 
9
9
  module CallbackWorkerCommon
10
10
  def perform(definition, event, opts, bid, parent_bid)
@@ -60,8 +60,13 @@ module CanvasSync
60
60
  batch_status = Status.new bid
61
61
  send(event, bid, batch_status, batch_status.parent_bid)
62
62
 
63
+ Batch.redis do |r|
64
+ r.srem("BID-#{bid}-pending_callbacks", "#{event}-finalize")
65
+ end
66
+
63
67
  if event == :success
64
68
  if opts['origin'].present?
69
+ # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
65
70
  origin_bid = opts['origin']['for_bid']
66
71
  _, pending, success_ran = Batch.redis do |r|
67
72
  r.multi do
@@ -72,7 +77,8 @@ module CanvasSync
72
77
  end
73
78
  Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
74
79
  end
75
- if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
80
+
81
+ if (Batch.redis{|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
76
82
  Batch.cleanup_redis(bid)
77
83
  end
78
84
  end
@@ -141,6 +147,12 @@ module CanvasSync
141
147
  end
142
148
  end
143
149
  end
150
+
151
+ def death(bid, status, parent_bid)
152
+ return unless parent_bid
153
+
154
+ Batch.enqueue_callbacks(:death, parent_bid)
155
+ end
144
156
  end
145
157
  end
146
158
  end
@@ -56,12 +56,11 @@ module CanvasSync
56
56
  placement = kwargs.keys[0]
57
57
  relative_to = kwargs.values[0]
58
58
 
59
- matching_jobs = find_matching_jobs(relative_to)
59
+ matching_jobs = find_matching_jobs(relative_to).to_a
60
60
  raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
61
61
  raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
62
62
 
63
- relative_job, sub_index = matching_jobs[0]
64
- parent_job = find_parent_job(relative_job)
63
+ relative_job, parent_job, sub_index = matching_jobs[0]
65
64
  needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
66
65
 
67
66
  chain = self.class.get_chain_parameter(parent_job)
@@ -91,7 +90,7 @@ module CanvasSync
91
90
  raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
92
91
  return nil if matching_jobs.count == 0
93
92
 
94
- new(matching_jobs[0])
93
+ new(matching_jobs[0][0])
95
94
  end
96
95
 
97
96
  def normalize!(job_def = self.base_job)
@@ -108,9 +107,7 @@ module CanvasSync
108
107
 
109
108
  # Legacy Support
110
109
  def merge_options(job, options)
111
- matching_jobs = find_matching_jobs(job)
112
-
113
- matching_jobs.each do |j|
110
+ find_matching_jobs(job).each do |j, parent, index|
114
111
  j[:options] ||= {}
115
112
  j[:options].deep_merge!(options)
116
113
  end
@@ -124,9 +121,9 @@ module CanvasSync
124
121
  sub_jobs = self.class.get_chain_parameter(parent_job)
125
122
  sub_jobs.each_with_index do |sub_job, i|
126
123
  if sub_job[:job].to_s == search_job.to_s
127
- yield [parent_job, i]
128
- elsif self.class._job_type_definitions[sub_job[:job]]
129
- find_matching_jobs(search_job) { |item| yield item }
124
+ yield [sub_job, parent_job, i]
125
+ elsif self.class._job_type_definitions[sub_job[:job].to_s]
126
+ find_matching_jobs(search_job, sub_job) { |item| yield item }
130
127
  end
131
128
  end
132
129
  end
@@ -163,7 +160,7 @@ module CanvasSync
163
160
 
164
161
  def get_chain_parameter(job_def, raise_error: true)
165
162
  unless _job_type_definitions[job_def[:job].to_s].present?
166
- raise "Job Type #{base_job[:job].to_s} does not accept a sub-chain" if raise_error
163
+ raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
167
164
  return nil
168
165
  end
169
166
 
@@ -0,0 +1,14 @@
1
+
2
+ local previously_scheduled = redis.call('HGET', KEYS[1], ARGV[1])
3
+ redis.call('HSET', KEYS[1], ARGV[1], 'true')
4
+
5
+ if previously_scheduled ~= 'true' then
6
+ local pcb_key = KEYS[1] + '-pending_callbacks'
7
+ redis.call('SADD', pcb_key, ARGV[1] + 'finalize')
8
+ if ARGV[2] == 'true' then
9
+ redis.call('SADD', pcb_key, ARGV[1])
10
+ end
11
+ redis.call('EXPIRE', pcb_key, ARGV[3])
12
+ end
13
+
14
+ return previously_scheduled
@@ -102,10 +102,10 @@ module CanvasSync
102
102
 
103
103
  if defined?(::Apartment)
104
104
  ::Apartment::Tenant.switch(job['apartment'] || 'public') do
105
- Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
105
+ ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
106
106
  end
107
107
  else
108
- Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
108
+ ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
109
109
  end
110
110
  end
111
111
  end
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.17.6".freeze
2
+ VERSION = "0.17.8.beta3".freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: canvas_sync
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.17.6
4
+ version: 0.17.8.beta3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nate Collings
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-12-17 00:00:00.000000000 Z
11
+ date: 2021-02-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -421,6 +421,7 @@ files:
421
421
  - lib/canvas_sync/job_batches/pool.rb
422
422
  - lib/canvas_sync/job_batches/redis_model.rb
423
423
  - lib/canvas_sync/job_batches/redis_script.rb
424
+ - lib/canvas_sync/job_batches/schedule_callback.lua
424
425
  - lib/canvas_sync/job_batches/sidekiq.rb
425
426
  - lib/canvas_sync/job_batches/sidekiq/web.rb
426
427
  - lib/canvas_sync/job_batches/sidekiq/web/helpers.rb
@@ -644,9 +645,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
644
645
  version: '0'
645
646
  required_rubygems_version: !ruby/object:Gem::Requirement
646
647
  requirements:
647
- - - ">="
648
+ - - ">"
648
649
  - !ruby/object:Gem::Version
649
- version: '0'
650
+ version: 1.3.1
650
651
  requirements: []
651
652
  rubygems_version: 3.0.3
652
653
  signing_key: