canvas_sync 0.17.7.beta1 → 0.17.8.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dce8b98794dc361c339eee16174fd5444df5e59b20a556de679d6308557a0285
-  data.tar.gz: 661f36b657c10a887cb8f1b7fc20af78e2790ed0ea08f2d1ead09d2a0118751e
+  metadata.gz: bf098a32ea7bd281cac2f9cd5a04940c9081bb01c77de5260dfe7fc76f762b7d
+  data.tar.gz: 45022a8e779828e05e448cae38661051684e1da480c6b1edd16c54304aceedbc
 SHA512:
-  metadata.gz: 8b952769ecfdb561ae22be39168614aa69a1d93eea3811ac4809e5b605a807887dc97db1af64acd140be0399b4a73f0d61076d645f0ebacee772370e27880d91
-  data.tar.gz: 60ccba0f231646e5b3634c16c525ca3fda187cf454ed91c0aabe93d2121a2a9a423c43fdac91019262a2129b1f3741af98fa3a6854b16e1a9e0445ff2c2c305c
+  metadata.gz: 6cd38061c1f217eb63904b62fc1e2d93baede56d81252f58667a65c83e04e358a76b0907b5817342cf4b1d3c7312a011c8039bf6dcddb4f6d78b11eb42a5b000
+  data.tar.gz: a0679d49f95dfeec40bb15f9416bb1083b8820e716b0ab96dd65840a0323e7b6d4d1e36c6286cf8c87c69b580c260b9f68a13b1b3f30a7140a85df9c1fabd8a1

data/lib/canvas_sync/job_batches/batch.rb

@@ -27,6 +27,7 @@ module CanvasSync
       delegate :redis, to: :class
 
       BID_EXPIRE_TTL = 2_592_000
+      SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
 
       attr_reader :bid
 
@@ -289,7 +290,7 @@ module CanvasSync
             end
           end
 
-          enqueue_callbacks(:dead, bid)
+          enqueue_callbacks(:death, bid)
         end
 
         def process_successful_job(bid, jid)
@@ -324,12 +325,12 @@ module CanvasSync
         def enqueue_callbacks(event, bid)
           batch_key = "BID-#{bid}"
           callback_key = "#{batch_key}-callbacks-#{event}"
-          already_processed, _, callbacks, queue, parent_bid, callback_params = redis do |r|
+
+          callbacks, queue, parent_bid, callback_params = redis do |r|
             return unless r.exists?(batch_key)
             return if r.hget(batch_key, 'keep_open') == 'true'
+
             r.multi do
-              r.hget(batch_key, event)
-              r.hset(batch_key, event, true)
               r.smembers(callback_key)
               r.hget(batch_key, "callback_queue")
               r.hget(batch_key, "parent_bid")
@@ -337,8 +338,6 @@ module CanvasSync
             end
           end
 
-          return if already_processed == 'true'
-
           queue ||= "default"
           parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
           callback_params = JSON.parse(callback_params) if callback_params.present?
@@ -348,14 +347,15 @@ module CanvasSync
           end
 
           opts = {"bid" => bid, "event" => event}
+          should_schedule_batch = callback_args.present? && !callback_params.present?
+          already_processed = redis do |r|
+            SCHEDULE_CALLBACK.call(r, [batch_key], [event.to_s, should_schedule_batch, BID_EXPIRE_TTL])
+          end
 
-          if callback_args.present? && !callback_params.present?
-            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+          return if already_processed == 'true'
 
-            redis do |r|
-              r.sadd("#{batch_key}-pending_callbacks", event)
-              r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
-            end
+          if should_schedule_batch
+            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
 
             with_batch(parent_bid) do
               cb_batch = self.new
@@ -367,7 +367,7 @@ module CanvasSync
 
               logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
               cb_batch.jobs do
-                push_callbacks callback_args, queue
+                push_callbacks(callback_args, queue)
               end
             end
           end

data/lib/canvas_sync/job_batches/callback.rb

@@ -4,7 +4,7 @@ module CanvasSync
       module Callback
         mattr_accessor :worker_class
 
-        VALID_CALLBACKS = %w[success complete dead].freeze
+        VALID_CALLBACKS = %w[success complete death].freeze
 
         module CallbackWorkerCommon
           def perform(definition, event, opts, bid, parent_bid)
@@ -60,8 +60,13 @@ module CanvasSync
             batch_status = Status.new bid
             send(event, bid, batch_status, batch_status.parent_bid)
 
+            Batch.redis do |r|
+              r.srem("BID-#{bid}-pending_callbacks", "#{event}-finalize")
+            end
+
             if event == :success
               if opts['origin'].present?
+                # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
                 origin_bid = opts['origin']['for_bid']
                 _, pending, success_ran = Batch.redis do |r|
                   r.multi do
@@ -72,6 +77,7 @@ module CanvasSync
                 end
                 Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
               end
+
               if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
                 Batch.cleanup_redis(bid)
               end
@@ -141,6 +147,12 @@ module CanvasSync
               end
             end
           end
+
+          def death(bid, status, parent_bid)
+            return unless parent_bid
+
+            Batch.enqueue_callbacks(:death, parent_bid)
+          end
         end
       end
     end

data/lib/canvas_sync/job_batches/schedule_callback.lua

@@ -0,0 +1,14 @@
+
+local previously_scheduled = redis.call('HGET', KEYS[1], ARGV[1])
+redis.call('HSET', KEYS[1], ARGV[1], 'true')
+
+if previously_scheduled ~= 'true' then
+    local pcb_key = KEYS[1] .. '-pending_callbacks'
+    redis.call('SADD', pcb_key, ARGV[1] .. '-finalize')
+    if ARGV[2] == 'true' then
+        redis.call('SADD', pcb_key, ARGV[1])
+    end
+    redis.call('EXPIRE', pcb_key, ARGV[3])
+end
+
+return previously_scheduled

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -102,10 +102,10 @@ module CanvasSync
 
             if defined?(::Apartment)
               ::Apartment::Tenant.switch(job['apartment'] || 'public') do
-                Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+                ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
               end
             else
-              Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+              ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
             end
           end
         end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.7.beta1".freeze
+  VERSION = "0.17.8.beta4".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.17.7.beta1
+  version: 0.17.8.beta4
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-11 00:00:00.000000000 Z
+date: 2021-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -421,6 +421,7 @@ files:
 - lib/canvas_sync/job_batches/pool.rb
 - lib/canvas_sync/job_batches/redis_model.rb
 - lib/canvas_sync/job_batches/redis_script.rb
+- lib/canvas_sync/job_batches/schedule_callback.lua
 - lib/canvas_sync/job_batches/sidekiq.rb
 - lib/canvas_sync/job_batches/sidekiq/web.rb
 - lib/canvas_sync/job_batches/sidekiq/web/helpers.rb