canvas_sync 0.17.7 → 0.17.8.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a95dc1f0ddf67f518f4b473bf277afeac14ec1b169385dc64bb5cb8e894b21bf
-  data.tar.gz: 4ea7587a023be51363a4c60d8b08f743c932fb9bfe0d6f7bd15a4c8e16777d78
+  metadata.gz: 207d529d702b1f403baa35df191839fb67468ca893aea969210dfe14cfc0060d
+  data.tar.gz: c613a717a96a4e4feffcd47ce19eb0dd3f0c8db109bfdb32f96c7f7bf72647a7
 SHA512:
-  metadata.gz: 783dff1463dd1371c9012a74f9f0b2cc776de6d305ebf53071b9ccd85b96dfea72b7dacd2430dfcd82ca2029b2071587187e964f814814f0736f74f7c2227a1d
-  data.tar.gz: 2e29a4225e913e572b4a11a5e356c3a9cdd718791d39e6565c27aa2a90e9417f256e8d1eda99edafcb6a7d0012cf187064f738b70fb57199362187e18334b586
+  metadata.gz: a65459a3ac20cd648d3184c052293a6d2c6c3d82a09b99156c3ad30e143ce57f414bd296c7ae8c3aa3f42b0cf31362d15da9e32a3dc8f1f1b05bd931b8d68166
+  data.tar.gz: 8261a9926c3a28bc5feedf781009374d998c05d8b35b9809ef9d3fabafb7e31fe77829d1c19d127a84289942b5d49a06f953cd41d1d95486aca61c772d1bda8c

data/lib/canvas_sync/job_batches/batch.rb

@@ -27,6 +27,7 @@ module CanvasSync
       delegate :redis, to: :class
 
       BID_EXPIRE_TTL = 2_592_000
+      SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
 
       attr_reader :bid
 
@@ -289,7 +290,7 @@ module CanvasSync
             end
           end
 
-          enqueue_callbacks(:dead, bid)
+          enqueue_callbacks(:death, bid)
         end
 
         def process_successful_job(bid, jid)
@@ -324,12 +325,12 @@ module CanvasSync
         def enqueue_callbacks(event, bid)
           batch_key = "BID-#{bid}"
           callback_key = "#{batch_key}-callbacks-#{event}"
-          already_processed, _, callbacks, queue, parent_bid, callback_params = redis do |r|
+
+          callbacks, queue, parent_bid, callback_params = redis do |r|
             return unless r.exists?(batch_key)
             return if r.hget(batch_key, 'keep_open') == 'true'
+
             r.multi do
-              r.hget(batch_key, event)
-              r.hset(batch_key, event, true)
               r.smembers(callback_key)
               r.hget(batch_key, "callback_queue")
               r.hget(batch_key, "parent_bid")
@@ -337,8 +338,6 @@ module CanvasSync
             end
           end
 
-          return if already_processed == 'true'
-
           queue ||= "default"
           parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
           callback_params = JSON.parse(callback_params) if callback_params.present?
@@ -348,14 +347,13 @@ module CanvasSync
           end
 
           opts = {"bid" => bid, "event" => event}
+          should_schedule_batch = callback_args.present? && !callback_params.present?
+          already_processed = SCHEDULE_CALLBACK.call(r, [batch_key], [event.to_s, should_schedule_batch, BID_EXPIRE_TTL])
 
-          if callback_args.present? && !callback_params.present?
-            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+          return if already_processed == 'true'
 
-            redis do |r|
-              r.sadd("#{batch_key}-pending_callbacks", event)
-              r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
-            end
+          if should_schedule_batch
+            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
 
             with_batch(parent_bid) do
               cb_batch = self.new
@@ -367,7 +365,7 @@ module CanvasSync
 
               logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
               cb_batch.jobs do
-                push_callbacks callback_args, queue
+                push_callbacks(callback_args, queue)
               end
             end
           end

data/lib/canvas_sync/job_batches/callback.rb

@@ -4,7 +4,7 @@ module CanvasSync
       module Callback
         mattr_accessor :worker_class
 
-        VALID_CALLBACKS = %w[success complete dead].freeze
+        VALID_CALLBACKS = %w[success complete death].freeze
 
         module CallbackWorkerCommon
           def perform(definition, event, opts, bid, parent_bid)
@@ -60,8 +60,13 @@ module CanvasSync
             batch_status = Status.new bid
             send(event, bid, batch_status, batch_status.parent_bid)
 
+            Batch.redis do |r|
+              r.srem("BID-#{bid}-pending_callbacks", "#{event}-finalize")
+            end
+
             if event == :success
               if opts['origin'].present?
+                # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
                 origin_bid = opts['origin']['for_bid']
                 _, pending, success_ran = Batch.redis do |r|
                   r.multi do
@@ -72,7 +77,8 @@ module CanvasSync
                 end
                 Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
               end
-              if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
+
+              if (Batch.redis{|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
                 Batch.cleanup_redis(bid)
               end
             end
@@ -141,6 +147,12 @@ module CanvasSync
               end
             end
           end
+
+          def death(bid, status, parent_bid)
+            return unless parent_bid
+
+            Batch.enqueue_callbacks(:death, parent_bid)
+          end
         end
       end
     end

data/lib/canvas_sync/job_batches/schedule_callback.lua

@@ -0,0 +1,14 @@
+
+local previously_scheduled = redis.call('HGET', KEYS[1], ARGV[1])
+redis.call('HSET', KEYS[1], ARGV[1], 'true')
+
+if previously_scheduled ~= 'true' then
+    local pcb_key = KEYS[1] + '-pending_callbacks'
+    redis.call('SADD', pcb_key, ARGV[1] + 'finalize')
+    if ARGV[2] == 'true' then
+        redis.call('SADD', pcb_key, ARGV[1])
+    end
+    redis.call('EXPIRE', pcb_key, ARGV[3])
+end
+
+return previously_scheduled

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -102,10 +102,10 @@ module CanvasSync
 
             if defined?(::Apartment)
               ::Apartment::Tenant.switch(job['apartment'] || 'public') do
-                Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+                ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
               end
             else
-              Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+              ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
             end
           end
         end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.7".freeze
+  VERSION = "0.17.8.beta1".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.17.7
+  version: 0.17.8.beta1
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-13 00:00:00.000000000 Z
+date: 2021-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -421,6 +421,7 @@ files:
 - lib/canvas_sync/job_batches/pool.rb
 - lib/canvas_sync/job_batches/redis_model.rb
 - lib/canvas_sync/job_batches/redis_script.rb
+- lib/canvas_sync/job_batches/schedule_callback.lua
 - lib/canvas_sync/job_batches/sidekiq.rb
 - lib/canvas_sync/job_batches/sidekiq/web.rb
 - lib/canvas_sync/job_batches/sidekiq/web/helpers.rb
@@ -644,9 +645,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: