canvas_sync 0.17.6 → 0.17.8.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06f6a868c0dd531a98b7999e7c132f75edb211856dd84ef5519e47b4634a0001
-  data.tar.gz: e305790ec6923cea6eabc085b67105b3ac6f66ea3f5eaf35083d3bfb552a5558
+  metadata.gz: 47913d27a72537ba9c33f5b282efd811dc0fb23d0114dabfdec52efa3eb062d0
+  data.tar.gz: ee5a090cfd2226868fc88324e767d7c61a6256d80ed0af35dd04e7967d76b602
 SHA512:
-  metadata.gz: fd34b80603ecaf90040446cb998e4ce7f33e4ed28c29436aee0ad5a017119d1187b0dbc7961f4300a542a97cef7af051428ba30ced733c68c085b609fcb4c225
-  data.tar.gz: e8227d369bbf5fab0f4bf8b4fcb90310a197986fa1a8f85c56689ab521ec9abfcc59ff6b4601b66a50742873aa1f98e969038f62ca1dca19a568e081a8f3e14a
+  metadata.gz: 46fcee4d05468452502ffb1660962e520e34f458f4d374c4d9e974c67364db574a7b3577f65016ddd77727e3131a97232466eda98b8855db11c164517df42bbb
+  data.tar.gz: 2fb317efd646549443680d8e9df41c2c2e228d4b8ea3a9d6e65f9be270b918d5423995571aaec72cb4616e083cc510cf5b0a63b0f3637d96e18c26ee7f7dd294

data/lib/canvas_sync/job_batches/batch.rb

@@ -27,6 +27,7 @@ module CanvasSync
       delegate :redis, to: :class
 
       BID_EXPIRE_TTL = 2_592_000
+      SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
 
       attr_reader :bid
 
@@ -289,7 +290,7 @@ module CanvasSync
             end
           end
 
-          enqueue_callbacks(:dead, bid)
+          enqueue_callbacks(:death, bid)
         end
 
         def process_successful_job(bid, jid)
@@ -324,12 +325,12 @@ module CanvasSync
         def enqueue_callbacks(event, bid)
           batch_key = "BID-#{bid}"
           callback_key = "#{batch_key}-callbacks-#{event}"
-          already_processed, _, callbacks, queue, parent_bid, callback_params = redis do |r|
+
+          callbacks, queue, parent_bid, callback_params = redis do |r|
             return unless r.exists?(batch_key)
             return if r.hget(batch_key, 'keep_open') == 'true'
+
             r.multi do
-              r.hget(batch_key, event)
-              r.hset(batch_key, event, true)
               r.smembers(callback_key)
               r.hget(batch_key, "callback_queue")
               r.hget(batch_key, "parent_bid")
@@ -337,8 +338,6 @@ module CanvasSync
             end
           end
 
-          return if already_processed == 'true'
-
           queue ||= "default"
           parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
           callback_params = JSON.parse(callback_params) if callback_params.present?
@@ -348,14 +347,15 @@ module CanvasSync
           end
 
           opts = {"bid" => bid, "event" => event}
+          should_schedule_batch = callback_args.present? && !callback_params.present?
+          already_processed = redis do |r|
+            SCHEDULE_CALLBACK.call(r, [batch_key], [event.to_s, should_schedule_batch, BID_EXPIRE_TTL])
+          end
 
-          if callback_args.present? && !callback_params.present?
-            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+          return if already_processed == 'true'
 
-            redis do |r|
-              r.sadd("#{batch_key}-pending_callbacks", event)
-              r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
-            end
+          if should_schedule_batch
+            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
 
             with_batch(parent_bid) do
               cb_batch = self.new
@@ -367,7 +367,7 @@ module CanvasSync
 
               logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
               cb_batch.jobs do
-                push_callbacks callback_args, queue
+                push_callbacks(callback_args, queue)
               end
             end
           end

data/lib/canvas_sync/job_batches/callback.rb

@@ -4,7 +4,7 @@ module CanvasSync
       module Callback
         mattr_accessor :worker_class
 
-        VALID_CALLBACKS = %w[success complete dead].freeze
+        VALID_CALLBACKS = %w[success complete death].freeze
 
         module CallbackWorkerCommon
           def perform(definition, event, opts, bid, parent_bid)
@@ -60,8 +60,13 @@ module CanvasSync
             batch_status = Status.new bid
             send(event, bid, batch_status, batch_status.parent_bid)
 
+            Batch.redis do |r|
+              r.srem("BID-#{bid}-pending_callbacks", "#{event}-finalize")
+            end
+
             if event == :success
               if opts['origin'].present?
+                # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
                 origin_bid = opts['origin']['for_bid']
                 _, pending, success_ran = Batch.redis do |r|
                   r.multi do
@@ -72,7 +77,8 @@ module CanvasSync
                 end
                 Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
               end
-              if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
+
+              if (Batch.redis{|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
                 Batch.cleanup_redis(bid)
               end
             end
@@ -141,6 +147,12 @@ module CanvasSync
               end
             end
           end
+
+          def death(bid, status, parent_bid)
+            return unless parent_bid
+
+            Batch.enqueue_callbacks(:death, parent_bid)
+          end
         end
       end
     end

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -56,12 +56,11 @@ module CanvasSync
           placement = kwargs.keys[0]
           relative_to = kwargs.values[0]
 
-          matching_jobs = find_matching_jobs(relative_to)
+          matching_jobs = find_matching_jobs(relative_to).to_a
           raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
           raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
 
-          relative_job, sub_index = matching_jobs[0]
-          parent_job = find_parent_job(relative_job)
+          relative_job, parent_job, sub_index = matching_jobs[0]
           needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
 
           chain = self.class.get_chain_parameter(parent_job)
@@ -91,7 +90,7 @@ module CanvasSync
         raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
         return nil if matching_jobs.count == 0
 
-        new(matching_jobs[0])
+        new(matching_jobs[0][0])
       end
 
       def normalize!(job_def = self.base_job)
@@ -108,9 +107,7 @@ module CanvasSync
 
       # Legacy Support
       def merge_options(job, options)
-        matching_jobs = find_matching_jobs(job)
-
-        matching_jobs.each do |j|
+        find_matching_jobs(job).each do |j, parent, index|
           j[:options] ||= {}
           j[:options].deep_merge!(options)
         end
@@ -124,9 +121,9 @@ module CanvasSync
         sub_jobs = self.class.get_chain_parameter(parent_job)
         sub_jobs.each_with_index do |sub_job, i|
           if sub_job[:job].to_s == search_job.to_s
-            yield [parent_job, i]
-          elsif self.class._job_type_definitions[sub_job[:job]]
-            find_matching_jobs(search_job) { |item| yield item }
+            yield [sub_job, parent_job, i]
+          elsif self.class._job_type_definitions[sub_job[:job].to_s]
+            find_matching_jobs(search_job, sub_job) { |item| yield item }
           end
         end
       end
@@ -163,7 +160,7 @@ module CanvasSync
 
         def get_chain_parameter(job_def, raise_error: true)
           unless _job_type_definitions[job_def[:job].to_s].present?
-            raise "Job Type #{base_job[:job].to_s} does not accept a sub-chain" if raise_error
+            raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
             return nil
           end
 

data/lib/canvas_sync/job_batches/schedule_callback.lua

@@ -0,0 +1,14 @@
+
+local previously_scheduled = redis.call('HGET', KEYS[1], ARGV[1])
+redis.call('HSET', KEYS[1], ARGV[1], 'true')
+
+if previously_scheduled ~= 'true' then
+    local pcb_key = KEYS[1] + '-pending_callbacks'
+    redis.call('SADD', pcb_key, ARGV[1] + 'finalize')
+    if ARGV[2] == 'true' then
+        redis.call('SADD', pcb_key, ARGV[1])
+    end
+    redis.call('EXPIRE', pcb_key, ARGV[3])
+end
+
+return previously_scheduled

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -102,10 +102,10 @@ module CanvasSync
 
             if defined?(::Apartment)
               ::Apartment::Tenant.switch(job['apartment'] || 'public') do
-                Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+                ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
               end
             else
-              Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+              ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
             end
           end
         end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.6".freeze
+  VERSION = "0.17.8.beta3".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.17.6
+  version: 0.17.8.beta3
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-17 00:00:00.000000000 Z
+date: 2021-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -421,6 +421,7 @@ files:
 - lib/canvas_sync/job_batches/pool.rb
 - lib/canvas_sync/job_batches/redis_model.rb
 - lib/canvas_sync/job_batches/redis_script.rb
+- lib/canvas_sync/job_batches/schedule_callback.lua
 - lib/canvas_sync/job_batches/sidekiq.rb
 - lib/canvas_sync/job_batches/sidekiq/web.rb
 - lib/canvas_sync/job_batches/sidekiq/web/helpers.rb
@@ -644,9 +645,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: