canvas_sync 0.17.6.beta1 → 0.17.8.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce407ae1dd566665cb8ae02856b61f033001cb25aadb903b5e9261846ad9e0c0
-  data.tar.gz: 349fa1f9e033101be253f2d5e5b8212f243a9ea28d143e683d6b2b6a877fcac6
+  metadata.gz: 0127c30973e69577ddf609684419422e2d0eb2d0c96b630158c93c9f16acaecf
+  data.tar.gz: 1ba81dc4c98e33f6ce6377c9d90e14f734820c343cea65f0e30fa94416eb0d1a
 SHA512:
-  metadata.gz: 32f2b3b973fdeec88c0a231112c3b82a275baa97937d3788b865d4eb50147a973022e77271970c3c58eab8a4ddbc4bdf10108d7f4f2df2f13c098d007d3bab67
-  data.tar.gz: 854ab10f0092ca6edfb280429c6cb600d8192a5c1189078ecca45a19ec3301c7e6afc3fdceec638ead82e17eab523e5669da010a42cd34761c7753dd06a0d7fb
+  metadata.gz: '0094f034d52be11a05de829e07ea76c62db5661bd30bf12aeba4897f2ce5fc2d95a4134c768c35874414414478256cb894d38da82f64e015c863dddb990f716b'
+  data.tar.gz: a2809d8663066739193b0507126d78140c235ade034b30f548c254fb0a8df4311ba1250a3c05ecb9a153d2e9983e52b3ba128a6b37370af122471294dd9bd5f1

data/lib/canvas_sync/job_batches/batch.rb

@@ -27,6 +27,7 @@ module CanvasSync
       delegate :redis, to: :class
 
       BID_EXPIRE_TTL = 2_592_000
+      SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
 
       attr_reader :bid
 
@@ -289,7 +290,7 @@ module CanvasSync
             end
           end
 
-          enqueue_callbacks(:dead, bid)
+          enqueue_callbacks(:death, bid)
         end
 
         def process_successful_job(bid, jid)
@@ -324,12 +325,12 @@ module CanvasSync
         def enqueue_callbacks(event, bid)
           batch_key = "BID-#{bid}"
           callback_key = "#{batch_key}-callbacks-#{event}"
-          already_processed, _, callbacks, queue, parent_bid, callback_params = redis do |r|
+
+          callbacks, queue, parent_bid, callback_params = redis do |r|
             return unless r.exists?(batch_key)
             return if r.hget(batch_key, 'keep_open') == 'true'
+
             r.multi do
-              r.hget(batch_key, event)
-              r.hset(batch_key, event, true)
               r.smembers(callback_key)
               r.hget(batch_key, "callback_queue")
               r.hget(batch_key, "parent_bid")
@@ -337,8 +338,6 @@ module CanvasSync
             end
           end
 
-          return if already_processed == 'true'
-
           queue ||= "default"
           parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
           callback_params = JSON.parse(callback_params) if callback_params.present?
@@ -348,14 +347,15 @@ module CanvasSync
           end
 
           opts = {"bid" => bid, "event" => event}
+          should_schedule_batch = callback_args.present? && !callback_params.present?
+          redis do |r|
+            already_processed = SCHEDULE_CALLBACK.call(r, [batch_key], [event.to_s, should_schedule_batch, BID_EXPIRE_TTL])
+          end
 
-          if callback_args.present? && !callback_params.present?
-            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+          return if already_processed == 'true'
 
-            redis do |r|
-              r.sadd("#{batch_key}-pending_callbacks", event)
-              r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
-            end
+          if should_schedule_batch
+            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
 
             with_batch(parent_bid) do
               cb_batch = self.new
@@ -367,7 +367,7 @@ module CanvasSync
 
               logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
               cb_batch.jobs do
-                push_callbacks callback_args, queue
+                push_callbacks(callback_args, queue)
               end
             end
           end

data/lib/canvas_sync/job_batches/callback.rb

@@ -4,7 +4,7 @@ module CanvasSync
       module Callback
         mattr_accessor :worker_class
 
-        VALID_CALLBACKS = %w[success complete dead].freeze
+        VALID_CALLBACKS = %w[success complete death].freeze
 
         module CallbackWorkerCommon
           def perform(definition, event, opts, bid, parent_bid)
@@ -60,8 +60,13 @@ module CanvasSync
             batch_status = Status.new bid
             send(event, bid, batch_status, batch_status.parent_bid)
 
+            Batch.redis do |r|
+              r.srem("BID-#{bid}-pending_callbacks", "#{event}-finalize")
+            end
+
             if event == :success
               if opts['origin'].present?
+                # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
                 origin_bid = opts['origin']['for_bid']
                 _, pending, success_ran = Batch.redis do |r|
                   r.multi do
@@ -72,7 +77,8 @@ module CanvasSync
                 end
                 Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
               end
-              if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
+
+              if (Batch.redis{|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
                 Batch.cleanup_redis(bid)
               end
             end
@@ -141,6 +147,12 @@ module CanvasSync
               end
             end
           end
+
+          def death(bid, status, parent_bid)
+            return unless parent_bid
+
+            Batch.enqueue_callbacks(:death, parent_bid)
+          end
         end
       end
     end

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -56,12 +56,11 @@ module CanvasSync
           placement = kwargs.keys[0]
           relative_to = kwargs.values[0]
 
-          matching_jobs = find_matching_jobs(relative_to)
+          matching_jobs = find_matching_jobs(relative_to).to_a
           raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
           raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
 
-          relative_job, sub_index = matching_jobs[0]
-          parent_job = find_parent_job(relative_job)
+          relative_job, parent_job, sub_index = matching_jobs[0]
           needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
 
           chain = self.class.get_chain_parameter(parent_job)
@@ -91,7 +90,7 @@ module CanvasSync
         raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
         return nil if matching_jobs.count == 0
 
-        new(matching_jobs[0])
+        new(matching_jobs[0][0])
       end
 
       def normalize!(job_def = self.base_job)
@@ -108,9 +107,7 @@ module CanvasSync
 
       # Legacy Support
       def merge_options(job, options)
-        matching_jobs = find_matching_jobs(job)
-
-        matching_jobs.each do |j|
+        find_matching_jobs(job).each do |j, parent, index|
           j[:options] ||= {}
           j[:options].deep_merge!(options)
         end
@@ -124,9 +121,9 @@ module CanvasSync
         sub_jobs = self.class.get_chain_parameter(parent_job)
         sub_jobs.each_with_index do |sub_job, i|
           if sub_job[:job].to_s == search_job.to_s
-            yield [parent_job, i]
-          elsif self.class._job_type_definitions[sub_job[:job]]
-            find_matching_jobs(search_job) { |item| yield item }
+            yield [sub_job, parent_job, i]
+          elsif self.class._job_type_definitions[sub_job[:job].to_s]
+            find_matching_jobs(search_job, sub_job) { |item| yield item }
           end
         end
       end
@@ -163,7 +160,7 @@ module CanvasSync
 
         def get_chain_parameter(job_def, raise_error: true)
           unless _job_type_definitions[job_def[:job].to_s].present?
-            raise "Job Type #{base_job[:job].to_s} does not accept a sub-chain" if raise_error
+            raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
             return nil
           end
 

data/lib/canvas_sync/job_batches/schedule_callback.lua

@@ -0,0 +1,14 @@
+
+local previously_scheduled = redis.call('HGET', KEYS[1], ARGV[1])
+redis.call('HSET', KEYS[1], ARGV[1], 'true')
+
+if previously_scheduled ~= 'true' then
+    local pcb_key = KEYS[1] + '-pending_callbacks'
+    redis.call('SADD', pcb_key, ARGV[1] + 'finalize')
+    if ARGV[2] == 'true' then
+        redis.call('SADD', pcb_key, ARGV[1])
+    end
+    redis.call('EXPIRE', pcb_key, ARGV[3])
+end
+
+return previously_scheduled

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -102,10 +102,10 @@ module CanvasSync
 
             if defined?(::Apartment)
               ::Apartment::Tenant.switch(job['apartment'] || 'public') do
-                Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+                ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
               end
             else
-              Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+              ::Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
             end
           end
         end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.6.beta1".freeze
+  VERSION = "0.17.8.beta2".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.17.6.beta1
+  version: 0.17.8.beta2
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-03 00:00:00.000000000 Z
+date: 2021-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -421,6 +421,7 @@ files:
 - lib/canvas_sync/job_batches/pool.rb
 - lib/canvas_sync/job_batches/redis_model.rb
 - lib/canvas_sync/job_batches/redis_script.rb
+- lib/canvas_sync/job_batches/schedule_callback.lua
 - lib/canvas_sync/job_batches/sidekiq.rb
 - lib/canvas_sync/job_batches/sidekiq/web.rb
 - lib/canvas_sync/job_batches/sidekiq/web/helpers.rb