canvas_sync 0.21.1 → 0.22.0.beta1

data/lib/canvas_sync/job_batches/batch.rb

@@ -1,9 +1,4 @@
 
-begin
-  require 'sidekiq'
-rescue LoadError
-end
-
 require_relative './redis_model'
 require_relative './redis_script'
 require_relative "./callback"
@@ -16,528 +11,563 @@ require_relative "./chain_builder"
 # Implement Job Batching similar to Sidekiq::Batch. Supports ActiveJob and Sidekiq, or a mix thereof.
 # Much of this code is modifed/extended from https://github.com/breamware/sidekiq-batch
 
-module CanvasSync
-  module JobBatches
-    CURRENT_BATCH_THREAD_KEY = :job_batches_batch
+module CanvasSync::JobBatches
+  CURRENT_BATCH_THREAD_KEY = :job_batches_batch
 
-    class Batch
-      include RedisModel
+  class Batch
+    include RedisModel
 
-      class NoBlockGivenError < StandardError; end
+    class NoBlockGivenError < StandardError; end
 
-      delegate :redis, to: :class
+    delegate :redis, to: :class
 
-      BID_EXPIRE_TTL = 90.days.to_i
-      INDEX_ALL_BATCHES = false
-      SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
-      BID_HIERARCHY = RedisScript.new(Pathname.new(__FILE__) + "../hier_batch_ids.lua")
+    BID_EXPIRE_TTL = 90.days.to_i
+    INDEX_ALL_BATCHES = false
+    SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
+    BID_HIERARCHY = RedisScript.new(Pathname.new(__FILE__) + "../hier_batch_ids.lua")
 
-      attr_reader :bid
+    attr_reader :bid
 
-      def self.current
-        Thread.current[CURRENT_BATCH_THREAD_KEY]
-      end
+    def self.current
+      Thread.current[CURRENT_BATCH_THREAD_KEY]
+    end
 
-      def self.current_context
-        self.current&.context
-      end
+    def self.current_context
+      self.current&.context
+    end
 
-      def initialize(existing_bid = nil)
-        @bid = existing_bid || SecureRandom.urlsafe_base64(10)
-        @existing = !(!existing_bid || existing_bid.empty?)  # Basically existing_bid.present?
-        @initialized = false
-        @bidkey = "BID-" + @bid.to_s
-        self.created_at = Time.now.utc.to_f unless @existing
-      end
+    def initialize(existing_bid = nil)
+      @bid = existing_bid || SecureRandom.urlsafe_base64(10)
+      @existing = !(!existing_bid || existing_bid.empty?)  # Basically existing_bid.present?
+      @initialized = false
+      @bidkey = "BID-" + @bid.to_s
+      self.created_at = Time.now.utc.to_f unless @existing
+    end
 
-      redis_attr :description
-      redis_attr :created_at
-      redis_attr :callback_queue, read_only: false
-      redis_attr :callback_params, :json
-      redis_attr :allow_context_changes, :bool
+    redis_attr :description
+    redis_attr :created_at
+    redis_attr :callback_queue, read_only: false
+    redis_attr :callback_params, :json
+    redis_attr :allow_context_changes, :bool
 
-      def context
-        return @context if defined?(@context)
+    def context
+      return @context if defined?(@context)
 
-        if (@initialized || @existing)
-          @context = ContextHash.new(bid)
-        else
-          @context = ContextHash.new(bid, {})
-        end
+      if (@initialized || @existing)
+        @context = ContextHash.new(bid)
+      else
+        @context = ContextHash.new(bid, {})
       end
+    end
 
-      def context=(value)
-        raise "context is read-only once the batch has been started" if (@initialized || @existing) # && !allow_context_changes
-        raise "context must be a Hash" unless value.is_a?(Hash) || value.nil?
-        return nil if value.nil? && @context.nil?
+    def context=(value)
+      raise "context is read-only once the batch has been started" if (@initialized || @existing) # && !allow_context_changes
+      raise "context must be a Hash" unless value.is_a?(Hash) || value.nil?
+      return nil if value.nil? && @context.nil?
 
-        value = {} if value.nil?
-        value = value.local if value.is_a?(ContextHash)
+      value = {} if value.nil?
+      value = value.local if value.is_a?(ContextHash)
 
-        @context ||= ContextHash.new(bid, {})
-        @context.set_local(value)
-        # persist_bid_attr('context', JSON.unparse(@context.local))
-      end
+      @context ||= ContextHash.new(bid, {})
+      @context.set_local(value)
+      # persist_bid_attr('context', JSON.unparse(@context.local))
+    end
 
-      def save_context_changes
-        @context&.save!
+    def save_context_changes
+      @context&.save!
+    end
+
+    # Events:
+    # :complete - triggered once all jobs have been executed, regardless of success or failure.
+    # :success - triggered once all jobs haves successfully executed.
+    # :death - triggered after any job enters the dead state.
+    # :stagnated - triggered when a job dies and no other jobs/batches are active.
+    def on(event, callback, options = {})
+      return unless Callback::VALID_CALLBACKS.include?(event.to_sym)
+
+      callback_key = "#{@bidkey}-callbacks-#{event}"
+      redis.multi do |r|
+        r.sadd(callback_key, JSON.unparse({
+          callback: callback,
+          opts: options
+        }))
+        r.expire(callback_key, BID_EXPIRE_TTL)
       end
+    end
+
+    def jobs
+      raise NoBlockGivenError unless block_given?
+
+      if !@existing && !@initialized
+        parent_bid = Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
 
-      def on(event, callback, options = {})
-        return unless Callback::VALID_CALLBACKS.include?(event.to_s)
-        callback_key = "#{@bidkey}-callbacks-#{event}"
         redis.multi do |r|
-          r.sadd(callback_key, JSON.unparse({
-            callback: callback,
-            opts: options
-          }))
-          r.expire(callback_key, BID_EXPIRE_TTL)
+          r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
+          r.expire(@bidkey, BID_EXPIRE_TTL)
+
+          if parent_bid
+            r.hincrby("BID-#{parent_bid}", "children", 1)
+            r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            r.zadd("BID-#{parent_bid}-bids", created_at, bid)
+          else
+            r.zadd("BID-ROOT-bids", created_at, bid)
+          end
         end
+
+        flush_pending_attrs
+        @context&.save!
+
+        @initialized = true
+      else
+        assert_batch_is_open
       end
 
-      def jobs
-        raise NoBlockGivenError unless block_given?
+      begin
+        parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
+        Thread.current[CURRENT_BATCH_THREAD_KEY] = self
+        yield
+      ensure
+        Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
+      end
 
-        if !@existing && !@initialized
-          parent_bid = Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
+      nil
+    end
 
-          redis.multi do |r|
-            r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
-            r.expire(@bidkey, BID_EXPIRE_TTL)
+    def increment_job_queue(jid)
+      assert_batch_is_open
+      append_jobs([jid])
+    end
 
-            if parent_bid
-              r.hincrby("BID-#{parent_bid}", "children", 1)
-              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
-              r.zadd("BID-#{parent_bid}-bids", created_at, bid)
-            else
-              r.zadd("BID-ROOT-bids", created_at, bid)
-            end
-          end
+    def invalidate_all
+      redis.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
+    end
 
-          flush_pending_attrs
-          @context&.save!
+    def parent_bid
+      redis.hget(@bidkey, "parent_bid")
+    end
 
-          @initialized = true
-        else
-          assert_batch_is_open
-        end
+    def parent
+      if parent_bid
+        Batch.new(parent_bid)
+      end
+    end
+
+    def valid?(batch = self)
+      valid = !redis.exists?("invalidated-bid-#{batch.bid}")
+      batch.parent ? valid && valid?(batch.parent) : valid
+    end
 
+    def keep_open!
+      if block_given?
         begin
-          parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
-          Thread.current[CURRENT_BATCH_THREAD_KEY] = self
+          keep_open!
           yield
         ensure
-          Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
+          let_close!
         end
-
-        nil
+      else
+        redis.hset(@bidkey, 'keep_open', "true")
       end
+    end
 
-      def increment_job_queue(jid)
-        assert_batch_is_open
-        append_jobs([jid])
-      end
+    def let_close!
+      _, failed, pending, children, complete, success = redis.multi do |r|
+        r.hset(@bidkey, 'keep_open', "false")
 
-      def invalidate_all
-        redis.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
+        r.scard("BID-#{bid}-failed")
+        r.hincrby("BID-#{bid}", "pending", 0)
+        r.hincrby("BID-#{bid}", "children", 0)
+        r.scard("BID-#{bid}-batches-complete")
+        r.scard("BID-#{bid}-batches-success")
       end
 
-      def parent_bid
-        redis.hget(@bidkey, "parent_bid")
+      all_success = pending.to_i.zero? && children == success
+      # if complete or successfull call complete callback (the complete callback may then call successful)
+      if (pending.to_i == failed.to_i && children == complete) || all_success
+        self.class.enqueue_callbacks(:complete, bid)
+        self.class.enqueue_callbacks(:success, bid) if all_success
       end
+    end
 
-      def parent
-        if parent_bid
-          Batch.new(parent_bid)
-        end
-      end
+    def self.with_batch(batch)
+      batch = self.new(batch) if batch.is_a?(String)
+      parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
+      Thread.current[CURRENT_BATCH_THREAD_KEY] = batch
+      yield
+    ensure
+      Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
+    end
 
-      def valid?(batch = self)
-        valid = !redis.exists?("invalidated-bid-#{batch.bid}")
-        batch.parent ? valid && valid?(batch.parent) : valid
-      end
+    # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
+    def self.without_batch(&blk)
+      with_batch(nil, &blk)
+    end
 
-      def keep_open!
-        if block_given?
-          begin
-            keep_open!
-            yield
-          ensure
-            let_close!
-          end
-        else
-          redis.hset(@bidkey, 'keep_open', "true")
-        end
-      end
+    protected
 
-      def let_close!
-        _, failed, pending, children, complete, success = redis.multi do |r|
-          r.hset(@bidkey, 'keep_open', "false")
+    def redis_key
+      @bidkey
+    end
 
-          r.scard("BID-#{bid}-failed")
-          r.hincrby("BID-#{bid}", "pending", 0)
-          r.hincrby("BID-#{bid}", "children", 0)
-          r.scard("BID-#{bid}-batches-complete")
-          r.scard("BID-#{bid}-batches-success")
-        end
+    def flush_pending_attrs
+      super
+      redis.zadd("batches", created_at, bid) if INDEX_ALL_BATCHES
+    end
 
-        all_success = pending.to_i.zero? && children == success
-        # if complete or successfull call complete callback (the complete callback may then call successful)
-        if (pending.to_i == failed.to_i && children == complete) || all_success
-          self.class.enqueue_callbacks(:complete, bid)
-          self.class.enqueue_callbacks(:success, bid) if all_success
-        end
-      end
+    private
 
-      def self.with_batch(batch)
-        batch = self.new(batch) if batch.is_a?(String)
-        parent = Thread.current[CURRENT_BATCH_THREAD_KEY]
-        Thread.current[CURRENT_BATCH_THREAD_KEY] = batch
-        yield
-      ensure
-        Thread.current[CURRENT_BATCH_THREAD_KEY] = parent
+    def assert_batch_is_open
+      unless defined?(@closed)
+        @closed = redis.hget(@bidkey, 'success') == 'true'
       end
+      raise "Cannot add jobs to Batch #{} bid - it has already entered the callback-stage" if @closed
+    end
 
-      # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
-      def self.without_batch(&blk)
-        with_batch(nil, &blk)
+    def append_jobs(jids)
+      jids = jids.uniq
+      return unless jids.size > 0
+
+      redis do |r|
+        tme = Time.now.utc.to_f
+        added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
+        r.multi do |r|
+          r.hincrby(@bidkey, "pending", added)
+          r.hincrby(@bidkey, "job_count", added)
+          r.expire(@bidkey, BID_EXPIRE_TTL)
+          r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
+        end
       end
+    end
 
-      protected
-
-      def redis_key
-        @bidkey
+    class << self
+      def current
+        Thread.current[CURRENT_BATCH_THREAD_KEY]
       end
 
-      def flush_pending_attrs
-        super
-        redis.zadd("batches", created_at, bid) if INDEX_ALL_BATCHES
+      def current_context
+        current&.context || {}
       end
 
-      private
+      # Perform a success/failure/complete/etc transaction against Redis, checking
+      #   if any callbacks should be triggered
+      def with_callback_check(bid, except: [], only: Callback::VALID_CALLBACKS, &blk)
+        except = Array(except).map(&:to_sym)
+        only = Array(only).map(&:to_sym)
 
-      def assert_batch_is_open
-        unless defined?(@closed)
-          @closed = redis.hget(@bidkey, 'success') == 'true'
-        end
-        raise "Cannot add jobs to Batch #{} bid - it has already entered the callback-stage" if @closed
-      end
+        precount = 0
 
-      def append_jobs(jids)
-        jids = jids.uniq
-        return unless jids.size > 0
+        all_results = redis do |r|
+          return unless r.exists?("BID-#{bid}")
 
-        redis do |r|
-          tme = Time.now.utc.to_f
-          added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
           r.multi do |r|
-            r.hincrby(@bidkey, "pending", added)
-            r.hincrby(@bidkey, "job_count", added)
-            r.expire(@bidkey, BID_EXPIRE_TTL)
-            r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
+            # The block probably doesn't _have_ to be part of the same transaction, but it was easier than
+            #   re-assessing possible race-conditions and is technically more performant
+            blk.call(r)
+            futures = r.instance_variable_get(:@futures) || r.instance_variable_get(:@pipeline)&.futures || r.instance_variable_get(:@client)&.futures
+            precount = futures.size
+
+            # Misc
+            r.hget("BID-#{bid}", "parent_bid")
+            r.hget("BID-#{bid}", "keep_open")
+
+            # Jobs
+            r.hincrby("BID-#{bid}", "pending", 0)
+            r.scard("BID-#{bid}-failed")
+            r.scard("BID-#{bid}-dead")
+
+            # Batches
+            r.hincrby("BID-#{bid}", "children", 0)
+            r.scard("BID-#{bid}-batches-complete")
+            r.scard("BID-#{bid}-batches-success")
+            r.scard("BID-#{bid}-batches-failed")
+            r.scard("BID-#{bid}-batches-stagnated")
+
+            # Touch Expirations
+            r.expire("BID-#{bid}", BID_EXPIRE_TTL)
+            r.expire("BID-#{bid}-batches-success", BID_EXPIRE_TTL)
           end
         end
-      end
 
-      class << self
-        def current
-          Thread.current[CURRENT_BATCH_THREAD_KEY]
-        end
+        # Exclude return values from the passed block
+        actual_results = all_results[precount..-1]
 
-        def current_context
-          current&.context || {}
-        end
+        # "pending" = not successful (yet)
+        # "failed" = dead or retrying
+        # "complete" = successful or failed
 
-        def process_failed_job(bid, jid)
-          _, pending, failed, children, complete, parent_bid = redis do |r|
-            return unless r.exists?("BID-#{bid}")
+        parent_bid, keep_open, \
+          pending_jobs, failed_jobs, dead_jobs, \
+          child_batches, complete_batches, success_batches, failed_batches, stagnated_batches \
+          = actual_results
 
-            r.multi do |r|
-              r.sadd("BID-#{bid}-failed", jid)
+        pending_batches = child_batches - success_batches
 
-              r.hincrby("BID-#{bid}", "pending", 0)
-              r.scard("BID-#{bid}-failed")
-              r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-batches-complete")
-              r.hget("BID-#{bid}", "parent_bid")
+        trigger_callback = ->(callback) {
+          next if except.include?(callback.to_sym)
+          next unless only.include?(callback.to_sym)
 
-              r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
-            end
-          end
+          Batch.logger.debug {"Finalize #{callback} bid: #{parent_bid}"}
+          enqueue_callbacks(callback, bid)
+        }
 
-          if pending.to_i == failed.to_i && children == complete
-            enqueue_callbacks(:complete, bid)
-          end
+        # Handling all of these cases in one method may be a little less performant than in more specialized methods, but
+        #   I was dealing with duplicate Redis queries and checks being made in up to 4 places and it was getting out of
+        #   hand - this combined method should be easier to maintain and reason about
+
+        all_successful = pending_jobs.zero? && child_batches == success_batches
+
+        if all_successful || (pending_jobs == failed_jobs && child_batches == complete_batches) # All Complete
+          trigger_callback.call(:complete)
         end
 
-        # Dead jobs are a Sidekiq feature.
-        # If this is called for a job, process_failed_job was also called
-        def process_dead_job(bid, jid)
-          _, dead_count = redis do |r|
-            return unless r.exists?("BID-#{bid}")
+        if all_successful # All Successfull
+          trigger_callback.call(:success)
+        elsif pending_jobs == dead_jobs && pending_batches == stagnated_batches # Stagnated
+          trigger_callback.call(:stagnated)
+        end
 
-            r.multi do |r|
-              r.sadd("BID-#{bid}-dead", jid)
-              r.scard("BID-#{bid}-dead")
-              r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
-            end
-          end
+        all_results[0...precount]
+      end
 
-          enqueue_callbacks(:death, bid)
+      def process_failed_job(bid, jid)
+        with_callback_check(bid, except: [:success]) do |r|
+          r.sadd("BID-#{bid}-failed", jid)
+          r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
         end
+      end
 
-        def process_successful_job(bid, jid)
-          _, failed, pending, children, complete, success, parent_bid, keep_open = redis do |r|
-            return unless r.exists?("BID-#{bid}")
+      # Dead jobs are a Sidekiq feature.
+      # If this is called for a job, process_failed_job was also called
+      def process_dead_job(bid, jid)
+        enqueue_callbacks(:death, bid)
 
-            r.multi do |r|
-              r.srem("BID-#{bid}-failed", jid)
+        with_callback_check(bid) do |r|
+          r.sadd("BID-#{bid}-dead", jid)
+          r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
+        end
+      end
 
-              r.scard("BID-#{bid}-failed")
-              r.hincrby("BID-#{bid}", "pending", -1)
-              r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-batches-complete")
-              r.scard("BID-#{bid}-batches-success")
-              r.hget("BID-#{bid}", "parent_bid")
-              r.hget("BID-#{bid}", "keep_open")
+      def process_successful_job(bid, jid)
+        with_callback_check(bid) do |r|
+          r.srem("BID-#{bid}-failed", jid)
+          r.hincrby("BID-#{bid}", "pending", -1)
+          r.hincrby("BID-#{bid}", "successful-jobs", 1)
+          r.zrem("BID-#{bid}-jids", jid)
+        end
+      end
 
-              r.hincrby("BID-#{bid}", "successful-jobs", 1)
-              r.zrem("BID-#{bid}-jids", jid)
-              r.expire("BID-#{bid}", BID_EXPIRE_TTL)
-            end
-          end
+      def enqueue_callbacks(event, bid)
+        batch_key = "BID-#{bid}"
+        callback_key = "#{batch_key}-callbacks-#{event}"
+
+        callbacks, queue, parent_bid, callback_params = redis do |r|
+          return unless r.exists?(batch_key)
+          return if r.hget(batch_key, 'keep_open') == 'true'
 
-          all_success = pending.to_i.zero? && children == success
-          # if complete or successfull call complete callback (the complete callback may then call successful)
-          if (pending.to_i == failed.to_i && children == complete) || all_success
-            enqueue_callbacks(:complete, bid)
-            enqueue_callbacks(:success, bid) if all_success
+          r.multi do |r|
+            r.smembers(callback_key)
+            r.hget(batch_key, "callback_queue")
+            r.hget(batch_key, "parent_bid")
+            r.hget(batch_key, "callback_params")
           end
         end
 
-        def enqueue_callbacks(event, bid)
-          batch_key = "BID-#{bid}"
-          callback_key = "#{batch_key}-callbacks-#{event}"
+        queue ||= "default"
+        parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
 
-          callbacks, queue, parent_bid, callback_params = redis do |r|
-            return unless r.exists?(batch_key)
-            return if r.hget(batch_key, 'keep_open') == 'true'
+        # Internal callback params. If this is present, we're trying to enqueue callbacks for a callback, which is a special case that
+        # indicates that the callback completed and we need to close the triggering batch (which is in a done-but-not-cleaned state)
+        callback_params = JSON.parse(callback_params) if callback_params.present?
 
-            r.multi do |r|
-              r.smembers(callback_key)
-              r.hget(batch_key, "callback_queue")
-              r.hget(batch_key, "parent_bid")
-              r.hget(batch_key, "callback_params")
-            end
-          end
+        # User-configured parameters/arguments to pass to the callback
+        callback_args = callbacks.reduce([]) do |memo, jcb|
+          cb = JSON.load(jcb)
+          memo << [cb['callback'], event.to_s, cb['opts'], bid, parent_bid]
+        end
 
-          queue ||= "default"
-          parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
+        opts = {"bid" => bid, "event" => event}
+        should_schedule_batch = callback_args.present? && !callback_params.present?
+        already_processed = redis do |r|
+          SCHEDULE_CALLBACK.call(r, [batch_key], [event.to_s, should_schedule_batch.to_s, BID_EXPIRE_TTL])
+        end
 
-          # Internal callback params. If this is present, we're trying to enqueue callbacks for a callback, which is a special case that
-          # indicates that the callback completed and we need to close the triggering batch (which is in a done-but-not-cleaned state)
-          callback_params = JSON.parse(callback_params) if callback_params.present?
+        return if already_processed == 'true'
 
-          # User-configured parameters/arguments to pass to the callback
-          callback_args = callbacks.reduce([]) do |memo, jcb|
-            cb = JSON.load(jcb)
-            memo << [cb['callback'], event.to_s, cb['opts'], bid, parent_bid]
-          end
+        if should_schedule_batch
+          logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
 
-          opts = {"bid" => bid, "event" => event}
-          should_schedule_batch = callback_args.present? && !callback_params.present?
-          already_processed = redis do |r|
-            SCHEDULE_CALLBACK.call(r, [batch_key], [event.to_s, should_schedule_batch.to_s, BID_EXPIRE_TTL])
-          end
+          # Create a new Batch to handle the callbacks and add it to the _parent_ batch
+          #   (this ensures that the parent's lifecycle status can't change until the child's callbacks are done)
+          with_batch(parent_bid) do
+            cb_batch = self.new
+            cb_batch.callback_params = {
+              for_bid: bid,
+              event: event,
+            }
+            opts['callback_bid'] = cb_batch.bid
 
-          return if already_processed == 'true'
-
-          if should_schedule_batch
-            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
-
-            # Create a new Batch to handle the callbacks and add it to the _parent_ batch
-            #   (this ensures that the parent's lifecycle status can't change until the child's callbacks are done)
-            with_batch(parent_bid) do
-              cb_batch = self.new
-              cb_batch.callback_params = {
-                for_bid: bid,
-                event: event,
-              }
-              opts['callback_bid'] = cb_batch.bid
-
-              logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
-              cb_batch.jobs do
-                push_callbacks(callback_args, queue)
-              end
+            logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
+            cb_batch.jobs do
+              push_callbacks(callback_args, queue)
             end
           end
+        end
 
-          if callback_params.present?
-            # This is a callback for a callback. Passing `origin` to the Finalizer allows it to also cleanup the original/callback-triggering batch
-            opts['origin'] = callback_params
-          end
-
-          # The Finalizer marks this batch as complete, bumps any necessary counters, cleans up this Batch _if_ no callbacks were scheduled,
-          #   and enqueues parent-Batch callbacks if needed.
-          logger.debug {"Run batch finalizer bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
-          finalizer = Batch::Callback::Finalize.new
-          status = Status.new bid
-          finalizer.dispatch(status, opts)
+        if callback_params.present?
+          # This is a callback for a callback. Passing `origin` to the Finalizer allows it to also cleanup the original/callback-triggering batch
+          opts['origin'] = callback_params
         end
 
-        def cleanup_redis(bid)
-          logger.debug {"Cleaning redis of batch #{bid}"}
-          redis do |r|
-            r.zrem("batches", bid)
-            r.zrem("BID-ROOT-bids", bid)
-            r.unlink(
-              "BID-#{bid}",
-              "BID-#{bid}-callbacks-complete",
-              "BID-#{bid}-callbacks-success",
-              "BID-#{bid}-failed",
-              "BID-#{bid}-dead",
-
-              "BID-#{bid}-batches-success",
-              "BID-#{bid}-batches-complete",
-              "BID-#{bid}-batches-failed",
-              "BID-#{bid}-bids",
-              "BID-#{bid}-jids",
-              "BID-#{bid}-pending_callbacks",
-            )
-          end
+        # The Finalizer marks this batch as complete, bumps any necessary counters, cleans up this Batch _if_ no callbacks were scheduled,
+        #   and enqueues parent-Batch callbacks if needed.
+        logger.debug {"Run batch finalizer bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+        finalizer = Batch::Callback::Finalize.new
+        status = Status.new bid
+        finalizer.dispatch(status, opts)
+      end
+
+      def cleanup_redis(bid)
+        logger.debug {"Cleaning redis of batch #{bid}"}
+        redis do |r|
+          r.zrem("batches", bid)
+          r.zrem("BID-ROOT-bids", bid)
+          r.unlink(
+            "BID-#{bid}",
+            "BID-#{bid}-callbacks-complete",
+            "BID-#{bid}-callbacks-success",
+            "BID-#{bid}-failed",
+            "BID-#{bid}-dead",
+
+            "BID-#{bid}-batches-success",
+            "BID-#{bid}-batches-complete",
+            "BID-#{bid}-batches-failed",
+            "BID-#{bid}-bids",
+            "BID-#{bid}-jids",
+            "BID-#{bid}-pending_callbacks",
+          )
         end
+      end
 
-        def delete_prematurely!(bid)
-          child_bids = redis do |r|
-            r.zrange("BID-#{bid}-bids", 0, -1)
-          end
-          child_bids.each do |cbid|
-            delete_prematurely!(cbid)
-          end
-          cleanup_redis(bid)
+      def delete_prematurely!(bid)
+        child_bids = redis do |r|
+          r.zrange("BID-#{bid}-bids", 0, -1)
+        end
+        child_bids.each do |cbid|
+          delete_prematurely!(cbid)
         end
+        cleanup_redis(bid)
+      end
 
-        # Internal method to cleanup a Redis Hash and related keys
-        def cleanup_redis_index_for(key, suffixes = [""])
-          if r.hget(k, "created_at").present?
-            r.multi do |r|
-              suffixes.each do |suffix|
-                r.expire(key + suffix, BID_EXPIRE_TTL)
-              end
+      # Internal method to cleanup a Redis Hash and related keys
+      def cleanup_redis_index_for(key, suffixes = [""])
+        if r.hget(k, "created_at").present?
+          r.multi do |r|
+            suffixes.each do |suffix|
+              r.expire(key + suffix, BID_EXPIRE_TTL)
             end
-            false
-          else
-            r.multi do |r|
-              suffixes.each do |suffix|
-                r.unlink(key + suffix)
-              end
+          end
+          false
+        else
+          r.multi do |r|
+            suffixes.each do |suffix|
+              r.unlink(key + suffix)
             end
-            true
           end
+          true
         end
+      end
 
-        # Administrative/console method to cleanup expired batches from the WebUI
-        def cleanup_redis_index!
-          suffixes = ["", "-callbacks-complete", "-callbacks-success", "-failed", "-dead", "-batches-success", "-batches-complete", "-batches-failed", "-bids", "-jids", "-pending_callbacks"]
+      # Administrative/console method to cleanup expired batches from the WebUI
+      def cleanup_redis_index!
+        suffixes = ["", "-callbacks-complete", "-callbacks-success", "-failed", "-dead", "-batches-success", "-batches-complete", "-batches-failed", "-bids", "-jids", "-pending_callbacks"]
 
-          cleanup_index = ->(index) {
-            r.zrangebyscore(index, "0", BID_EXPIRE_TTL.seconds.ago.to_i).each do |bid|
-              r.zrem(index, bid) if cleanup_redis_index_for("BID-#{bid}", suffixes)
-            end
-          }
+        cleanup_index = ->(index) {
+          r.zrangebyscore(index, "0", BID_EXPIRE_TTL.seconds.ago.to_i).each do |bid|
+            r.zrem(index, bid) if cleanup_redis_index_for("BID-#{bid}", suffixes)
+          end
+        }
 
-          cleanup_index.call("BID-ROOT-bids")
-          cleanup_index.call("batches")
-        end
+        cleanup_index.call("BID-ROOT-bids")
+        cleanup_index.call("batches")
+      end
 
-        def redis(&blk)
-          return RedisProxy.new unless block_given?
+      def redis(&blk)
+        return RedisProxy.new unless block_given?
 
-          if Thread.current[:job_batches_redis]
-            yield Thread.current[:job_batches_redis]
-          else
-            ::Bearcat.redis do |r|
-              Thread.current[:job_batches_redis] = r
-              yield r
-            ensure
-              Thread.current[:job_batches_redis] = nil
-            end
+        if Thread.current[:job_batches_redis]
+          yield Thread.current[:job_batches_redis]
+        else
+          ::Bearcat.redis do |r|
+            Thread.current[:job_batches_redis] = r
+            yield r
+          ensure
+            Thread.current[:job_batches_redis] = nil
           end
         end
+      end
 
-        def logger
-          ::CanvasSync.logger
-        end
+      def logger
+        ::CanvasSync.logger
+      end
 
-        def push_callbacks(args, queue)
-          Batch::Callback::worker_class.enqueue_all(args, queue)
-        end
+      def push_callbacks(args, queue)
+        Batch::Callback::worker_class.enqueue_all(args, queue)
+      end
 
-        def bid_hierarchy(bid, depth: 4, per_depth: 5, slice: nil)
-          args = [bid, depth, per_depth]
-          args << slice if slice
-          redis do |r|
-            BID_HIERARCHY.call(r, [], args)
-          end
+      def bid_hierarchy(bid, depth: 4, per_depth: 5, slice: nil)
+        args = [bid, depth, per_depth]
+        args << slice if slice
+        redis do |r|
+          BID_HIERARCHY.call(r, [], args)
         end
       end
+    end
 
-      class RedisProxy
-        def multi(*args, &block)
-          Batch.redis do |r|
-            r.multi(*args) do |r|
-              block.call(r)
-            end
+    class RedisProxy
+      def multi(*args, &block)
+        Batch.redis do |r|
+          r.multi(*args) do |r|
+            block.call(r)
           end
         end
+      end
 
-        def pipelined(*args, &block)
-          Batch.redis do |r|
-            r.pipelined(*args) do |r2|
-              block.call(r2 || r)
-            end
+      def pipelined(*args, &block)
+        Batch.redis do |r|
+          r.pipelined(*args) do |r2|
+            block.call(r2 || r)
           end
         end
+      end
 
-        def uget(key)
-          Batch.redis do |r|
-            case r.type(key)
-            when 'string'
-              r.get(key)
-            when 'list'
-              r.lrange(key, 0, -1)
-            when 'hash'
-              r.hgetall(key)
-            when 'set'
-              r.smembers(key)
-            when 'zset'
-              r.zrange(key, 0, -1)
-            end
+      def uget(key)
+        Batch.redis do |r|
+          case r.type(key)
+          when 'string'
+            r.get(key)
+          when 'list'
+            r.lrange(key, 0, -1)
+          when 'hash'
+            r.hgetall(key)
+          when 'set'
+            r.smembers(key)
+          when 'zset'
+            r.zrange(key, 0, -1)
           end
         end
+      end
 
-        def method_missing(method_name, *arguments, &block)
-          Batch.redis do |r|
-            r.send(method_name, *arguments, &block)
-          end
+      def method_missing(method_name, *arguments, &block)
+        Batch.redis do |r|
+          r.send(method_name, *arguments, &block)
         end
+      end
 
-        def respond_to_missing?(method_name, include_private = false)
-          super || Redis.method_defined?(method_name)
-        end
+      def respond_to_missing?(method_name, include_private = false)
+        super || Redis.method_defined?(method_name)
       end
     end
   end
 end
 
-# Automatically integrate with Sidekiq if it is present.
-if defined?(::Sidekiq)
-  require_relative './sidekiq'
-  CanvasSync::JobBatches::Sidekiq.configure
-end
-
-# Automatically integrate with ActiveJob if it is present.
-if defined?(::ActiveJob)
-  require_relative './active_job'
-  CanvasSync::JobBatches::ActiveJob.configure
-end
+require_relative './compat'