canvas_sync 0.18.12 → 0.19.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1223506dbbaaa180f70770aaabd92c3ffe931ae491bd392b2bbaa2ce520de2c8
-  data.tar.gz: b0b5bc307fe639c3281f51658e995e2fc661186cd5b73918027e0fe01f7487e6
+  metadata.gz: bc839ab64ec8076616ca77afa2d66f727ed48e0a1ee6ffd10b8a843f32444c03
+  data.tar.gz: 2f2706695832f1c0ca6a11708695cba06019947109d3a4cb4e2af4dc4748b488
 SHA512:
-  metadata.gz: 40d9c62d733a904611600f4fe6d37e0eecf956d835fb0db31d10e40af8d0bed2269a72b386ae9135a2b8dc05219df80002497cf19ab88d6e579be81d80180c42
-  data.tar.gz: 659c99a14af5f186b3df3522642fec00b6ce7e3337a01e23e11f8c261208065834b9754d6d619b7445209dd2ed8a0eddbaabdc257a90038399a00de48eae88a1
+  metadata.gz: ba8d9abc64bd0df6fa2eb59c018ce265f85625fbbe5b83a82b53def9c0e731a6863a425e52bfac0c36dd435d91d1ec067a4a25ed199dd2302261f2a185fb1e14
+  data.tar.gz: 16935770f2023e2e6892a5f3fe1fd05473855a9fb45d459d87b382f618500200fdaf9578457904388c0e074a7ab26acd977f0c7f6ea8a518f74b6d78d810aa37

data/README.md

@@ -128,13 +128,12 @@ chain = CanvasSync.default_provisioning_report_chain(
 )
 
 # Add a custom job to the end of the chain.
-chain << { job: CanvasSyncCompleteWorker, parameters: [{ job_id: job.id }] }
-chain << { job: CanvasSyncCompleteWorker, options: { job_id: job.id } } # If an options key is provided, it will be automatically appended to the end of the :parameters array
+chain << { job: CanvasSyncCompleteWorker, args: [job.id], kwargs: { job_id: job.id } }
 
 chain.process!
 
 # The chain object provides a fairly extensive API:
-chain.insert({ job: SomeOtherJob }) # Adds the job to the end of the chain
+chain.insert({ job: SomeOtherJob, args: [], kwargs: {} }) # Adds the job to the end of the chain
 chain.insert_at(0, { job: SomeOtherJob }) # Adds the job to the beginning of the chain
 chain.insert({ job: SomeOtherJob }, after: 'CanvasSync::Jobs::SyncTermsJob') # Adds the job right after the SyncTermsJob
 chain.insert({ job: SomeOtherJob }, before: 'CanvasSync::Jobs::SyncTermsJob') # Adds the job right before the SyncTermsJob
@@ -163,7 +162,7 @@ end
 
 You must implement a job that will enqueue a report starter for your report. (TODO: would be nice to make some sort of builder for this, so you just define the report and its params and then the gem runs it in a pre-defined job.)
 
-Let's say we have a custom Canvas report called "my_really_cool_report_csv". First, we would need to create a job class that will enqueue a report starter. To work with the `CanvasSync` interface, your class must accept 2 parameters: `job_chain`, and `options`.
+Let's say we have a custom Canvas report called "my_really_cool_report_csv". First, we would need to create a job class that will enqueue a report starter.
 
 ```ruby
 class MyReallyCoolReportJob < CanvasSync::Jobs::ReportStarter
@@ -309,7 +308,8 @@ pool_id = pool.pid
 # Add a job to the pool
 pool << {
   job: SomeJob, # The Class of a ActiveJob Job or Sidekiq Worker
-  parameters: [1, 2, 3], # Array of params to pass th e Job
+  args: [1, 2, 3], # Array of params to pass th e Job
+  kwargs: {},
   priority: 100, # Only effective if order=:priority, higher is higher
 }
 
@@ -317,7 +317,8 @@ pool << {
 pool.add_jobs([
   {
     job: SomeJob, # The Class of a ActiveJob Job or Sidekiq Worker
-    parameters: [1, 2, 3], # Array of params to pass th e Job
+    args: [1, 2, 3], # Array of params to pass th e Job
+    kwargs: {},
     priority: 100, # Only effective if order=:priority, higher is higher
   },
   # ...
@@ -367,7 +368,7 @@ If you want your own jobs to also log to the table all you have to do is have yo
 
 If you want to be able to utilize the `CanvasSync::JobLog` without `ActiveJob` (so you can get access to `Sidekiq` features that `ActiveJob` doesn't support), then add the following to an initializer in your Rails app:
 
-```
+```ruby
 Sidekiq.configure_server do |config|
   config.server_middleware do |chain|
     chain.add CanvasSync::Sidekiq::Middleware
@@ -397,7 +398,7 @@ chain.process!
 
 You can configure CanvasSync settings by doing the following:
 
-```
+```ruby
 CanvasSync.configure do |config|
   config.classes_to_only_log_errors_on << "ClassToOnlyLogErrorsOn"
 end
@@ -412,7 +413,7 @@ You can pass in global_options to a job chain.  Global options are added to the
 various internal processes.
 
 Pass global options into a job chain, using the options param nested in a :global key.
-options: { global: {...} }
+`options: { global: {...} }`
 
 report_timeout (integer):             Number of days until a Canvas report should timeout.  Default is 1.
 report_compilation_timeout (integer): Number of days until a Canvas report should timeout.  Default is 1 hour.
@@ -421,12 +422,14 @@ report_max_tries (integer):           The number of times to attempt a report be
                                       if it has an 'error' status in Canvas or is deleted.
 
 This is an example job chain with global options:
+```ruby
 job_chain = CanvasSync.default_provisioning_report_chain(
   MODELS_TO_SYNC,
   term_scope: :active,
   full_sync_every: 'sunday',
   options: { global: { report_timeout: 2 } }
 )
+```
 
 ## Handling Job errors
 

data/lib/canvas_sync/concerns/ability_helper.rb

@@ -35,6 +35,23 @@ module CanvasSync::Concerns
       labels.is_a?(String) ? labels.split(',') : []
     end
 
+    def canvas_account_role_labels(account = 'self')
+      account = 'self' if account.to_s == "root"
+      account = account.canvas_id if account.respond_to?(:canvas_id)
+
+      if "::Admin".safe_constantize && ::Admin < ::ActiveRecord::Base
+        account = current_organization.canvas_account_id if account == 'self'
+        adm_query = ::Admin.where(canvas_account_id: account, workflow_state: "active")
+        adm_query.pluck(:role_name)
+      else
+        Rails.cache.fetch([self.class.name, "AccountAdminLinks", account, canvas_user_id], expires_in: 1.hour) do
+          admin_entries = canvas_sync_client.account_admins(account, user_id: [canvas_user_id])
+          admin_entries = admin_entries.select{|ent| ent[:workflow_state] == 'active' }
+          admin_entries.map{|ent| ent[:role] }
+        end
+      end
+    end
+
     def lti_roles
       @lti_roles ||= RoleStore.new(launch_params["https://purl.imsglobal.org/spec/lti/claim/roles"] || launch_params['ext_roles'] || '')
     end
@@ -148,16 +165,7 @@ module CanvasSync::Concerns
     end
 
     def canvas_root_account_roles
-      role_labels = if "::Admin".safe_constantize && ::Admin < ::ActiveRecord::Base
-        adm_query = ::Admin.where(canvas_account_id: current_organization.canvas_account_id, workflow_state: "active")
-        adm_query.pluck(:role_name)
-      else
-        Rails.cache.fetch([self.class.name, "RootAccountAdminLinks", canvas_user_id], expires_in: 1.hour) do
-          admin_entries = canvas_sync_client.account_admins('self', user_id: [canvas_user_id])
-          admin_entries = admin_entries.select{|ent| ent[:workflow_state] == 'active' }
-          admin_entries.map{|ent| ent[:role] }
-        end
-      end
+      role_labels = canvas_account_role_labels('self')
       ::Role.for_labels(role_labels, ::Account.find_by(canvas_parent_account_id: nil))
     end
 
@@ -171,7 +179,7 @@ module CanvasSync::Concerns
 
     def canvas_super_user?
       cache_on_session(:canvas_super_user?) do
-        canvas_site_admin? || canvas_root_account_roles.map(&:label).include?("Account Admin")
+        canvas_site_admin? || canvas_account_role_labels(:root).include?("Account Admin")
       end
     end
 

data/lib/canvas_sync/concerns/account/base.rb

@@ -0,0 +1,15 @@
+module CanvasSync::Concerns
+  module Account
+    module Base
+      extend ActiveSupport::Concern
+
+      CanvasSync::Record.define_feature self, default: true
+
+      class_methods do
+        def root_account
+          where(canvas_parent_account_id: nil).last
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/batch.rb

@@ -28,6 +28,7 @@ module CanvasSync
       delegate :redis, to: :class
 
       BID_EXPIRE_TTL = 90.days.to_i
+      INDEX_ALL_BATCHES = false
       SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
       BID_HIERARCHY = RedisScript.new(Pathname.new(__FILE__) + "../hier_batch_ids.lua")
 
@@ -202,7 +203,7 @@ module CanvasSync
 
       def flush_pending_attrs
         super
-        redis.zadd("batches", created_at, bid)
+        redis.zadd("batches", created_at, bid) if INDEX_ALL_BATCHES
       end
 
       private
@@ -323,8 +324,13 @@ module CanvasSync
           end
 
           queue ||= "default"
-          parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
+          parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank?
+
+          # Internal callback params. If this is present, we're trying to enqueue callbacks for a callback, which is a special case that
+          # indicates that the callback completed and we need to close the triggering batch (which is in a done-but-not-cleaned state)
           callback_params = JSON.parse(callback_params) if callback_params.present?
+
+          # User-configured parameters/arguments to pass to the callback
           callback_args = callbacks.reduce([]) do |memo, jcb|
             cb = JSON.load(jcb)
             memo << [cb['callback'], event.to_s, cb['opts'], bid, parent_bid]
@@ -341,6 +347,8 @@ module CanvasSync
           if should_schedule_batch
             logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
 
+            # Create a new Batch to handle the callbacks and add it to the _parent_ batch
+            #   (this ensures that the parent's lifecycle status can't change until the child's callbacks are done)
             with_batch(parent_bid) do
               cb_batch = self.new
               cb_batch.callback_params = {
@@ -357,9 +365,12 @@ module CanvasSync
           end
 
           if callback_params.present?
+            # This is a callback for a callback. Passing `origin` to the Finalizer allows it to also cleanup the original/callback-triggering batch
             opts['origin'] = callback_params
           end
 
+          # The Finalizer marks this batch as complete, bumps any necessary counters, cleans up this Batch _if_ no callbacks were scheduled,
+          #   and enqueues parent-Batch callbacks if needed.
           logger.debug {"Run batch finalizer bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
           finalizer = Batch::Callback::Finalize.new
           status = Status.new bid
@@ -398,6 +409,39 @@ module CanvasSync
           cleanup_redis(bid)
         end
 
+        # Internal method to cleanup a Redis Hash and related keys
+        def cleanup_redis_index_for(key, suffixes = [""])
+          if r.hget(k, "created_at").present?
+            r.multi do |r|
+              suffixes.each do |suffix|
+                r.expire(key + suffix, BID_EXPIRE_TTL)
+              end
+            end
+            false
+          else
+            r.multi do |r|
+              suffixes.each do |suffix|
+                r.unlink(key + suffix)
+              end
+            end
+            true
+          end
+        end
+
+        # Administrative/console method to cleanup expired batches from the WebUI
+        def cleanup_redis_index!
+          suffixes = ["", "-callbacks-complete", "-callbacks-success", "-failed", "-dead", "-batches-success", "-batches-complete", "-batches-failed", "-bids", "-jids", "-pending_callbacks"]
+
+          cleanup_index = ->(index) {
+            r.zrangebyscore(index, "0", BID_EXPIRE_TTL.seconds.ago.to_i).each do |bid|
+              r.zrem(index, bid) if cleanup_redis_index_for("BID-#{bid}", suffixes)
+            end
+          }
+
+          cleanup_index.call("BID-ROOT-bids")
+          cleanup_index.call("batches")
+        end
+
         def redis(&blk)
           return RedisProxy.new unless block_given?
 

data/lib/canvas_sync/job_batches/callback.rb

@@ -112,10 +112,13 @@ module CanvasSync
             end
 
             if parent_bid && !(pending.to_i.zero? && children == success)
-              # if batch was not successfull check and see if its parent is complete
-              # if the parent is complete we trigger the complete callback
-              # We don't want to run this if the batch was successfull because the success
-              # callback may add more jobs to the parent batch
+              # If batch was not successfull check and see if its parent is complete
+              # if the parent is complete we trigger its complete callback.
+              #
+              # Otherwise, we don't want to to trigger the parent's :complete here (and
+              # instead opt to have success tigger parent :complete) - this
+              # allows the success callback to add additional jobs to the parent batch
+              # before triggering :complete.
 
               Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
               _, _, complete, pending, children, failure = Batch.redis do |r|

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -8,12 +8,13 @@ module CanvasSync
       def initialize(base_type = SerialBatchJob)
         if base_type.is_a?(Hash)
           @base_job = base_type
+          @base_job[:args] ||= @base_job[:parameters] || []
+          @base_job[:kwargs] ||= {}
         else
-          @base_job = {
-            job: base_type,
-            parameters: [],
-          }
+          @base_job = build_job_hash(base_type)
         end
+
+        self.class.get_chain_parameter(base_job)
       end
 
       def process!
@@ -25,23 +26,25 @@ module CanvasSync
         if key.is_a?(Class)
           get_sub_chain(key)
         else
+          # Legacy Support
+          key = :args if key == :parameters
+
           @base_job[key]
         end
       end
 
-      def params
-        ParamsMapper.new(self[:parameters])
-      end
+      def args; return self[:args]; end
+      def kwargs; return self[:kwargs]; end
 
       def <<(new_job)
         insert_at(-1, new_job)
       end
 
-      def insert_at(position, new_jobs, *args, &blk)
+      def insert_at(position, new_jobs, *args, **kwargs, &blk)
         chain = self.class.get_chain_parameter(base_job)
         if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
-          new_jobs = build_job_hash(new_jobs, *args, &blk)
-        elsif args.length > 0
+          new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
+        elsif args.count > 0 || kwargs.count > 0
           raise "Unexpected number of arguments"
         end
         new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
@@ -51,8 +54,7 @@ module CanvasSync
       def insert(new_jobs, *args, **kwargs, &blk)
         if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
           job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
-          args << job_kwargs if job_kwargs.present?
-          new_jobs = build_job_hash(new_jobs, *args, &blk)
+          new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
           kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
         else
           invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
@@ -124,14 +126,6 @@ module CanvasSync
         end
       end
 
-      # Legacy Support
-      def merge_options(job, options)
-        find_matching_jobs(job).each do |j, parent, index|
-          j[:options] ||= {}
-          j[:options].deep_merge!(options)
-        end
-      end
-
       def apply_block(&blk)
         return unless blk.present?
         instance_exec(&blk)
@@ -139,10 +133,11 @@ module CanvasSync
 
       private
 
-      def build_job_hash(job, *params, &blk)
+      def build_job_hash(job, args: [], kwargs: {}, &blk)
         hsh = {
           job: job,
-          parameters: params,
+          args: args,
+          kwargs: kwargs,
         }
         self.class.new(hsh).apply_block(&blk) if blk.present?
         hsh
@@ -180,9 +175,17 @@ module CanvasSync
       end
 
       class << self
-        def build(job, *args, &blk)
+        # Support builder syntaxt/DSL
+        # Chain.build(ConcurrentBatchJob) do
+        #   insert(SomeJob, arg1, kwarg: 1)
+        #   insert(SerialBatchJob) do
+        #     insert(SomeJob, arg1, kwarg: 1)
+        #   end
+        # end
+        def build(job, *args, **kwargs, &blk)
           new(job).tap do |ch|
-            ch[:parameters] = args
+            ch.base_job[:args] = args
+            ch.base_job[:kwargs] = kwargs
             ch.apply_block(&blk)
           end
         end
@@ -205,8 +208,11 @@ module CanvasSync
           end
 
           key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
-          mapper = ParamsMapper.new(job_def[:parameters])
-          mapper[key] ||= []
+          if key.is_a?(Numeric)
+            job_def[:args][key] ||= []
+          else
+            job_def[:kwargs][key] ||= []
+          end
         end
 
         # TODO: Add a Chain progress web View
@@ -219,18 +225,19 @@ module CanvasSync
         #   > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
         def enqueue_job(job_def)
           job_class = job_def[:job].constantize
-          job_options = job_def[:parameters] || []
+          job_args = job_def[:args] || job_def[:parameters] || []
+          job_kwargs = job_def[:kwargs] || {}
 
           # Legacy Support
           if job_def[:options]
-            job_options << {} unless job_options[-1].is_a?(Hash)
-            job_options[-1].merge!(job_def[:options])
+            job_args << {} unless job_args[-1].is_a?(Hash)
+            job_args[-1].merge!(job_def[:options])
           end
 
           if job_class.respond_to? :perform_async
-            job_class.perform_async(*job_options)
+            job_class.perform_async(*job_args, **job_kwargs)
           else
-            job_class.perform_later(*job_options)
+            job_class.perform_later(*job_args, **job_kwargs)
           end
         end
       end
@@ -238,48 +245,5 @@ module CanvasSync
 
     ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
     ChainBuilder.register_chain_job(SerialBatchJob, 0)
-
-    class ParamsMapper
-      def initialize(backend)
-        @backend = backend
-      end
-
-      def [](key)
-        get_parameter(key)
-      end
-
-      def []=(key, value)
-        set_parameter(key, value)
-      end
-
-      def to_a
-        @backend
-      end
-
-      private
-
-      def get_parameter(key)
-        if key.is_a?(Numeric)
-          @backend[key]
-        else
-          kwargs = @backend.last
-          return nil unless kwargs.is_a?(Hash)
-          kwargs[key]
-        end
-      end
-
-      def set_parameter(key, value)
-        if key.is_a?(Numeric)
-          @backend[key] = value
-        else
-          kwargs = @backend.last
-          unless kwargs.is_a?(Hash)
-            kwargs = {}
-            @backend.push(kwargs)
-          end
-          kwargs[key] = value
-        end
-      end
-    end
   end
 end

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -3,19 +3,19 @@ require_relative './base_job'
 module CanvasSync
   module JobBatches
     class ConcurrentBatchJob < BaseJob
-      def self.make_batch(sub_jobs, context: nil, &blk)
+      def self.make_batch(sub_jobs, **kwargs, &blk)
         ManagedBatchJob.make_batch(
           sub_jobs,
+          **kwargs,
           ordered: false,
           concurrency: true,
-          context: context,
-          desc_prefix: 'ConcurrentBatchJob',
+          desc_prefix: 'ConcurrentBatchJob: ',
           &blk
         )
       end
 
-      def perform(sub_jobs, context: nil)
-        self.class.make_batch(sub_jobs, context: context)
+      def perform(sub_jobs, **kwargs)
+        self.class.make_batch(sub_jobs, **kwargs)
       end
     end
   end

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -3,7 +3,7 @@ require_relative './base_job'
 module CanvasSync
   module JobBatches
     class ManagedBatchJob < BaseJob
-      def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, desc_prefix: nil, &blk)
+      def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, preflight_check: nil, desc_prefix: nil, &blk)
         desc_prefix ||= ''
 
         if concurrency == 0 || concurrency == nil || concurrency == true
@@ -13,6 +13,7 @@ module CanvasSync
         end
 
         root_batch = Batch.new
+        man_batch_id = nil
 
         if concurrency < sub_jobs.count
           man_batch_id = SecureRandom.urlsafe_base64(10)
@@ -22,6 +23,7 @@ module CanvasSync
               r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
               r.hset("MNGBID-#{man_batch_id}", "ordered", ordered ? 1 : 0)
               r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
+              r.hset("MNGBID-#{man_batch_id}", "preflight_check", preflight_check) if preflight_check.present?
               r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
 
               mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
@@ -48,12 +50,14 @@ module CanvasSync
 
         blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
 
-        root_batch.description = "#{desc_prefix}: #{root_batch.description || 'Root'}"
+        root_batch.description = "#{desc_prefix}#{root_batch.description || 'Root'}"
+
+        root_batch.context["managed_batch_bid"] = man_batch_id if man_batch_id
 
         if concurrency < sub_jobs.count
           root_batch.jobs {}
           concurrency.times do
-            perform_next_sequence_job(man_batch_id)
+            perform_next_sequence_job(man_batch_id, skip_preflight: true)
           end
         else
           root_batch.jobs do
@@ -62,10 +66,12 @@ module CanvasSync
             end
           end
         end
+
+        root_batch
       end
 
-      def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
-        self.class.make_batch(sub_jobs, ordered: ordered, concurrency: concurrency, context: context)
+      def perform(sub_jobs, **kwargs)
+        self.class.make_batch(sub_jobs, **kwargs)
       end
 
       def self.cleanup_redis(status, options)
@@ -85,37 +91,63 @@ module CanvasSync
 
       protected
 
-      def self.perform_next_sequence_job(man_batch_id)
-        root_bid, ordered = Batch.redis do |r|
+      def self.perform_next_sequence_job(man_batch_id, skip_preflight: false)
+        root_bid, ordered, preflight_check = Batch.redis do |r|
           r.multi do |r|
             r.hget("MNGBID-#{man_batch_id}", "root_bid")
             r.hget("MNGBID-#{man_batch_id}", "ordered")
+            r.hget("MNGBID-#{man_batch_id}", "preflight_check")
           end
         end
 
-        ordered = CanvasSync::MiscHelper.to_boolean(ordered)
-
-        next_job_json = Batch.redis do |r|
-          if ordered
-            r.lpop("MNGBID-#{man_batch_id}-jobs")
+        if !skip_preflight && preflight_check.present?
+          if preflight_check.include?(".")
+            clazz, method_name = preflight_check.split('.')
+            clazz = clazz.constantize
           else
-            r.spop("MNGBID-#{man_batch_id}-jobs")
+            clazz = Object
+            method_name = preflight_check
           end
+          preflight_check = ->(*args) { clazz.send(method_name, *args) }
+        else
+          preflight_check = ->(*args) { true }
         end
 
-        return unless next_job_json.present?
+        ordered = CanvasSync::MiscHelper.to_boolean(ordered)
+
+        loop do
+          next_job_json = Batch.redis do |r|
+            if ordered
+              r.lpop("MNGBID-#{man_batch_id}-jobs")
+            else
+              r.spop("MNGBID-#{man_batch_id}-jobs")
+            end
+          end
 
-        next_job = JSON.parse(next_job_json)
-        next_job = ::ActiveJob::Arguments.deserialize(next_job)[0]
+          break unless next_job_json.present?
 
-        Batch.new(root_bid).jobs do
-          Batch.new.tap do |batch|
-            batch.description = "Managed Batch Fiber (#{man_batch_id})"
-            batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
-            batch.jobs do
-              ChainBuilder.enqueue_job(next_job)
+          next_job = JSON.parse(next_job_json)
+          next_job = ::ActiveJob::Arguments.deserialize(next_job)[0]
+
+          preflight_result = preflight_check.call(next_job)
+          if preflight_result == :abort
+            cleanup_redis(nil, { "managed_batch_id" => man_batch_id })
+            break
+          elsif !preflight_check
+            next
+          end
+
+          Batch.new(root_bid).jobs do
+            Batch.new.tap do |batch|
+              batch.description = "Managed Batch Fiber (#{man_batch_id})"
+              batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
+              batch.jobs do
+                ChainBuilder.enqueue_job(next_job)
+              end
             end
           end
+
+          break
         end
       end
 

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -3,19 +3,19 @@ require_relative './base_job'
 module CanvasSync
   module JobBatches
     class SerialBatchJob < BaseJob
-      def self.make_batch(sub_jobs, context: nil, &blk)
+      def self.make_batch(sub_jobs, **kwargs, &blk)
         ManagedBatchJob.make_batch(
           sub_jobs,
+          **kwargs,
           ordered: true,
           concurrency: false,
-          context: context,
-          desc_prefix: 'SerialBatchJob',
+          desc_prefix: 'SerialBatchJob: ',
           &blk
         )
       end
 
-      def perform(sub_jobs, context: nil)
-        self.class.make_batch(sub_jobs, context: context)
+      def perform(sub_jobs, **kwargs)
+        self.class.make_batch(sub_jobs, **kwargs)
       end
     end
   end