canvas_sync 0.16.5 → 0.17.0.beta5

data/lib/canvas_sync/job_batches/jobs/base_job.rb

@@ -0,0 +1,7 @@
+module CanvasSync
+  module JobBatches
+    class BaseJob < ActiveJob::Base
+
+    end
+  end
+end

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -0,0 +1,19 @@
+require_relative './base_job'
+
+module CanvasSync
+  module JobBatches
+    class ConcurrentBatchJob < BaseJob
+      def perform(sub_jobs, context: nil)
+        Batch.new.tap do |b|
+          b.description = "Concurrent Batch Root"
+          b.context = context
+          b.jobs do
+            sub_jobs.each do |j|
+              ChainBuilder.enqueue_job(j)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -0,0 +1,75 @@
+require_relative './base_job'
+
+module CanvasSync
+  module JobBatches
+    class SerialBatchJob < BaseJob
+      def perform(sub_jobs, context: nil)
+        serial_id = SecureRandom.urlsafe_base64(10)
+
+        root_batch = Batch.new
+
+        Batch.redis do |r|
+          r.multi do
+            mapped_sub_jobs = sub_jobs.map do |j|
+              j = ActiveJob::Arguments.serialize([j])
+              JSON.unparse(j)
+            end
+            r.hset("SERBID-#{serial_id}", "root_bid", root_batch.bid)
+            r.expire("SERBID-#{serial_id}", Batch::BID_EXPIRE_TTL)
+            r.rpush("SERBID-#{serial_id}-jobs", mapped_sub_jobs)
+            r.expire("SERBID-#{serial_id}-jobs", Batch::BID_EXPIRE_TTL)
+          end
+        end
+
+        root_batch.description = "Serial Batch Root (#{serial_id})"
+        root_batch.allow_context_changes = true
+        root_batch.context = context
+        root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", serial_batch_id: serial_id)
+        root_batch.jobs do
+          self.class.perform_next_sequence_job(serial_id)
+        end
+      end
+
+      def self.cleanup_redis(status, options)
+        serial_id = options['serial_batch_id']
+        Batch.redis do |r|
+          r.del(
+            "SERBID-#{serial_id}",
+            "SERBID-#{serial_id}-jobs",
+          )
+        end
+      end
+
+      def self.job_succeeded_callback(status, options)
+        serial_id = options['serial_batch_id']
+        perform_next_sequence_job(serial_id)
+      end
+
+      protected
+
+      def self.perform_next_sequence_job(serial_id)
+        root_bid, next_job_json = Batch.redis do |r|
+          r.multi do
+            r.hget("SERBID-#{serial_id}", "root_bid")
+            r.lpop("SERBID-#{serial_id}-jobs")
+          end
+        end
+
+        return unless next_job_json.present?
+
+        next_job = JSON.parse(next_job_json)
+        next_job = ActiveJob::Arguments.deserialize(next_job)[0]
+
+        Batch.new(root_bid).jobs do
+          Batch.new.tap do |batch|
+          batch.description = "Serial Batch Fiber (#{serial_id})"
+          batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
+            batch.jobs do
+              ChainBuilder.enqueue_job(next_job)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -0,0 +1,93 @@
+begin
+  require 'sidekiq/batch'
+rescue LoadError
+end
+
+module CanvasSync
+  module JobBatches
+    module Sidekiq
+      module WorkerExtension
+        def bid
+          Thread.current[:batch].bid
+        end
+
+        def batch
+          Thread.current[:batch]
+        end
+
+        def valid_within_batch?
+          batch.valid?
+        end
+      end
+
+      class ClientMiddleware
+        def call(_worker, msg, _queue, _redis_pool = nil)
+          if (batch = Thread.current[:batch])
+            batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
+          end
+          yield
+        end
+      end
+
+      class ServerMiddleware
+        def call(_worker, msg, _queue)
+          if (bid = msg['bid'])
+            begin
+              Thread.current[:batch] = Batch.new(bid)
+              yield
+              Thread.current[:batch] = nil
+              Batch.process_successful_job(bid, msg['jid'])
+            rescue
+              Batch.process_failed_job(bid, msg['jid'])
+              raise
+            ensure
+              Thread.current[:batch] = nil
+            end
+          else
+            yield
+          end
+        end
+      end
+
+      def self.configure
+        if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != JobBatches::Batch
+          print "WARNING: Detected Sidekiq Pro or sidekiq-batch. CanvasSync JobBatches may not be fully compatible!"
+        end
+
+        ::Sidekiq.configure_client do |config|
+          config.client_middleware do |chain|
+            chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
+            chain.add JobBatches::Sidekiq::ClientMiddleware
+          end
+        end
+        ::Sidekiq.configure_server do |config|
+          config.client_middleware do |chain|
+            chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
+            chain.add JobBatches::Sidekiq::ClientMiddleware
+          end
+
+          config.server_middleware do |chain|
+            chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
+            chain.add JobBatches::Sidekiq::ServerMiddleware
+          end
+
+          config.death_handlers << ->(job, ex) do
+            return unless job['bid'].present?
+
+            if defined?(::Apartment)
+              ::Apartment::Tenant.switch(job['apartment'] || 'public') do
+                Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+              end
+            else
+              Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+            end
+          end
+        end
+        ::Sidekiq.const_set(:Batch, CanvasSync::JobBatches::Batch)
+        # This alias helps apartment-sidekiq set itself up correctly
+        ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
+        ::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/status.rb

@@ -0,0 +1,83 @@
+module CanvasSync
+  module JobBatches
+    class Batch
+      class Status
+        attr_reader :bid
+
+        def initialize(bid)
+          @bid = bid
+        end
+
+        def join
+          raise "Not supported"
+        end
+
+        def pending
+          Batch.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i
+        end
+
+        def failures
+          Batch.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
+        end
+
+        def created_at
+          Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
+        end
+
+        def total
+          Batch.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
+        end
+
+        def parent_bid
+          Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
+        end
+
+        def failure_info
+          Batch.redis { |r| r.smembers("BID-#{bid}-failed") } || []
+        end
+
+        def complete?
+          'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'complete') }
+        end
+
+        def success?
+          'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
+        end
+
+        def child_count
+          Batch.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
+        end
+
+        def completed_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-complete") }.to_i
+        end
+
+        def successful_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-success") }.to_i
+        end
+
+        def failed_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-failed") }.to_i
+        end
+
+        def data
+          {
+            bid: bid,
+            total: total,
+            failures: failures,
+            pending: pending,
+            created_at: created_at,
+            complete: complete?,
+            success: success?,
+            failure_info: failure_info,
+            parent_bid: parent_bid,
+            child_count: child_count,
+            completed_children_count: completed_children_count,
+            successful_children_count: successful_children_count,
+            failed_children_count: failed_children_count,
+          }
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -0,0 +1,35 @@
+module CanvasSync
+  module Jobs
+    class BeginSyncChainJob < CanvasSync::Job
+      def perform(chain_definition, globals = {})
+        if !globals[:updated_after].present? || globals[:updated_after] == true
+          last_batch = SyncBatch.where(status: 'completed').last
+          globals[:updated_after] = last_batch&.started_at&.iso8601
+        end
+
+        sync_batch = SyncBatch.create!(
+          started_at: DateTime.now,
+          status: 'pending',
+        )
+
+        JobBatches::Batch.new.tap do |b|
+          b.description = "CanvasSync Root Batch"
+          b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
+          b.on(:success, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
+          b.context = globals
+          b.jobs do
+            JobBatches::SerialBatchJob.perform_now(chain_definition)
+          end
+        end
+      end
+
+      def self.batch_completed(status, options)
+        sbatch = SyncBatch.find(options['sync_batch_id'])
+        sbatch.update!(
+          status: status.success? ? 'completed' : 'failed',
+          completed_at: DateTime.now,
+        )
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/report_checker.rb

@@ -4,21 +4,19 @@ module CanvasSync
     # Re-enqueues itself if the report is still processing on Canvas.
     # Enqueues the ReportProcessor when the report has completed.
     class ReportChecker < CanvasSync::Job
-      # @param job_chain [Hash]
       # @param report_name [Hash] e.g., 'provisioning_csv'
       # @param report_id [Integer]
       # @param processor [String] a stringified report processor class name
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
-      def perform(job_chain, report_name, report_id, processor, options) # rubocop:disable Metrics/AbcSize
-        account_id = options[:account_id] || job_chain[:global_options][:account_id] || "self"
-        report_status = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
+      def perform(report_name, report_id, processor, options) # rubocop:disable Metrics/AbcSize
+        account_id = options[:account_id] || batch_context[:account_id] || "self"
+        report_status = CanvasSync.get_canvas_sync_client(batch_context)
                                   .report_status(account_id, report_name, report_id)
 
         case report_status["status"].downcase
         when "complete"
           CanvasSync::Jobs::ReportProcessorJob.perform_later(
-            job_chain,
             report_name,
             report_status["attachment"]["url"],
             processor,
@@ -33,7 +31,6 @@ module CanvasSync
           CanvasSync::Jobs::ReportChecker
             .set(wait: report_checker_wait_time)
             .perform_later(
-              job_chain,
               report_name,
               report_id,
               processor,

data/lib/canvas_sync/jobs/report_processor_job.rb

@@ -6,22 +6,19 @@ module CanvasSync
     # download the report, and then pass the file path and options into the
     # process method on the processor.
     class ReportProcessorJob < CanvasSync::Job
-      # @param job_chain [Hash]
       # @param report_name [Hash] e.g., 'provisioning_csv'
       # @param report_url [String]
       # @param processor [String] a stringified report processor class name
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
-      def perform(job_chain, report_name, report_url, processor, options, report_id)
+      def perform(report_name, report_url, processor, options, report_id)
         @job_log.update_attributes(job_class: processor)
         download(report_name, report_url) do |file_path|
-          options = job_chain[:global_options].merge(options).merge({
+          options = batch_context.merge(options).merge({
             report_processor_job_id: @job_log.job_id
           })
           processor.constantize.process(file_path, options, report_id)
         end
-
-        CanvasSync.invoke_next(job_chain)
       end
 
       private

data/lib/canvas_sync/jobs/report_starter.rb

@@ -2,7 +2,6 @@ module CanvasSync
   module Jobs
     # Starts a Canvas report and enqueues a ReportChecker
     class ReportStarter < CanvasSync::Job
-      # @param job_chain [Hash]
       # @param report_name [Hash] e.g., 'provisioning_csv'
       # @param report_params [Hash] The Canvas report parameters
       # @param processor [String] a stringified report processor class name
@@ -10,31 +9,39 @@ module CanvasSync
       # @param allow_redownloads [Boolean] whether you want the job_chain to cache this report,
       #   so that any later jobs in the chain will use the same generated report
       # @return [nil]
-      def perform(job_chain, report_name, report_params, processor, options, allow_redownloads: false)
-        account_id = options[:account_id] || job_chain[:global_options][:account_id] || "self"
-	options[:sync_start_time] = DateTime.now.utc.iso8601
-        report_id = if allow_redownloads
-                      get_cached_report(job_chain, account_id, report_name, report_params)
-                    else
-                      start_report(job_chain, account_id, report_name, report_params)
-                    end
+      def perform(report_name, report_params, processor, options, allow_redownloads: false)
+        account_id = options[:account_id] || batch_context[:account_id] || "self"
 
-        CanvasSync::Jobs::ReportChecker.set(wait: report_checker_wait_time).perform_later(
-          job_chain,
-          report_name,
-          report_id,
-          processor,
-          options,
-        )
+        report_id = start_report(account_id, report_name, report_params)
+        # TODO: Restore report caching support (does nayone actually use it?)
+        # report_id = if allow_redownloads
+        #               get_cached_report(account_id, report_name, report_params)
+        #             else
+        #               start_report(account_id, report_name, report_params)
+        #             end
+
+        batch = JobBatches::Batch.new
+        batch.description = "CanvasSync #{report_name} Fiber"
+        batch.jobs do
+          CanvasSync::Jobs::ReportChecker.set(wait: report_checker_wait_time).perform_later(
+            report_name,
+            report_id,
+            processor,
+            options,
+          )
+        end
       end
 
       protected
 
-      def merge_report_params(job_chain, options={}, params={}, term_scope: true)
-        term_scope = job_chain[:global_options][:canvas_term_id] if term_scope == true
+      def merge_report_params(options={}, params={}, term_scope: true)
+        term_scope = batch_context[:canvas_term_id] if term_scope == true
         if term_scope.present?
           params[:enrollment_term_id] = term_scope
         end
+        if (updated_after = batch_context[:updated_after]).present?
+          params[:updated_after] = updated_after
+        end
         params.merge!(options[:report_params]) if options[:report_params].present?
         { parameters: params }
       end
@@ -42,6 +49,7 @@ module CanvasSync
       private
 
       def get_cached_report(job_chain, account_id, report_name, report_params)
+        # TODO: job_chain[:global_options] is no longer available and batch_context won't work for this
         if job_chain[:global_options][report_name].present?
           job_chain[:global_options][report_name]
         else
@@ -51,8 +59,8 @@ module CanvasSync
         end
       end
 
-      def start_report(job_chain, account_id, report_name, report_params)
-        report = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
+      def start_report(account_id, report_name, report_params)
+        report = CanvasSync.get_canvas_sync_client(batch_context)
                            .start_report(account_id, report_name, report_params)
         report["id"]
       end