canvas_sync 0.16.5 → 0.17.0.beta1

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -0,0 +1,203 @@
+module CanvasSync
+  module JobBatches
+    class ChainBuilder
+      VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
+
+      attr_reader :base_job
+
+      def initialize(base_type = SerialBatchJob)
+        if base_type.is_a?(Hash)
+          @base_job = base_type
+        else
+          @base_job = {
+            job: base_type,
+            parameters: [],
+          }
+        end
+      end
+
+      def process!
+        normalize!
+        self.class.enqueue_job(base_job)
+      end
+
+      def [](key)
+        if key.is_a?(Class)
+          get_sub_chain(key)
+        else
+          @base_job[key]
+        end
+      end
+
+      def params
+        ParamsMapper.new(self[:parameters])
+      end
+
+      def <<(new_job)
+        insert_at(-1, new_job)
+      end
+
+      def insert_at(position, new_jobs)
+        chain = self.class.get_chain_parameter(base_job)
+        new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
+        chain.insert(-1, *new_jobs)
+      end
+
+      def insert(new_jobs, **kwargs)
+        invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
+        raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
+        raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
+
+        new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
+
+        if !kwargs.present?
+          insert_at(-1, new_jobs)
+        else
+          placement = kwargs.keys[0]
+          relative_to = kwargs.values[0]
+
+          matching_jobs = find_matching_jobs(relative_to)
+          raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
+          raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
+
+          parent_job, sub_index = matching_jobs[0]
+          chain = self.class.get_chain_parameter(parent_job)
+          needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
+
+          if parent_job[:job] != needed_parent_type
+            old_job = chain[sub_index]
+            parent_job = chain[sub_index] = {
+              job: needed_parent_type,
+              parameters: [],
+            }
+            sub_index = 0
+            chain = self.class.get_chain_parameter(parent_job)
+            chain << old_job
+          end
+
+          if placement == :with
+            chain.insert(-1, *new_jobs)
+          else
+            sub_index += 1 if placement == :after
+            chain.insert(sub_index, *new_jobs)
+          end
+        end
+      end
+
+      def get_sub_chain(sub_type)
+        matching_jobs = find_matching_jobs(sub_type)
+        raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
+        return nil if matching_jobs.count == 0
+
+        new(matching_jobs[0])
+      end
+
+      def normalize!(job_def = self.base_job)
+        if job_def.is_a?(ChainBuilder)
+          job_def.normalize!
+        else
+          job_def[:job] = job_def[:job].to_s
+          if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
+            chain.map! { |sub_job| normalize!(sub_job) }
+          end
+          job_def
+        end
+      end
+
+      private
+
+      def find_matching_jobs(search_job, parent_job = self.base_job)
+        return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
+
+        sub_jobs = self.class.get_chain_parameter(parent_job)
+        sub_jobs.each_with_index do |sub_job, i|
+          if sub_job[:job].to_s == search_job.to_s
+            yield [parent_job, i]
+          elsif self.class._job_type_definitions[sub_job[:job]]
+            find_matching_jobs(search_job) { |item| yield item }
+          end
+        end
+      end
+
+      class << self
+        def _job_type_definitions
+          @job_type_definitions ||= {}
+        end
+
+        def register_chain_job(job_class, chain_parameter, **options)
+          _job_type_definitions[job_class.to_s] = {
+            **options,
+            chain_parameter: chain_parameter,
+          }
+        end
+
+        def get_chain_parameter(job_def, raise_error: true)
+          unless _job_type_definitions[job_def[:job].to_s].present?
+            raise "Job Type #{base_job[:job].to_s} does not accept a sub-chain" if raise_error
+            return nil
+          end
+
+          key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
+          mapper = ParamsMapper.new(job_def[:parameters])
+          mapper[key] ||= []
+        end
+
+        def enqueue_job(job_def)
+          job_class = job_def[:job].constantize
+          job_options = job_def[:parameters] || []
+          if job_class.respond_to? :perform_async
+            job_class.perform_async(*job_options)
+          else
+            job_class.perform_later(*job_options)
+          end
+        end
+      end
+    end
+
+    ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
+    ChainBuilder.register_chain_job(SerialBatchJob, 0)
+
+    class ParamsMapper
+      def initialize(backend)
+        @backend = backend
+      end
+
+      def [](key)
+        get_parameter(key)
+      end
+
+      def []=(key, value)
+        set_parameter(key, value)
+      end
+
+      def to_a
+        @backend
+      end
+
+      private
+
+      def get_parameter(key)
+        if key.is_a?(Numeric)
+          @backend[key]
+        else
+          kwargs = @backend.last
+          return nil unless kwargs.is_a?(Hash)
+          kwargs[key]
+        end
+      end
+
+      def set_parameter(key, value)
+        if key.is_a?(Numeric)
+          @backend[key] = value
+        else
+          kwargs = @backend.last
+          unless kwargs.is_a?(Hash)
+            kwargs = {}
+            @backend.push(kwargs)
+          end
+          kwargs[key] = value
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -0,0 +1,147 @@
+module CanvasSync
+  module JobBatches
+    class ContextHash
+      delegate_missing_to :flatten
+
+      def initialize(bid, hash = nil)
+        @bid_stack = [bid]
+        @hash_map = {}
+        @dirty = false
+        @flattened = nil
+        @hash_map[bid] = hash.with_indifferent_access if hash
+      end
+
+      # Local is "the nearest batch with a context value"
+      # This allows for, for example, SerialBatchJob to have a modifiable context stored on it's main Batch
+      # that can be accessed transparently from one of it's internal, context-less Batches
+      def local_bid
+        bid = @bid_stack[-1]
+        while bid.present?
+          bhash = reolve_hash(bid)
+          return bid if bhash
+          bid = get_parent_bid(bid)
+        end
+        nil
+      end
+
+      def local
+        @hash_map[local_bid]
+      end
+
+      def set_local(new_hash)
+        @dirty = true
+        local.clear.merge!(new_hash)
+      end
+
+      def clear
+        local.clear
+        @flattened = nil
+        @dirty = true
+        self
+      end
+
+      def []=(key, value)
+        @flattened = nil
+        @dirty = true
+        local[key] = value
+      end
+
+      def [](key)
+        bid = @bid_stack[-1]
+        while bid.present?
+          bhash = reolve_hash(bid)
+          return bhash[key] if bhash&.key?(key)
+          bid = get_parent_bid(bid)
+        end
+        nil
+      end
+
+      def reload!
+        @dirty = false
+        @hash_map = {}
+        self
+      end
+
+      def save!(force: false)
+        return unless dirty? || force
+        Batch.redis do |r|
+          r.hset("BID-#{local_bid}", 'context', JSON.unparse(local))
+        end
+      end
+
+      def dirty?
+        @dirty
+      end
+
+      def is_a?(arg)
+        return true if Hash <= arg
+        super
+      end
+
+      def flatten
+        return @flattened if @flattened
+
+        load_all
+        flattened = {}
+        @bid_stack.compact.each do |bid|
+          flattened.merge!(@hash_map[bid]) if @hash_map[bid]
+        end
+        flattened.freeze
+
+        @flattened = flattened.with_indifferent_access
+      end
+
+      private
+
+      def get_parent_hash(bid)
+        reolve_hash(get_parent_bid(bid)).freeze
+      end
+
+      def get_parent_bid(bid)
+        index = @bid_stack.index(bid)
+        raise "Invalid BID #{bid}" if index.nil? # Sanity Check - this shouldn't happen
+
+        index -= 1
+        if index >= 0
+          @bid_stack[index]
+        else
+          pbid = Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
+          @bid_stack.unshift(pbid)
+          pbid
+        end
+      end
+
+      def reolve_hash(bid)
+        return nil unless bid.present?
+        return @hash_map[bid] if @hash_map.key?(bid)
+
+        context_json, editable = Batch.redis do |r|
+          r.multi do
+            r.hget("BID-#{bid}", "context")
+            r.hget("BID-#{bid}", "allow_context_changes")
+          end
+        end
+
+        if context_json.present?
+          context_hash = JSON.parse(context_json)
+          context_hash = context_hash.with_indifferent_access
+          context_hash.each do |k, v|
+            v.freeze
+          end
+          context_hash.freeze unless editable
+
+          @hash_map[bid] = context_hash
+        else
+          @hash_map[bid] = nil
+        end
+      end
+
+      def load_all
+        while @bid_stack[0].present?
+          get_parent_hash(@bid_stack[0])
+        end
+        @hash_map
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/jobs/base_job.rb

@@ -0,0 +1,7 @@
+module CanvasSync
+  module JobBatches
+    class BaseJob < ActiveJob::Base
+
+    end
+  end
+end

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -0,0 +1,18 @@
+require_relative './base_job'
+
+module CanvasSync
+  module JobBatches
+    class ConcurrentBatchJob < BaseJob
+      def perform(sub_jobs, context: nil)
+        Batch.new.tap do |b|
+          b.context = context
+          b.jobs do
+            sub_jobs.each do |j|
+              ChainBuilder.enqueue_job(j)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -0,0 +1,73 @@
+require_relative './base_job'
+
+module CanvasSync
+  module JobBatches
+    class SerialBatchJob < BaseJob
+      def perform(sub_jobs, context: nil)
+        serial_id = SecureRandom.urlsafe_base64(10)
+
+        root_batch = Batch.new
+
+        Batch.redis do |r|
+          r.multi do
+            mapped_sub_jobs = sub_jobs.map do |j|
+              j = ActiveJob::Arguments.serialize([j])
+              JSON.unparse(j)
+            end
+            r.hset("SERBID-#{serial_id}", "root_bid", root_batch.bid)
+            r.expire("SERBID-#{serial_id}", Batch::BID_EXPIRE_TTL)
+            r.rpush("SERBID-#{serial_id}-jobs", mapped_sub_jobs)
+            r.expire("SERBID-#{serial_id}-jobs", Batch::BID_EXPIRE_TTL)
+          end
+        end
+
+        root_batch.allow_context_changes = true
+        root_batch.context = context
+        root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", serial_batch_id: serial_id)
+        root_batch.jobs do
+          self.class.perform_next_sequence_job(serial_id)
+        end
+      end
+
+      def self.cleanup_redis(status, options)
+        serial_id = options['serial_batch_id']
+        Batch.redis do |r|
+          r.del(
+            "SERBID-#{serial_id}",
+            "SERBID-#{serial_id}-jobs",
+          )
+        end
+      end
+
+      def self.job_succeeded_callback(status, options)
+        serial_id = options['serial_batch_id']
+        perform_next_sequence_job(serial_id)
+      end
+
+      protected
+
+      def self.perform_next_sequence_job(serial_id)
+        root_bid, next_job_json = Batch.redis do |r|
+          r.multi do
+            r.hget("SERBID-#{serial_id}", "root_bid")
+            r.lpop("SERBID-#{serial_id}-jobs")
+          end
+        end
+
+        return unless next_job_json.present?
+
+        next_job = JSON.parse(next_job_json)
+        next_job = ActiveJob::Arguments.deserialize(next_job)[0]
+
+        Batch.new(root_bid).jobs do
+          Batch.new.tap do |batch|
+            batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
+            batch.jobs do
+              ChainBuilder.enqueue_job(next_job)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -0,0 +1,91 @@
+begin
+  require 'sidekiq/batch'
+rescue LoadError
+end
+
+module CanvasSync
+  module JobBatches
+    module Sidekiq
+      module WorkerExtension
+        def bid
+          Thread.current[:batch].bid
+        end
+
+        def batch
+          Thread.current[:batch]
+        end
+
+        def valid_within_batch?
+          batch.valid?
+        end
+      end
+
+      class ClientMiddleware
+        def call(_worker, msg, _queue, _redis_pool = nil)
+          if (batch = Thread.current[:batch])
+            batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
+          end
+          yield
+        end
+      end
+
+      class ServerMiddleware
+        def call(_worker, msg, _queue)
+          if (bid = msg['bid'])
+            begin
+              Thread.current[:batch] = Batch.new(bid)
+              yield
+              Thread.current[:batch] = nil
+              Batch.process_successful_job(bid, msg['jid'])
+            rescue
+              Batch.process_failed_job(bid, msg['jid'])
+              raise
+            ensure
+              Thread.current[:batch] = nil
+            end
+          else
+            yield
+          end
+        end
+      end
+
+      def self.configure
+        if defined?(::Sidekiq::Batch) && ::Sidekiq::Batch != JobBatches::Batch
+          print "WARNING: Detected Sidekiq Pro or sidekiq-batch. CanvasSync JobBatches may not be fully compatible!"
+        end
+
+        ::Sidekiq.configure_client do |config|
+          config.client_middleware do |chain|
+            chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
+            chain.add JobBatches::Sidekiq::ClientMiddleware
+          end
+        end
+        ::Sidekiq.configure_server do |config|
+          config.client_middleware do |chain|
+            chain.remove ::Sidekiq::Batch::Middleware::ClientMiddleware if defined?(::Sidekiq::Batch::Middleware::ClientMiddleware)
+            chain.add JobBatches::Sidekiq::ClientMiddleware
+          end
+
+          config.server_middleware do |chain|
+            chain.remove ::Sidekiq::Batch::Middleware::ServerMiddleware if defined?(::Sidekiq::Batch::Middleware::ServerMiddleware)
+            chain.add JobBatches::Sidekiq::ServerMiddleware
+          end
+
+          config.death_handlers << ->(job, ex) do
+            return unless job['bid'].present?
+
+            if defined?(::Apartment)
+              ::Apartment::Tenant.switch(job['apartment'] || 'public') do
+                Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+              end
+            else
+              Sidekiq::Batch.process_dead_job(job['bid'], job['jid'])
+            end
+          end
+        end
+        ::Sidekiq.const_set(:Batch, CanvasSync::JobBatches::Batch)
+        ::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/status.rb

@@ -0,0 +1,63 @@
+module CanvasSync
+  module JobBatches
+    class Batch
+      class Status
+        attr_reader :bid
+
+        def initialize(bid)
+          @bid = bid
+        end
+
+        def join
+          raise "Not supported"
+        end
+
+        def pending
+          Batch.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i
+        end
+
+        def failures
+          Batch.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
+        end
+
+        def created_at
+          Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
+        end
+
+        def total
+          Batch.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
+        end
+
+        def parent_bid
+          Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
+        end
+
+        def failure_info
+          Batch.redis { |r| r.smembers("BID-#{bid}-failed") } || []
+        end
+
+        def complete?
+          'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'complete') }
+        end
+
+        def child_count
+          Batch.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
+        end
+
+        def data
+          {
+            bid: bid,
+            total: total,
+            failures: failures,
+            pending: pending,
+            created_at: created_at,
+            complete: complete?,
+            failure_info: failure_info,
+            parent_bid: parent_bid,
+            child_count: child_count
+          }
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -0,0 +1,34 @@
+module CanvasSync
+  module Jobs
+    class BeginSyncChainJob < CanvasSync::Job
+      def perform(chain_definition, globals = {})
+        if !globals[:updated_after].present? || globals[:updated_after] == true
+          last_batch = SyncBatch.where(status: 'completed').last
+          globals[:updated_after] = last_batch&.started_at&.iso8601
+        end
+
+        sync_batch = SyncBatch.create!(
+          started_at: DateTime.now,
+          status: 'pending',
+        )
+
+        JobBatches::Batch.new.tap do |b|
+          b.description = "CanvasSync Root Batch"
+          b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
+          b.context = globals
+          b.jobs do
+            JobBatches::SerialBatchJob.perform_now(chain_definition)
+          end
+        end
+      end
+
+      def self.batch_completed(status, options)
+        sbatch = SyncBatch.find(options['sync_batch_id'])
+        sbatch.update!(
+          status: status.failures.positive? ? 'failed' : 'completed',
+          completed_at: DateTime.now,
+        )
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/report_checker.rb

@@ -4,21 +4,19 @@ module CanvasSync
     # Re-enqueues itself if the report is still processing on Canvas.
     # Enqueues the ReportProcessor when the report has completed.
     class ReportChecker < CanvasSync::Job
-      # @param job_chain [Hash]
       # @param report_name [Hash] e.g., 'provisioning_csv'
       # @param report_id [Integer]
       # @param processor [String] a stringified report processor class name
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
-      def perform(job_chain, report_name, report_id, processor, options) # rubocop:disable Metrics/AbcSize
-        account_id = options[:account_id] || job_chain[:global_options][:account_id] || "self"
-        report_status = CanvasSync.get_canvas_sync_client(job_chain[:global_options])
+      def perform(report_name, report_id, processor, options) # rubocop:disable Metrics/AbcSize
+        account_id = options[:account_id] || batch_context[:account_id] || "self"
+        report_status = CanvasSync.get_canvas_sync_client(batch_context)
                                   .report_status(account_id, report_name, report_id)
 
         case report_status["status"].downcase
         when "complete"
           CanvasSync::Jobs::ReportProcessorJob.perform_later(
-            job_chain,
             report_name,
             report_status["attachment"]["url"],
             processor,
@@ -33,7 +31,6 @@ module CanvasSync
           CanvasSync::Jobs::ReportChecker
             .set(wait: report_checker_wait_time)
             .perform_later(
-              job_chain,
               report_name,
               report_id,
               processor,

data/lib/canvas_sync/jobs/report_processor_job.rb

@@ -6,22 +6,19 @@ module CanvasSync
     # download the report, and then pass the file path and options into the
     # process method on the processor.
     class ReportProcessorJob < CanvasSync::Job
-      # @param job_chain [Hash]
       # @param report_name [Hash] e.g., 'provisioning_csv'
       # @param report_url [String]
       # @param processor [String] a stringified report processor class name
       # @param options [Hash] hash of options that will be passed to the job processor
       # @return [nil]
-      def perform(job_chain, report_name, report_url, processor, options, report_id)
+      def perform(report_name, report_url, processor, options, report_id)
         @job_log.update_attributes(job_class: processor)
         download(report_name, report_url) do |file_path|
-          options = job_chain[:global_options].merge(options).merge({
+          options = batch_context.merge(options).merge({
             report_processor_job_id: @job_log.job_id
           })
           processor.constantize.process(file_path, options, report_id)
         end
-
-        CanvasSync.invoke_next(job_chain)
       end
 
       private