canvas_sync 0.16.2 → 0.17.0.beta3

data/lib/canvas_sync/job_batches/batch_aware_job.rb

@@ -0,0 +1,62 @@
+module CanvasSync
+  module JobBatches
+    module BatchAwareJob
+      extend ActiveSupport::Concern
+
+      included do
+        around_perform do |job, block|
+          if (@bid) # This _must_ be @bid - not just bid
+            prev_batch = Thread.current[:batch]
+            begin
+              Thread.current[:batch] = Batch.new(@bid)
+              block.call
+              batch&.save_context_changes
+              Batch.process_successful_job(@bid, job_id)
+            rescue
+              Batch.process_failed_job(@bid, job_id)
+              raise
+            ensure
+              Thread.current[:batch] = prev_batch
+            end
+          else
+            block.call
+          end
+        end
+
+        around_enqueue do |job, block|
+          if (batch = Thread.current[:batch])
+            batch.increment_job_queue(job_id) if (@bid = batch.bid)
+          end
+          block.call
+        end
+      end
+
+      def bid
+        @bid || Thread.current[:batch]&.bid
+      end
+
+      def batch
+        Thread.current[:batch]
+      end
+
+      def batch_context
+        batch&.context || {}
+      end
+
+      def valid_within_batch?
+        batch.valid?
+      end
+
+      def serialize
+        super.tap do |data|
+          data['batch_id'] = @bid # This _must_ be @bid - not just bid
+        end
+      end
+
+      def deserialize(data)
+        super
+        @bid = data['batch_id']
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/callback.rb

@@ -0,0 +1,153 @@
+module CanvasSync
+  module JobBatches
+    class Batch
+      module Callback
+
+        VALID_CALLBACKS = %w[success complete dead].freeze
+
+        module CallbackWorkerCommon
+          def perform(definition, event, opts, bid, parent_bid)
+            return unless VALID_CALLBACKS.include?(event)
+
+            method = nil
+            target = :instance
+            clazz = definition
+            if clazz.is_a?(String)
+              if clazz.include?('#')
+                clazz, method = clazz.split("#")
+              elsif clazz.include?('.')
+                clazz, method = clazz.split(".")
+                target = :class
+              end
+            end
+
+            method ||= "on_#{event}"
+            status = Batch::Status.new(bid)
+
+            if clazz && object = Object.const_get(clazz)
+              target = target == :instance ? object.new : object
+              if target.respond_to?(method)
+                target.send(method, status, opts)
+              else
+                Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")
+              end
+            else
+              Batch.logger.warn("Invalid callback method #{definition} - Class #{clazz} not found")
+            end
+          end
+        end
+
+        class ActiveJobCallbackWorker < ActiveJob::Base
+          include CallbackWorkerCommon
+
+          def self.enqueue_all(args, queue)
+            args.each do |arg_set|
+              set(queue: queue).perform_later(*arg_set)
+            end
+          end
+        end
+
+        if defined?(::Sidekiq)
+          class SidekiqCallbackWorker
+            include ::Sidekiq::Worker
+            include CallbackWorkerCommon
+
+            def self.enqueue_all(args, queue)
+              return if args.empty?
+
+              ::Sidekiq::Client.push_bulk(
+                'class' => self,
+                'args' => args,
+                'queue' => queue
+              )
+            end
+          end
+          Worker = SidekiqCallbackWorker
+        else
+          Worker = ActiveJobCallbackWorker
+        end
+
+        class Finalize
+          def dispatch status, opts
+            bid = opts["bid"]
+            callback_bid = status.bid
+            event = opts["event"].to_sym
+            callback_batch = bid != callback_bid
+
+            Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
+
+            batch_status = Status.new bid
+            send(event, bid, batch_status, batch_status.parent_bid)
+
+            # Different events are run in different callback batches
+            Batch.cleanup_redis callback_bid if callback_batch
+            Batch.cleanup_redis bid if event == :success
+          end
+
+          def success(bid, status, parent_bid)
+            return unless parent_bid
+
+            _, _, success, _, complete, pending, children, failure = Batch.redis do |r|
+              r.multi do
+                r.sadd("BID-#{parent_bid}-success", bid)
+                r.expire("BID-#{parent_bid}-success", Batch::BID_EXPIRE_TTL)
+                r.scard("BID-#{parent_bid}-success")
+                r.sadd("BID-#{parent_bid}-complete", bid)
+                r.scard("BID-#{parent_bid}-complete")
+                r.hincrby("BID-#{parent_bid}", "pending", 0)
+                r.hincrby("BID-#{parent_bid}", "children", 0)
+                r.scard("BID-#{parent_bid}-failed")
+              end
+            end
+            # if job finished successfully and parent batch completed call parent complete callback
+            # Success callback is called after complete callback
+            if complete == children && pending == failure
+              Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
+              Batch.enqueue_callbacks(:complete, parent_bid)
+            end
+          end
+
+          def complete(bid, status, parent_bid)
+            pending, children, success = Batch.redis do |r|
+              r.multi do
+                r.hincrby("BID-#{bid}", "pending", 0)
+                r.hincrby("BID-#{bid}", "children", 0)
+                r.scard("BID-#{bid}-success")
+              end
+            end
+
+            # if we batch was successful run success callback
+            if pending.to_i.zero? && children == success
+              Batch.enqueue_callbacks(:success, bid)
+
+            elsif parent_bid
+              # if batch was not successfull check and see if its parent is complete
+              # if the parent is complete we trigger the complete callback
+              # We don't want to run this if the batch was successfull because the success
+              # callback may add more jobs to the parent batch
+
+              Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
+              _, complete, pending, children, failure = Batch.redis do |r|
+                r.multi do
+                  r.sadd("BID-#{parent_bid}-complete", bid)
+                  r.scard("BID-#{parent_bid}-complete")
+                  r.hincrby("BID-#{parent_bid}", "pending", 0)
+                  r.hincrby("BID-#{parent_bid}", "children", 0)
+                  r.scard("BID-#{parent_bid}-failed")
+                end
+              end
+              if complete == children && pending == failure
+                Batch.enqueue_callbacks(:complete, parent_bid)
+              end
+            end
+          end
+
+          def cleanup_redis bid, callback_bid=nil
+            Batch.cleanup_redis bid
+            Batch.cleanup_redis callback_bid if callback_bid
+          end
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -0,0 +1,210 @@
+module CanvasSync
+  module JobBatches
+    class ChainBuilder
+      VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
+
+      attr_reader :base_job
+
+      def initialize(base_type = SerialBatchJob)
+        if base_type.is_a?(Hash)
+          @base_job = base_type
+        else
+          @base_job = {
+            job: base_type,
+            parameters: [],
+          }
+        end
+      end
+
+      def process!
+        normalize!
+        self.class.enqueue_job(base_job)
+      end
+
+      def [](key)
+        if key.is_a?(Class)
+          get_sub_chain(key)
+        else
+          @base_job[key]
+        end
+      end
+
+      def params
+        ParamsMapper.new(self[:parameters])
+      end
+
+      def <<(new_job)
+        insert_at(-1, new_job)
+      end
+
+      def insert_at(position, new_jobs)
+        chain = self.class.get_chain_parameter(base_job)
+        new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
+        chain.insert(-1, *new_jobs)
+      end
+
+      def insert(new_jobs, **kwargs)
+        invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
+        raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
+        raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
+
+        new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
+
+        if !kwargs.present?
+          insert_at(-1, new_jobs)
+        else
+          placement = kwargs.keys[0]
+          relative_to = kwargs.values[0]
+
+          matching_jobs = find_matching_jobs(relative_to)
+          raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
+          raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
+
+          parent_job, sub_index = matching_jobs[0]
+          chain = self.class.get_chain_parameter(parent_job)
+          needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
+
+          if parent_job[:job] != needed_parent_type
+            old_job = chain[sub_index]
+            parent_job = chain[sub_index] = {
+              job: needed_parent_type,
+              parameters: [],
+            }
+            sub_index = 0
+            chain = self.class.get_chain_parameter(parent_job)
+            chain << old_job
+          end
+
+          if placement == :with
+            chain.insert(-1, *new_jobs)
+          else
+            sub_index += 1 if placement == :after
+            chain.insert(sub_index, *new_jobs)
+          end
+        end
+      end
+
+      def get_sub_chain(sub_type)
+        matching_jobs = find_matching_jobs(sub_type)
+        raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
+        return nil if matching_jobs.count == 0
+
+        new(matching_jobs[0])
+      end
+
+      def normalize!(job_def = self.base_job)
+        if job_def.is_a?(ChainBuilder)
+          job_def.normalize!
+        else
+          job_def[:job] = job_def[:job].to_s
+          if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
+            chain.map! { |sub_job| normalize!(sub_job) }
+          end
+          job_def
+        end
+      end
+
+      private
+
+      def find_matching_jobs(search_job, parent_job = self.base_job)
+        return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
+
+        sub_jobs = self.class.get_chain_parameter(parent_job)
+        sub_jobs.each_with_index do |sub_job, i|
+          if sub_job[:job].to_s == search_job.to_s
+            yield [parent_job, i]
+          elsif self.class._job_type_definitions[sub_job[:job]]
+            find_matching_jobs(search_job) { |item| yield item }
+          end
+        end
+      end
+
+      class << self
+        def _job_type_definitions
+          @job_type_definitions ||= {}
+        end
+
+        def register_chain_job(job_class, chain_parameter, **options)
+          _job_type_definitions[job_class.to_s] = {
+            **options,
+            chain_parameter: chain_parameter,
+          }
+        end
+
+        def get_chain_parameter(job_def, raise_error: true)
+          unless _job_type_definitions[job_def[:job].to_s].present?
+            raise "Job Type #{base_job[:job].to_s} does not accept a sub-chain" if raise_error
+            return nil
+          end
+
+          key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
+          mapper = ParamsMapper.new(job_def[:parameters])
+          mapper[key] ||= []
+        end
+
+        def enqueue_job(job_def)
+          job_class = job_def[:job].constantize
+          job_options = job_def[:parameters] || []
+
+          # Legacy Support
+          if job_def[:options]
+            job_options << {} unless job_options[-1].is_a?(Hash)
+            job_options[-1].merge!(job_def[:options])
+          end
+
+          if job_class.respond_to? :perform_async
+            job_class.perform_async(*job_options)
+          else
+            job_class.perform_later(*job_options)
+          end
+        end
+      end
+    end
+
+    ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
+    ChainBuilder.register_chain_job(SerialBatchJob, 0)
+
+    class ParamsMapper
+      def initialize(backend)
+        @backend = backend
+      end
+
+      def [](key)
+        get_parameter(key)
+      end
+
+      def []=(key, value)
+        set_parameter(key, value)
+      end
+
+      def to_a
+        @backend
+      end
+
+      private
+
+      def get_parameter(key)
+        if key.is_a?(Numeric)
+          @backend[key]
+        else
+          kwargs = @backend.last
+          return nil unless kwargs.is_a?(Hash)
+          kwargs[key]
+        end
+      end
+
+      def set_parameter(key, value)
+        if key.is_a?(Numeric)
+          @backend[key] = value
+        else
+          kwargs = @backend.last
+          unless kwargs.is_a?(Hash)
+            kwargs = {}
+            @backend.push(kwargs)
+          end
+          kwargs[key] = value
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -0,0 +1,147 @@
+module CanvasSync
+  module JobBatches
+    class ContextHash
+      delegate_missing_to :flatten
+
+      def initialize(bid, hash = nil)
+        @bid_stack = [bid]
+        @hash_map = {}
+        @dirty = false
+        @flattened = nil
+        @hash_map[bid] = hash.with_indifferent_access if hash
+      end
+
+      # Local is "the nearest batch with a context value"
+      # This allows for, for example, SerialBatchJob to have a modifiable context stored on it's main Batch
+      # that can be accessed transparently from one of it's internal, context-less Batches
+      def local_bid
+        bid = @bid_stack[-1]
+        while bid.present?
+          bhash = reolve_hash(bid)
+          return bid if bhash
+          bid = get_parent_bid(bid)
+        end
+        nil
+      end
+
+      def local
+        @hash_map[local_bid]
+      end
+
+      def set_local(new_hash)
+        @dirty = true
+        local.clear.merge!(new_hash)
+      end
+
+      def clear
+        local.clear
+        @flattened = nil
+        @dirty = true
+        self
+      end
+
+      def []=(key, value)
+        @flattened = nil
+        @dirty = true
+        local[key] = value
+      end
+
+      def [](key)
+        bid = @bid_stack[-1]
+        while bid.present?
+          bhash = reolve_hash(bid)
+          return bhash[key] if bhash&.key?(key)
+          bid = get_parent_bid(bid)
+        end
+        nil
+      end
+
+      def reload!
+        @dirty = false
+        @hash_map = {}
+        self
+      end
+
+      def save!(force: false)
+        return unless dirty? || force
+        Batch.redis do |r|
+          r.hset("BID-#{local_bid}", 'context', JSON.unparse(local))
+        end
+      end
+
+      def dirty?
+        @dirty
+      end
+
+      def is_a?(arg)
+        return true if Hash <= arg
+        super
+      end
+
+      def flatten
+        return @flattened if @flattened
+
+        load_all
+        flattened = {}
+        @bid_stack.compact.each do |bid|
+          flattened.merge!(@hash_map[bid]) if @hash_map[bid]
+        end
+        flattened.freeze
+
+        @flattened = flattened.with_indifferent_access
+      end
+
+      private
+
+      def get_parent_hash(bid)
+        reolve_hash(get_parent_bid(bid)).freeze
+      end
+
+      def get_parent_bid(bid)
+        index = @bid_stack.index(bid)
+        raise "Invalid BID #{bid}" if index.nil? # Sanity Check - this shouldn't happen
+
+        index -= 1
+        if index >= 0
+          @bid_stack[index]
+        else
+          pbid = Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
+          @bid_stack.unshift(pbid)
+          pbid
+        end
+      end
+
+      def reolve_hash(bid)
+        return nil unless bid.present?
+        return @hash_map[bid] if @hash_map.key?(bid)
+
+        context_json, editable = Batch.redis do |r|
+          r.multi do
+            r.hget("BID-#{bid}", "context")
+            r.hget("BID-#{bid}", "allow_context_changes")
+          end
+        end
+
+        if context_json.present?
+          context_hash = JSON.parse(context_json)
+          context_hash = context_hash.with_indifferent_access
+          context_hash.each do |k, v|
+            v.freeze
+          end
+          context_hash.freeze unless editable
+
+          @hash_map[bid] = context_hash
+        else
+          @hash_map[bid] = nil
+        end
+      end
+
+      def load_all
+        while @bid_stack[0].present?
+          get_parent_hash(@bid_stack[0])
+        end
+        @hash_map
+      end
+    end
+  end
+end