canvas_sync 0.16.4 → 0.21.0

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -0,0 +1,249 @@
+module CanvasSync
+  module JobBatches
+    class ChainBuilder
+      VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
+
+      attr_reader :base_job
+
+      def initialize(base_type = SerialBatchJob)
+        if base_type.is_a?(Hash)
+          @base_job = base_type
+          @base_job[:args] ||= @base_job[:parameters] || []
+          @base_job[:kwargs] ||= {}
+        else
+          @base_job = build_job_hash(base_type)
+        end
+
+        self.class.get_chain_parameter(base_job)
+      end
+
+      def process!
+        normalize!
+        self.class.enqueue_job(base_job)
+      end
+
+      def [](key)
+        if key.is_a?(Class)
+          get_sub_chain(key)
+        else
+          # Legacy Support
+          key = :args if key == :parameters
+
+          @base_job[key]
+        end
+      end
+
+      def args; return self[:args]; end
+      def kwargs; return self[:kwargs]; end
+
+      def <<(new_job)
+        insert_at(-1, new_job)
+      end
+
+      def insert_at(position, new_jobs, *args, **kwargs, &blk)
+        chain = self.class.get_chain_parameter(base_job)
+        if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
+          new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
+        elsif args.count > 0 || kwargs.count > 0
+          raise "Unexpected number of arguments"
+        end
+        new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
+        chain.insert(position, *new_jobs)
+      end
+
+      def insert(new_jobs, *args, **kwargs, &blk)
+        if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
+          job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
+          new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
+          kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
+        else
+          invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
+          raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
+          raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
+          raise "Unexpected number of arguments" if args.length > 0
+        end
+
+        new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
+
+        if !kwargs.present?
+          insert_at(-1, new_jobs)
+        else
+          placement = kwargs.keys[0]
+          relative_to = kwargs.values[0]
+
+          matching_jobs = find_matching_jobs(relative_to).to_a
+          raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
+          raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
+
+          relative_job, parent_job, sub_index = matching_jobs[0]
+          needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
+
+          chain = self.class.get_chain_parameter(parent_job)
+
+          if parent_job[:job] != needed_parent_type
+            old_job = chain[sub_index]
+            parent_job = chain[sub_index] = {
+              job: needed_parent_type,
+              parameters: [],
+            }
+            sub_index = 0
+            chain = self.class.get_chain_parameter(parent_job)
+            chain << old_job
+          end
+
+          if placement == :with
+            chain.insert(-1, *new_jobs)
+          else
+            sub_index += 1 if placement == :after
+            chain.insert(sub_index, *new_jobs)
+          end
+        end
+      end
+
+      def empty?
+        self.class.get_chain_parameter(self).empty?
+      end
+
+      def get_sub_chain(sub_type)
+        matching_jobs = find_matching_jobs(sub_type).to_a
+        raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
+        return nil if matching_jobs.count == 0
+
+        job = matching_jobs[0][0]
+        job = self.class.new(job) unless job.is_a?(ChainBuilder)
+        job
+      end
+
+      def normalize!(job_def = self.base_job)
+        if job_def.is_a?(ChainBuilder)
+          job_def.normalize!
+        else
+          job_def[:job] = job_def[:job].to_s
+          if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
+            chain.map! { |sub_job| normalize!(sub_job) }
+          end
+          job_def
+        end
+      end
+
+      def apply_block(&blk)
+        return unless blk.present?
+        instance_exec(&blk)
+      end
+
+      private
+
+      def build_job_hash(job, args: [], kwargs: {}, &blk)
+        hsh = {
+          job: job,
+          args: args,
+          kwargs: kwargs,
+        }
+        self.class.new(hsh).apply_block(&blk) if blk.present?
+        hsh
+      end
+
+      def find_matching_jobs(search_job, parent_job = self.base_job)
+        return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
+
+        sub_jobs = self.class.get_chain_parameter(parent_job)
+        sub_jobs.each_with_index do |sub_job, i|
+          if sub_job[:job].to_s == search_job.to_s
+            yield [sub_job, parent_job, i]
+          elsif self.class._job_type_definitions[sub_job[:job].to_s]
+            find_matching_jobs(search_job, sub_job) { |item| yield item }
+          end
+        end
+      end
+
+      def find_parent_job(job_def)
+        iterate_job_tree do |job, path|
+          return path[-1] if job == job_def
+        end
+        nil
+      end
+
+      def iterate_job_tree(root: self.base_job, path: [], &blk)
+        blk.call(root, path)
+
+        if self.class._job_type_definitions[root[:job]]
+          sub_jobs = self.class.get_chain_parameter(root)
+          sub_jobs.each_with_index do |sub_job, i|
+            iterate_job_tree(root: sub_job, path: [*path, root], &blk)
+          end
+        end
+      end
+
+      class << self
+        # Support builder syntaxt/DSL
+        # Chain.build(ConcurrentBatchJob) do
+        #   insert(SomeJob, arg1, kwarg: 1)
+        #   insert(SerialBatchJob) do
+        #     insert(SomeJob, arg1, kwarg: 1)
+        #   end
+        # end
+        def build(job, *args, **kwargs, &blk)
+          new(job).tap do |ch|
+            ch.base_job[:args] = args
+            ch.base_job[:kwargs] = kwargs
+            ch.apply_block(&blk)
+          end
+        end
+
+        def _job_type_definitions
+          @job_type_definitions ||= {}
+        end
+
+        def register_chain_job(job_class, chain_parameter, **options)
+          _job_type_definitions[job_class.to_s] = {
+            **options,
+            chain_parameter: chain_parameter,
+          }
+        end
+
+        def get_chain_parameter(job_def, raise_error: true)
+          unless _job_type_definitions[job_def[:job].to_s].present?
+            raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
+            return nil
+          end
+
+          key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
+          if key.is_a?(Numeric)
+            job_def[:args][key] ||= []
+          else
+            job_def[:kwargs][key] ||= []
+          end
+        end
+
+        # TODO: Add a Chain progress web View
+        # Augment Batch tree-view with Chain data
+        #   > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
+        #   > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
+        #     > Chain-jobs will be supplied chain_id and chain_step_id metadata
+        #     > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
+        #     > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
+        #   > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
+        def enqueue_job(job_def)
+          job_class = job_def[:job].constantize
+          job_args = job_def[:args] || job_def[:parameters] || []
+          job_kwargs = job_def[:kwargs] || {}
+
+          # Legacy Support
+          if job_def[:options]
+            job_args << {} unless job_args[-1].is_a?(Hash)
+            job_args[-1].merge!(job_def[:options])
+          end
+
+          if job_class.respond_to? :perform_async
+            job_class.perform_async(*job_args, **job_kwargs)
+          else
+            job_class.perform_later(*job_args, **job_kwargs)
+          end
+        end
+      end
+    end
+
+    ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
+    ChainBuilder.register_chain_job(SerialBatchJob, 0)
+  end
+end

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -0,0 +1,159 @@
+module CanvasSync
+  module JobBatches
+    class ContextHash
+      delegate_missing_to :flatten
+
+      def initialize(bid, hash = nil)
+        @bid_stack = [bid]
+        @hash_map = {}
+        @dirty = false
+        @flattened = nil
+        @hash_map[bid] = hash.with_indifferent_access if hash
+      end
+
+      # Local is "the nearest batch with a context value"
+      # This allows for, for example, SerialBatchJob to have a modifiable context stored on it's main Batch
+      # that can be accessed transparently from one of it's internal, context-less Batches
+      def local_bid
+        bid = @bid_stack[-1]
+        while bid.present?
+          bhash = resolve_hash(bid)
+          return bid if bhash
+          bid = get_parent_bid(bid)
+        end
+        nil
+      end
+
+      def local
+        @hash_map[local_bid]
+      end
+
+      def own
+        resolve_hash(@bid_stack[-1]) || {}
+      end
+
+      def set_local(new_hash)
+        @dirty = true
+        local.clear.merge!(new_hash)
+      end
+
+      def clear
+        local.clear
+        @flattened = nil
+        @dirty = true
+        self
+      end
+
+      def []=(key, value)
+        @flattened = nil
+        @dirty = true
+        local[key] = value
+      end
+
+      def [](key)
+        bid = @bid_stack[-1]
+        while bid.present?
+          bhash = resolve_hash(bid)
+          return bhash[key] if bhash&.key?(key)
+          bid = get_parent_bid(bid)
+        end
+        nil
+      end
+
+      def reload!
+        @dirty = false
+        @hash_map = {}
+        self
+      end
+
+      def save!(force: false)
+        return unless dirty? || force
+        Batch.redis do |r|
+          r.hset("BID-#{local_bid}", 'context', JSON.unparse(local))
+        end
+      end
+
+      def dirty?
+        @dirty
+      end
+
+      def is_a?(arg)
+        return true if Hash <= arg
+        super
+      end
+
+      def flatten
+        return @flattened if @flattened
+
+        load_all
+        flattened = {}
+        @bid_stack.compact.each do |bid|
+          flattened.merge!(@hash_map[bid]) if @hash_map[bid]
+        end
+        flattened.freeze
+
+        @flattened = flattened.with_indifferent_access
+      end
+
+      def to_h
+        flatten
+      end
+
+      private
+
+      def get_parent_hash(bid)
+        resolve_hash(get_parent_bid(bid)).freeze
+      end
+
+      def get_parent_bid(bid)
+        index = @bid_stack.index(bid)
+        raise "Invalid BID #{bid}" if index.nil? # Sanity Check - this shouldn't happen
+
+        index -= 1
+        if index >= 0
+          @bid_stack[index]
+        else
+          pbid = Batch.redis do |r|
+            callback_params = JSON.parse(r.hget("BID-#{bid}", "callback_params") || "{}")
+            callback_params['for_bid'] || r.hget("BID-#{bid}", "parent_bid")
+          end
+          @bid_stack.unshift(pbid)
+          pbid
+        end
+      end
+
+      def resolve_hash(bid)
+        return nil unless bid.present?
+        return @hash_map[bid] if @hash_map.key?(bid)
+
+        context_json, editable = Batch.redis do |r|
+          r.multi do |r|
+            r.hget("BID-#{bid}", "context")
+            r.hget("BID-#{bid}", "allow_context_changes")
+          end
+        end
+
+        if context_json.present?
+          context_hash = JSON.parse(context_json)
+          context_hash = context_hash.with_indifferent_access
+          context_hash.each do |k, v|
+            v.freeze
+          end
+          context_hash.freeze unless editable
+
+          @hash_map[bid] = context_hash
+        else
+          @hash_map[bid] = nil
+        end
+      end
+
+      def load_all
+        resolve_hash(@bid_stack[0]).freeze
+        while @bid_stack[0].present?
+          get_parent_hash(@bid_stack[0])
+        end
+        @hash_map
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/hier_batch_ids.lua

@@ -0,0 +1,25 @@
+
+local function add_bids(root, depth)
+    local result_data = {}
+
+    if depth > 0 then
+        local sbids
+        if depth == tonumber(ARGV[2]) and ARGV[4] then
+            local min, max = ARGV[4]:match('(%d+):(%d+)')
+            sbids = redis.call('ZRANGE', 'BID-' .. root .. '-bids', min, max)
+        else
+            sbids = redis.call('ZRANGE', 'BID-' .. root .. '-bids', 0, tonumber(ARGV[3]) - 1)
+        end
+
+        local sub_data = {}
+        for _,v in ipairs(sbids) do
+            table.insert(sub_data, add_bids(v, depth - 1))
+        end
+
+        return { root, sub_data }
+    end
+
+    return { root, result_data}
+end
+
+return add_bids(ARGV[1], tonumber(ARGV[2]))

data/lib/canvas_sync/job_batches/jobs/base_job.rb

@@ -0,0 +1,7 @@
+module CanvasSync
+  module JobBatches
+    class BaseJob < ::ActiveJob::Base
+
+    end
+  end
+end

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -0,0 +1,22 @@
+require_relative './base_job'
+
+module CanvasSync
+  module JobBatches
+    class ConcurrentBatchJob < BaseJob
+      def self.make_batch(sub_jobs, **kwargs, &blk)
+        ManagedBatchJob.make_batch(
+          sub_jobs,
+          **kwargs,
+          ordered: false,
+          concurrency: true,
+          desc_prefix: 'ConcurrentBatchJob: ',
+          &blk
+        )
+      end
+
+      def perform(sub_jobs, **kwargs)
+        self.class.make_batch(sub_jobs, **kwargs)
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -0,0 +1,170 @@
+require_relative './base_job'
+
+module CanvasSync
+  module JobBatches
+    class ManagedBatchJob < BaseJob
+      def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, preflight_check: nil, desc_prefix: nil, &blk)
+        desc_prefix ||= ''
+
+        if concurrency == 0 || concurrency == nil || concurrency == true
+          concurrency = sub_jobs.count
+        elsif concurrency == false
+          concurrency = 1
+        end
+
+        root_batch = Batch.new
+        man_batch_id = nil
+
+        if concurrency < sub_jobs.count
+          man_batch_id = SecureRandom.urlsafe_base64(10)
+
+          Batch.redis do |r|
+            r.multi do |r|
+              r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
+              r.hset("MNGBID-#{man_batch_id}", "ordered", ordered ? 1 : 0)
+              r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
+              r.hset("MNGBID-#{man_batch_id}", "preflight_check", preflight_check) if preflight_check.present?
+              r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
+
+              mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
+                j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
+                j = ::ActiveJob::Arguments.serialize([j])
+                JSON.unparse(j)
+              end
+              if ordered
+                r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+              else
+                r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+              end
+              r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
+            end
+          end
+
+          root_batch.allow_context_changes = (concurrency == 1)
+          root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
+
+          desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
+        end
+
+        root_batch.context = context
+
+        blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
+
+        root_batch.description = "#{desc_prefix}#{root_batch.description || 'Root'}"
+
+        root_batch.context["managed_batch_bid"] = man_batch_id if man_batch_id
+
+        if concurrency < sub_jobs.count
+          root_batch.jobs {}
+          concurrency.times do
+            perform_next_sequence_job(man_batch_id, skip_preflight: true)
+          end
+        else
+          root_batch.jobs do
+            sub_jobs.each do |j|
+              ChainBuilder.enqueue_job(j)
+            end
+          end
+        end
+
+        root_batch
+      end
+
+      def perform(sub_jobs, **kwargs)
+        self.class.make_batch(sub_jobs, **kwargs)
+      end
+
+      def self.cleanup_redis(status, options)
+        man_batch_id = options['managed_batch_id']
+        Batch.redis do |r|
+          r.del(
+            "MNGBID-#{man_batch_id}",
+            "MNGBID-#{man_batch_id}-jobs",
+          )
+        end
+      end
+
+      def self.job_succeeded_callback(status, options)
+        man_batch_id = options['managed_batch_id']
+        perform_next_sequence_job(man_batch_id)
+      end
+
+      protected
+
+      def self.perform_next_sequence_job(man_batch_id, skip_preflight: false)
+        root_bid, ordered, preflight_check = Batch.redis do |r|
+          r.multi do |r|
+            r.hget("MNGBID-#{man_batch_id}", "root_bid")
+            r.hget("MNGBID-#{man_batch_id}", "ordered")
+            r.hget("MNGBID-#{man_batch_id}", "preflight_check")
+          end
+        end
+
+        if !skip_preflight && preflight_check.present?
+          if preflight_check.include?(".")
+            clazz, method_name = preflight_check.split('.')
+            clazz = clazz.constantize
+          else
+            clazz = Object
+            method_name = preflight_check
+          end
+          preflight_check = ->(*args) { clazz.send(method_name, *args) }
+        else
+          preflight_check = ->(*args) { true }
+        end
+
+        ordered = CanvasSync::MiscHelper.to_boolean(ordered)
+
+        loop do
+          next_job_json = Batch.redis do |r|
+            if ordered
+              r.lpop("MNGBID-#{man_batch_id}-jobs")
+            else
+              r.spop("MNGBID-#{man_batch_id}-jobs")
+            end
+          end
+
+          break unless next_job_json.present?
+
+          next_job = JSON.parse(next_job_json)
+          next_job = ::ActiveJob::Arguments.deserialize(next_job)[0]
+
+          preflight_result = preflight_check.call(next_job)
+          if preflight_result == :abort
+            cleanup_redis(nil, { "managed_batch_id" => man_batch_id })
+            break
+          elsif !preflight_check
+            next
+          end
+
+          Batch.new(root_bid).jobs do
+            Batch.new.tap do |batch|
+              batch.description = "Managed Batch Fiber (#{man_batch_id})"
+              batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
+              batch.jobs do
+                ChainBuilder.enqueue_job(next_job)
+              end
+            end
+          end
+
+          break
+        end
+      end
+
+      class ManagedBatchProxy
+        def initialize(real_batch)
+          @real_batch = real_batch
+        end
+
+        delegate_missing_to :real_batch
+
+        def jobs
+          raise "Managed Batches do not support calling .jobs directly!"
+        end
+
+        private
+        attr_reader :real_batch
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -0,0 +1,22 @@
+require_relative './base_job'
+
+module CanvasSync
+  module JobBatches
+    class SerialBatchJob < BaseJob
+      def self.make_batch(sub_jobs, **kwargs, &blk)
+        ManagedBatchJob.make_batch(
+          sub_jobs,
+          **kwargs,
+          ordered: true,
+          concurrency: false,
+          desc_prefix: 'SerialBatchJob: ',
+          &blk
+        )
+      end
+
+      def perform(sub_jobs, **kwargs)
+        self.class.make_batch(sub_jobs, **kwargs)
+      end
+    end
+  end
+end