canvas_sync 0.23.5 → 0.24.0

data/lib/canvas_sync/job_batches/callback.rb

@@ -1,135 +0,0 @@
-module CanvasSync::JobBatches
-  class Batch
-    module Callback
-      mattr_accessor :worker_class
-
-      VALID_CALLBACKS = %i[success complete death stagnated].freeze
-
-      module CallbackWorkerCommon
-        def perform(definition, event, opts, bid, parent_bid)
-          return unless VALID_CALLBACKS.include?(event.to_sym)
-
-          method = nil
-          target = :instance
-          clazz = definition
-          if clazz.is_a?(String)
-            if clazz.include?('#')
-              clazz, method = clazz.split("#")
-            elsif clazz.include?('.')
-              clazz, method = clazz.split(".")
-              target = :class
-            end
-          end
-
-          method ||= "on_#{event}"
-          status = Batch::Status.new(bid)
-
-          if clazz && object = Object.const_get(clazz)
-            target = target == :instance ? object.new : object
-            if target.respond_to?(method, true)
-              target.send(method, status, opts.with_indifferent_access)
-            else
-              Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")
-            end
-          else
-            Batch.logger.warn("Invalid callback method #{definition} - Class #{clazz} not found")
-          end
-        end
-      end
-
-      class Finalize
-        # The methods in this class are called after all same-named callbacks have been
-        # completed for the passed Batch.
-        # These methods mainly handle bubbling events up to the parent Batch
-        # You could say that they are the callbacks for callbacks.
-
-        def dispatch(status, opts)
-          bid = opts["bid"]
-          event = opts["event"].to_sym
-
-          Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}"}
-
-          batch_status = Status.new bid
-          send(event, bid, batch_status, batch_status.parent_bid)
-
-          Batch.redis do |r|
-            r.srem("BID-#{bid}-pending_callbacks", "#{event}-finalize")
-          end
-
-          if event == :success
-            if opts['origin'].present?
-              # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
-              origin_bid = opts['origin']['for_bid']
-              _, pending, success_ran = Batch.redis do |r|
-                r.multi do |r|
-                  r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
-                  r.scard("BID-#{origin_bid}-pending_callbacks")
-                  r.hget("BID-#{origin_bid}", "success")
-                end
-              end
-              Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
-            end
-
-            if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
-              Batch.cleanup_redis(bid)
-            end
-          end
-        end
-
-        def success(bid, status, parent_bid)
-          return unless parent_bid
-
-          Batch.with_callback_check(parent_bid) do |r|
-            r.sadd("BID-#{parent_bid}-batches-success", bid)
-            r.srem("BID-#{parent_bid}-batches-failed", bid)
-            r.sadd("BID-#{parent_bid}-batches-complete", bid)
-          end
-        end
-
-        def complete(bid, status, parent_bid)
-          return unless parent_bid
-
-          pending, children, success = Batch.redis do |r|
-            r.multi do |r|
-              r.hincrby("BID-#{bid}", "pending", 0)
-              r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-batches-success")
-            end
-          end
-
-          if !(pending.to_i.zero? && children == success)
-            # If batch was not successfull check and see if its parent is complete
-            # if the parent is complete we can trigger its complete callback.
-            #
-            # Otherwise, we don't want to to trigger the parent's :complete here (and
-            # we instead opt to have success tigger parent :complete) - this
-            # allows the success callback to add additional jobs to the parent batch
-            # before triggering :complete.
-
-            Batch.with_callback_check(parent_bid, except: [:success]) do |r|
-              r.sadd("BID-#{parent_bid}-batches-complete", bid)
-              r.sadd("BID-#{parent_bid}-batches-failed", bid)
-            end
-          end
-        end
-
-        def death(bid, status, parent_bid)
-          return unless parent_bid
-
-          # We only need to bubble the event here - other events (eg stagnation) will be checked and bubbled elsewhere.
-
-          Batch.enqueue_callbacks(:death, parent_bid)
-        end
-
-        def stagnated(bid, status, parent_bid)
-          return unless parent_bid
-
-          Batch.with_callback_check(parent_bid) do |r|
-            r.sadd("BID-#{parent_bid}-batches-stagnated", bid)
-            r.expire("BID-#{parent_bid}-batches-stagnated", BID_EXPIRE_TTL)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -1,247 +0,0 @@
-module CanvasSync::JobBatches
-  class ChainBuilder
-    VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
-
-    attr_reader :base_job
-
-    def initialize(base_type = SerialBatchJob)
-      if base_type.is_a?(Hash)
-        @base_job = base_type
-        @base_job[:args] ||= @base_job[:parameters] || []
-        @base_job[:kwargs] ||= {}
-      else
-        @base_job = build_job_hash(base_type)
-      end
-
-      self.class.get_chain_parameter(base_job)
-    end
-
-    def process!
-      normalize!
-      self.class.enqueue_job(base_job)
-    end
-
-    def [](key)
-      if key.is_a?(Class)
-        get_sub_chain(key)
-      else
-        # Legacy Support
-        key = :args if key == :parameters
-
-        @base_job[key]
-      end
-    end
-
-    def args; return self[:args]; end
-    def kwargs; return self[:kwargs]; end
-
-    def <<(new_job)
-      insert_at(-1, new_job)
-    end
-
-    def insert_at(position, new_jobs, *args, **kwargs, &blk)
-      chain = self.class.get_chain_parameter(base_job)
-      if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
-        new_jobs = build_job_hash(new_jobs, args: args, kwargs: kwargs, &blk)
-      elsif args.count > 0 || kwargs.count > 0
-        raise "Unexpected number of arguments"
-      end
-      new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
-      chain.insert(position, *new_jobs)
-    end
-
-    def insert(new_jobs, *args, **kwargs, &blk)
-      if new_jobs.is_a?(Class) || new_jobs.is_a?(String)
-        job_kwargs = kwargs.except(*VALID_PLACEMENT_PARAMETERS)
-        new_jobs = build_job_hash(new_jobs, args: args, kwargs: job_kwargs, &blk)
-        kwargs = kwargs.slice(*VALID_PLACEMENT_PARAMETERS)
-      else
-        invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
-        raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
-        raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
-        raise "Unexpected number of arguments" if args.length > 0
-      end
-
-      new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
-
-      if !kwargs.present?
-        insert_at(-1, new_jobs)
-      else
-        placement = kwargs.keys[0]
-        relative_to = kwargs.values[0]
-
-        matching_jobs = find_matching_jobs(relative_to).to_a
-        raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
-        raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
-
-        relative_job, parent_job, sub_index = matching_jobs[0]
-        needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
-
-        chain = self.class.get_chain_parameter(parent_job)
-
-        if parent_job[:job] != needed_parent_type
-          old_job = chain[sub_index]
-          parent_job = chain[sub_index] = {
-            job: needed_parent_type,
-            parameters: [],
-          }
-          sub_index = 0
-          chain = self.class.get_chain_parameter(parent_job)
-          chain << old_job
-        end
-
-        if placement == :with
-          chain.insert(-1, *new_jobs)
-        else
-          sub_index += 1 if placement == :after
-          chain.insert(sub_index, *new_jobs)
-        end
-      end
-    end
-
-    def empty?
-      self.class.get_chain_parameter(self).empty?
-    end
-
-    def get_sub_chain(sub_type)
-      matching_jobs = find_matching_jobs(sub_type).to_a
-      raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
-      return nil if matching_jobs.count == 0
-
-      job = matching_jobs[0][0]
-      job = self.class.new(job) unless job.is_a?(ChainBuilder)
-      job
-    end
-
-    def normalize!(job_def = self.base_job)
-      if job_def.is_a?(ChainBuilder)
-        job_def.normalize!
-      else
-        job_def[:job] = job_def[:job].to_s
-        if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
-          chain.map! { |sub_job| normalize!(sub_job) }
-        end
-        job_def
-      end
-    end
-
-    def apply_block(&blk)
-      return unless blk.present?
-      instance_exec(&blk)
-    end
-
-    private
-
-    def build_job_hash(job, args: [], kwargs: {}, &blk)
-      hsh = {
-        job: job,
-        args: args,
-        kwargs: kwargs,
-      }
-      self.class.new(hsh).apply_block(&blk) if blk.present?
-      hsh
-    end
-
-    def find_matching_jobs(search_job, parent_job = self.base_job)
-      return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
-
-      sub_jobs = self.class.get_chain_parameter(parent_job)
-      sub_jobs.each_with_index do |sub_job, i|
-        if sub_job[:job].to_s == search_job.to_s
-          yield [sub_job, parent_job, i]
-        elsif self.class._job_type_definitions[sub_job[:job].to_s]
-          find_matching_jobs(search_job, sub_job) { |item| yield item }
-        end
-      end
-    end
-
-    def find_parent_job(job_def)
-      iterate_job_tree do |job, path|
-        return path[-1] if job == job_def
-      end
-      nil
-    end
-
-    def iterate_job_tree(root: self.base_job, path: [], &blk)
-      blk.call(root, path)
-
-      if self.class._job_type_definitions[root[:job]]
-        sub_jobs = self.class.get_chain_parameter(root)
-        sub_jobs.each_with_index do |sub_job, i|
-          iterate_job_tree(root: sub_job, path: [*path, root], &blk)
-        end
-      end
-    end
-
-    class << self
-      # Support builder syntaxt/DSL
-      # Chain.build(ConcurrentBatchJob) do
-      #   insert(SomeJob, arg1, kwarg: 1)
-      #   insert(SerialBatchJob) do
-      #     insert(SomeJob, arg1, kwarg: 1)
-      #   end
-      # end
-      def build(job, *args, **kwargs, &blk)
-        new(job).tap do |ch|
-          ch.base_job[:args] = args
-          ch.base_job[:kwargs] = kwargs
-          ch.apply_block(&blk)
-        end
-      end
-
-      def _job_type_definitions
-        @job_type_definitions ||= {}
-      end
-
-      def register_chain_job(job_class, chain_parameter, **options)
-        _job_type_definitions[job_class.to_s] = {
-          **options,
-          chain_parameter: chain_parameter,
-        }
-      end
-
-      def get_chain_parameter(job_def, raise_error: true)
-        unless _job_type_definitions[job_def[:job].to_s].present?
-          raise "Job Type #{job_def[:job].to_s} does not accept a sub-chain" if raise_error
-          return nil
-        end
-
-        key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
-        if key.is_a?(Numeric)
-          job_def[:args][key] ||= []
-        else
-          job_def[:kwargs][key] ||= []
-        end
-      end
-
-      # TODO: Add a Chain progress web View
-      # Augment Batch tree-view with Chain data
-      #   > [DONE] Tree view w/o Chain will only show Parent/Current batches and Job Counts
-      #   > If augmented with Chain data, the above will be annotated with Chain-related info and will be able to show Jobs defined in the Chain
-      #     > Chain-jobs will be supplied chain_id and chain_step_id metadata
-      #     > Using server-middleware, if a Chain-job (has chain_id and chain_step_id) creates a Batch, tag the Batch w/ the chain_id and chain_step_id
-      #     > UI will map Batches to Chain-steps using the chain_step_id. UI will add entries for any Chain-steps that were not tied to a Batch
-      #   > [DONE] Use a Lua script to find child batch IDs. Support max_depth, items_per_depth, top_depth_slice parameters
-      def enqueue_job(job_def)
-        job_class = job_def[:job].constantize
-        job_args = job_def[:args] || job_def[:parameters] || []
-        job_kwargs = job_def[:kwargs] || {}
-
-        # Legacy Support
-        if job_def[:options]
-          job_args << {} unless job_args[-1].is_a?(Hash)
-          job_args[-1].merge!(job_def[:options])
-        end
-
-        if job_class.respond_to? :perform_async
-          job_class.perform_async(*job_args, **job_kwargs)
-        else
-          job_class.perform_later(*job_args, **job_kwargs)
-        end
-      end
-    end
-  end
-
-  ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
-  ChainBuilder.register_chain_job(SerialBatchJob, 0)
-end

data/lib/canvas_sync/job_batches/compat/active_job.rb

@@ -1,108 +0,0 @@
-
-module CanvasSync::JobBatches
-  module Compat
-    module ActiveJob
-      module BatchAwareJob
-        extend ActiveSupport::Concern
-
-        included do
-          around_perform do |job, block|
-            if (@bid) # This _must_ be @bid - not just bid
-              prev_batch = Thread.current[CURRENT_BATCH_THREAD_KEY]
-              begin
-                Thread.current[CURRENT_BATCH_THREAD_KEY] = Batch.new(@bid)
-                block.call
-                Thread.current[CURRENT_BATCH_THREAD_KEY].save_context_changes
-                Batch.process_successful_job(@bid, job_id)
-              rescue
-                Batch.process_failed_job(@bid, job_id)
-                raise
-              ensure
-                Thread.current[CURRENT_BATCH_THREAD_KEY] = prev_batch
-              end
-            else
-              block.call
-            end
-          end
-
-          around_enqueue do |job, block|
-            if (batch = Thread.current[CURRENT_BATCH_THREAD_KEY])
-              @bid = batch.bid
-              batch.append_jobs(job_id) if @bid
-            end
-            block.call
-          end
-        end
-
-        def bid
-          @bid || Thread.current[CURRENT_BATCH_THREAD_KEY]&.bid
-        end
-
-        def batch
-          Thread.current[CURRENT_BATCH_THREAD_KEY]
-        end
-
-        def batch_context
-          batch&.context || {}
-        end
-
-        def valid_within_batch?
-          batch.valid?
-        end
-
-        def serialize
-          super.tap do |data|
-            data['batch_id'] = @bid # This _must_ be @bid - not just bid
-            data
-          end
-        end
-
-        def deserialize(data)
-          super
-          @bid = data['batch_id']
-        end
-      end
-
-      class ActiveJobCallbackWorker < ::ActiveJob::Base
-        include Batch::Callback::CallbackWorkerCommon
-
-        def self.enqueue_all(args, queue)
-          args.each do |arg_set|
-            set(queue: queue).perform_later(*arg_set)
-          end
-        end
-      end
-
-      def self.handle_job_death(job, error = nil)
-        if job.is_a?(Array)
-          event = ActiveSupport::Notifications::Event.new(*job)
-          payload = event.payload
-          job = payload[:job].serialize
-          error = payload[:error]
-        end
-
-        if job["job_id"].present? && job["batch_id"].present?
-          CanvasSync::JobBatches::Batch.process_dead_job(job['batch_id'], job['job_id'])
-        end
-      end
-
-      def self.configure
-        ::ActiveJob::Base.include BatchAwareJob
-
-        begin
-          ActiveSupport::Notifications.subscribe "discard.active_job" do |*args|
-            handle_job_death(args)
-          end
-
-          ActiveSupport::Notifications.subscribe "retry_stopped.active_job" do |*args|
-            handle_job_death(args)
-          end
-        rescue => err
-          Rails.logger.warn(err)
-        end
-
-        Batch::Callback.worker_class ||= ActiveJobCallbackWorker
-      end
-    end
-  end
-end

data/lib/canvas_sync/job_batches/compat/sidekiq/web/batches_assets/css/styles.less

@@ -1,182 +0,0 @@
-
-@color-green: #25c766;
-@color-darkred: #8f0000;
-@color-red: #e03963;
-@color-yellow: #c4c725;
-
-.code-wrap.batch-context .args-extended {
-    white-space: pre;
-
-    .key {
-        white-space: pre-wrap;
-        margin-left: 2em;
-        text-indent: -2em;
-        display: inline-block;
-    }
-
-    .own {
-        color: @color-green;
-    }
-}
-
-
-.batch-tree {
-    .status-block {
-        .tree-stat {
-            margin: 0 4px;
-    
-            &.pending {
-                color: @color-yellow;
-            }
-            &.failed {
-                color: @color-red;
-            }
-            &.dead {
-                color: @color-darkred;
-            }
-            &.success {
-                color: @color-green;
-            }
-            &.total {
-    
-            }
-        }
-    }
-
-    .text-inactive {
-        color: darken(#fff, 35%);
-        font-size: 80%;
-    }
-
-    .tree-header {
-        position: relative;
-
-        .status-block {
-            position: absolute;
-            bottom: 0;
-            width: 100%;
-
-            margin-right: 8px;
-            font-size: 90%;
-            text-align: right;
-
-            .tree-stat {
-                font-style: italic;
-            }
-        }
-    }
-
-    .tree-entry {
-        > .header {
-            display: flex;
-            align-items: center;
-
-            .header-inner {
-                padding: 4px 0;
-                border-bottom: 1px dashed white;
-                display: flex;
-                align-items: center;
-                flex: 1;
-            }
-
-            &:hover {
-                background-color: rgba(0,0,0,0.20);
-                border-radius: 3px;
-            }
-
-            .row-toggle {
-                width: 16px;
-                height: 16px;
-                text-align: center;
-                align-self: center;
-                border-radius: 50%;
-                border: 1px solid #999;
-                text-decoration: none;
-                margin: 0 4px;
-                font-size: 16px;
-                line-height: 15px;
-
-                &.not_applicable {
-                    opacity: 0;
-                    pointer-events: none;
-                }
-            }
-
-            .main {
-                flex: 1;
-                display: flex;
-                align-items: baseline;
-
-                .bid {
-                    font-family: monospace;
-                    padding: 3px 6px;
-                    background: rgba(0,0,0,0.2);
-                    border-radius: 3px;
-                    font-size: 12px;
-                    margin: 0 12px 0 0;
-
-                    &:hover {
-                        .bid-goto {
-                            display: inline-block;
-                            padding: 0 0 0 4px;
-                            font-size: 200%;
-                            line-height: 10px;
-                            vertical-align: sub;
-                            text-decoration: dotted;
-                        }
-                    }
-
-                    .bid-goto {
-                        display: none;
-                    }
-                }
-            }
-
-            .goto-link {
-                margin: 0 8px;
-                display: inline-block;
-                height: 16px;
-                font-size: 90%;
-                border-bottom: 1px dotted white;
-            }
-
-            .status-label {
-                font-family: monospace;
-                padding: 3px 6px;
-                background: rgba(0,0,0,0.2);
-                border-radius: 3px;
-                font-size: 12px;
-                margin: 0 12px 0 0;
-
-                &.deleted {
-                    background: #99999933;
-                }
-                &.failed, &.complete {
-                    background: #99000033;
-                }
-                &.success {
-                    background: #00990033;
-                }
-            }
-
-            .status-block {
-                width: 12em;
-                text-align: center;
-            }
-        }
-
-        > .subitems {
-            padding-left: 16px;
-
-            >.load-more {
-                padding: 4px 0;
-                text-align: center;
-                border-bottom: 1px dashed white;
-                a {
-                    border-bottom: 1px dotted white;
-                    text-decoration: none;
-                }
-            }
-        }
-    }
-}