canvas_sync 0.16.5 → 0.17.0.beta1

data/lib/canvas_sync/job_batches/batch.rb

@@ -0,0 +1,399 @@
+
+begin
+  require 'sidekiq'
+rescue LoadError
+end
+
+require_relative './batch_aware_job'
+require_relative "./callback"
+require_relative "./context_hash"
+require_relative "./status"
+Dir[File.dirname(__FILE__) + "/jobs/*.rb"].each { |file| require file }
+require_relative "./chain_builder"
+
+# Implement Job Batching similar to Sidekiq::Batch. Supports ActiveJob and Sidekiq, or a mix thereof.
+# Much of this code is modifed/extended from https://github.com/breamware/sidekiq-batch
+
+module CanvasSync
+  module JobBatches
+    class Batch
+      class NoBlockGivenError < StandardError; end
+
+      def self.batch_attr(key, read_only: true)
+        class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          def #{key}=(value)
+            raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
+            @#{key} = value
+            persist_bid_attr('#{key}', value)
+          end
+
+          def #{key}
+            return @#{key} if defined?(@#{key})
+            if (@initialized || @existing)
+              @#{key} = read_bid_attr('#{key}')
+            end
+          end
+        RUBY
+      end
+
+      delegate :redis, to: :class
+
+      BID_EXPIRE_TTL = 2_592_000
+
+      attr_reader :bid
+
+      def initialize(existing_bid = nil)
+        @bid = existing_bid || SecureRandom.urlsafe_base64(10)
+        @existing = !(!existing_bid || existing_bid.empty?)  # Basically existing_bid.present?
+        @initialized = false
+        @bidkey = "BID-" + @bid.to_s
+        @pending_attrs = {}
+        @ready_to_queue = []
+        self.created_at = Time.now.utc.to_f unless @existing
+      end
+
+      batch_attr :description
+      batch_attr :created_at
+      batch_attr :callback_queue, read_only: false
+      batch_attr :callback_batch, read_only: false
+      batch_attr :allow_context_changes
+
+      def context
+        return @context if defined?(@context)
+
+        if (@initialized || @existing)
+          @context = ContextHash.new(bid)
+        else
+          @context = ContextHash.new(bid, {})
+        end
+      end
+
+      def context=(value)
+        raise "context is read-only once the batch has been started" if (@initialized || @existing) # && !allow_context_changes
+        raise "context must be a Hash" unless value.is_a?(Hash) || value.nil?
+        return nil if value.nil? && @context.nil?
+
+        value = {} if value.nil?
+        value = value.local if value.is_a?(ContextHash)
+
+        @context ||= ContextHash.new(bid, {})
+        @context.set_local(value)
+        # persist_bid_attr('context', JSON.unparse(@context.local))
+      end
+
+      def save_context_changes
+        @context&.save!
+      end
+
+      def on(event, callback, options = {})
+        return unless Callback::VALID_CALLBACKS.include?(event.to_s)
+        callback_key = "#{@bidkey}-callbacks-#{event}"
+        redis do |r|
+          r.multi do
+            r.sadd(callback_key, JSON.unparse({
+              callback: callback,
+              opts: options
+            }))
+            r.expire(callback_key, BID_EXPIRE_TTL)
+          end
+        end
+      end
+
+      def jobs
+        raise NoBlockGivenError unless block_given?
+
+        if !@existing && !@initialized
+          parent_bid = Thread.current[:batch]&.bid
+
+          redis do |r|
+            r.multi do
+              r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
+              r.expire(@bidkey, BID_EXPIRE_TTL)
+            end
+          end
+
+          flush_pending_attrs
+          @context&.save!
+
+          @initialized = true
+        end
+
+        @ready_to_queue = []
+
+        begin
+          parent = Thread.current[:batch]
+          Thread.current[:batch] = self
+          yield
+        ensure
+          Thread.current[:batch] = parent
+        end
+
+        redis do |r|
+          r.multi do
+            if parent_bid
+              r.hincrby("BID-#{parent_bid}", "children", 1)
+              r.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
+              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            end
+
+            r.hincrby(@bidkey, "pending", @ready_to_queue.size)
+            r.hincrby(@bidkey, "total", @ready_to_queue.size)
+            r.expire(@bidkey, BID_EXPIRE_TTL)
+
+            if @ready_to_queue.size > 0
+              r.sadd(@bidkey + "-jids", @ready_to_queue)
+              r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
+            end
+          end
+        end
+
+        @ready_to_queue
+      end
+
+      def increment_job_queue(jid)
+        @ready_to_queue << jid
+      end
+
+      def invalidate_all
+        redis do |r|
+          r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
+        end
+      end
+
+      def parent_bid
+        redis do |r|
+          r.hget(@bidkey, "parent_bid")
+        end
+      end
+
+      def parent
+        if parent_bid
+          Batch.new(parent_bid)
+        end
+      end
+
+      def valid?(batch = self)
+        valid = !redis { |r| r.exists("invalidated-bid-#{batch.bid}") }
+        batch.parent ? valid && valid?(batch.parent) : valid
+      end
+
+      # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
+      def self.without_batch
+        parent = Thread.current[:batch]
+        Thread.current[:batch] = nil
+        yield
+      ensure
+        Thread.current[:batch] = parent
+      end
+
+      private
+
+      def persist_bid_attr(attribute, value)
+        if @initialized || @existing
+          redis do |r|
+            r.multi do
+              r.hset(@bidkey, attribute, value)
+              r.expire(@bidkey, BID_EXPIRE_TTL)
+            end
+          end
+        else
+          @pending_attrs[attribute] = value
+        end
+      end
+
+      def read_bid_attr(attribute)
+        redis do |r|
+          r.hget(@bidkey, attribute)
+        end
+      end
+
+      def flush_pending_attrs
+        redis do |r|
+          r.mapped_hmset(@bidkey, @pending_attrs)
+        end
+        @pending_attrs = {}
+      end
+
+      class << self
+        def process_failed_job(bid, jid)
+          _, pending, failed, children, complete, parent_bid = redis do |r|
+            r.multi do
+              r.sadd("BID-#{bid}-failed", jid)
+
+              r.hincrby("BID-#{bid}", "pending", 0)
+              r.scard("BID-#{bid}-failed")
+              r.hincrby("BID-#{bid}", "children", 0)
+              r.scard("BID-#{bid}-complete")
+              r.hget("BID-#{bid}", "parent_bid")
+
+              r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
+            end
+          end
+
+          # if the batch failed, and has a parent, update the parent to show one pending and failed job
+          if parent_bid
+            redis do |r|
+              r.multi do
+                r.hincrby("BID-#{parent_bid}", "pending", 1)
+                r.sadd("BID-#{parent_bid}-failed", jid)
+                r.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL)
+              end
+            end
+          end
+
+          if pending.to_i == failed.to_i && children == complete
+            enqueue_callbacks(:complete, bid)
+          end
+        end
+
+        def process_dead_job(bid, jid)
+          _, failed, children, complete, parent_bid = redis do |r|
+            r.multi do
+              r.sadd("BID-#{bid}-dead", jid)
+
+              r.scard("BID-#{bid}-dead")
+              r.hincrby("BID-#{bid}", "children", 0)
+              r.scard("BID-#{bid}-complete")
+              r.hget("BID-#{bid}", "parent_bid")
+
+              r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
+            end
+          end
+
+          if parent_bid
+            redis do |r|
+              r.multi do
+                r.sadd("BID-#{parent_bid}-dead", jid)
+                r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
+              end
+            end
+          end
+
+          enqueue_callbacks(:dead, bid)
+        end
+
+        def process_successful_job(bid, jid)
+          failed, pending, children, complete, success, total, parent_bid = redis do |r|
+            r.multi do
+              r.scard("BID-#{bid}-failed")
+              r.hincrby("BID-#{bid}", "pending", -1)
+              r.hincrby("BID-#{bid}", "children", 0)
+              r.scard("BID-#{bid}-complete")
+              r.scard("BID-#{bid}-success")
+              r.hget("BID-#{bid}", "total")
+              r.hget("BID-#{bid}", "parent_bid")
+
+              r.srem("BID-#{bid}-failed", jid)
+              r.srem("BID-#{bid}-jids", jid)
+              r.expire("BID-#{bid}", BID_EXPIRE_TTL)
+            end
+          end
+
+          all_success = pending.to_i.zero? && children == success
+          # if complete or successfull call complete callback (the complete callback may then call successful)
+          if (pending.to_i == failed.to_i && children == complete) || all_success
+            enqueue_callbacks(:complete, bid)
+            enqueue_callbacks(:success, bid) if all_success
+          end
+        end
+
+        def enqueue_callbacks(event, bid)
+          batch_key = "BID-#{bid}"
+          callback_key = "#{batch_key}-callbacks-#{event}"
+          already_processed, _, callbacks, queue, parent_bid, callback_batch = redis do |r|
+            r.multi do
+              r.hget(batch_key, event)
+              r.hset(batch_key, event, true)
+              r.smembers(callback_key)
+              r.hget(batch_key, "callback_queue")
+              r.hget(batch_key, "parent_bid")
+              r.hget(batch_key, "callback_batch")
+            end
+          end
+
+          return if already_processed == 'true'
+
+          queue ||= "default"
+          parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
+          callback_args = callbacks.reduce([]) do |memo, jcb|
+            cb = JSON.load(jcb)
+            memo << [cb['callback'], event.to_s, cb['opts'], bid, parent_bid]
+          end
+
+          opts = {"bid" => bid, "event" => event}
+
+          # Run callback batch finalize synchronously
+          if callback_batch
+            # Extract opts from cb_args or use current
+            # Pass in stored event as callback finalize is processed on complete event
+            cb_opts = callback_args.first&.at(2) || opts
+
+            logger.debug {"Run callback batch bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+            # Finalize now
+            finalizer = Batch::Callback::Finalize.new
+            status = Status.new bid
+            finalizer.dispatch(status, cb_opts)
+
+            return
+          end
+
+          logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+
+          if callback_args.empty?
+            # Finalize now
+            finalizer = Batch::Callback::Finalize.new
+            status = Status.new bid
+            finalizer.dispatch(status, opts)
+          else
+            # Otherwise finalize in sub batch complete callback
+            cb_batch = self.new
+            cb_batch.callback_batch = true
+            logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
+            cb_batch.on(:complete, "#{Batch::Callback::Finalize.to_s}#dispatch", opts)
+            cb_batch.jobs do
+              push_callbacks callback_args, queue
+            end
+          end
+        end
+
+        def cleanup_redis(bid)
+          logger.debug {"Cleaning redis of batch #{bid}"}
+          redis do |r|
+            r.del(
+              "BID-#{bid}",
+              "BID-#{bid}-callbacks-complete",
+              "BID-#{bid}-callbacks-success",
+              "BID-#{bid}-failed",
+
+              "BID-#{bid}-success",
+              "BID-#{bid}-complete",
+              "BID-#{bid}-jids",
+            )
+          end
+        end
+
+        def redis(*args, &blk)
+          defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
+        end
+
+        def logger
+          defined?(::Sidekiq) ? ::Sidekiq.logger : Rails.logger
+        end
+
+        private
+
+        def push_callbacks(args, queue)
+          Batch::Callback::Worker.enqueue_all(args, queue)
+        end
+      end
+    end
+
+    ActiveJob::Base.include BatchAwareJob
+  end
+end
+
+# Automatically integrate with Sidekiq if it is present.
+if defined?(::Sidekiq)
+  require_relative './sidekiq'
+  CanvasSync::JobBatches::Sidekiq.configure
+end

data/lib/canvas_sync/job_batches/batch_aware_job.rb

@@ -0,0 +1,62 @@
+module CanvasSync
+  module JobBatches
+    module BatchAwareJob
+      extend ActiveSupport::Concern
+
+      included do
+        around_perform do |job, block|
+          if (@bid) # This _must_ be @bid - not just bid
+            prev_batch = Thread.current[:batch]
+            begin
+              Thread.current[:batch] = Batch.new(@bid)
+              block.call
+              batch&.save_context_changes
+              Batch.process_successful_job(@bid, job_id)
+            rescue
+              Batch.process_failed_job(@bid, job_id)
+              raise
+            ensure
+              Thread.current[:batch] = prev_batch
+            end
+          else
+            block.call
+          end
+        end
+
+        around_enqueue do |job, block|
+          if (batch = Thread.current[:batch])
+            batch.increment_job_queue(job_id) if (@bid = batch.bid)
+          end
+          block.call
+        end
+      end
+
+      def bid
+        @bid || Thread.current[:batch]&.bid
+      end
+
+      def batch
+        Thread.current[:batch]
+      end
+
+      def batch_context
+        batch&.context || {}
+      end
+
+      def valid_within_batch?
+        batch.valid?
+      end
+
+      def serialize
+        super.tap do |data|
+          data['batch_id'] = @bid # This _must_ be @bid - not just bid
+        end
+      end
+
+      def deserialize(data)
+        super
+        @bid = data['batch_id']
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/callback.rb

@@ -0,0 +1,153 @@
+module CanvasSync
+  module JobBatches
+    class Batch
+      module Callback
+
+        VALID_CALLBACKS = %w[success complete dead].freeze
+
+        module CallbackWorkerCommon
+          def perform(definition, event, opts, bid, parent_bid)
+            return unless VALID_CALLBACKS.include?(event)
+
+            method = nil
+            target = :instance
+            clazz = definition
+            if clazz.is_a?(String)
+              if clazz.include?('#')
+                clazz, method = clazz.split("#")
+              elsif clazz.include?('.')
+                clazz, method = clazz.split(".")
+                target = :class
+              end
+            end
+
+            method ||= "on_#{event}"
+            status = Batch::Status.new(bid)
+
+            if clazz && object = Object.const_get(clazz)
+              target = target == :instance ? object.new : object
+              if target.respond_to?(method)
+                target.send(method, status, opts)
+              else
+                Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")
+              end
+            else
+              Batch.logger.warn("Invalid callback method #{definition} - Class #{clazz} not found")
+            end
+          end
+        end
+
+        class ActiveJobCallbackWorker < ActiveJob::Base
+          include CallbackWorkerCommon
+
+          def self.enqueue_all(args, queue)
+            args.each do |arg_set|
+              set(queue: queue).perform_later(*arg_set)
+            end
+          end
+        end
+
+        if defined?(::Sidekiq)
+          class SidekiqCallbackWorker
+            include ::Sidekiq::Worker
+            include CallbackWorkerCommon
+
+            def self.enqueue_all(args, queue)
+              return if args.empty?
+
+              ::Sidekiq::Client.push_bulk(
+                'class' => self,
+                'args' => args,
+                'queue' => queue
+              )
+            end
+          end
+          Worker = SidekiqCallbackWorker
+        else
+          Worker = ActiveJobCallbackWorker
+        end
+
+        class Finalize
+          def dispatch status, opts
+            bid = opts["bid"]
+            callback_bid = status.bid
+            event = opts["event"].to_sym
+            callback_batch = bid != callback_bid
+
+            Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
+
+            batch_status = Status.new bid
+            send(event, bid, batch_status, batch_status.parent_bid)
+
+            # Different events are run in different callback batches
+            Batch.cleanup_redis callback_bid if callback_batch
+            Batch.cleanup_redis bid if event == :success
+          end
+
+          def success(bid, status, parent_bid)
+            return unless parent_bid
+
+            _, _, success, _, complete, pending, children, failure = Batch.redis do |r|
+              r.multi do
+                r.sadd("BID-#{parent_bid}-success", bid)
+                r.expire("BID-#{parent_bid}-success", Batch::BID_EXPIRE_TTL)
+                r.scard("BID-#{parent_bid}-success")
+                r.sadd("BID-#{parent_bid}-complete", bid)
+                r.scard("BID-#{parent_bid}-complete")
+                r.hincrby("BID-#{parent_bid}", "pending", 0)
+                r.hincrby("BID-#{parent_bid}", "children", 0)
+                r.scard("BID-#{parent_bid}-failed")
+              end
+            end
+            # if job finished successfully and parent batch completed call parent complete callback
+            # Success callback is called after complete callback
+            if complete == children && pending == failure
+              Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
+              Batch.enqueue_callbacks(:complete, parent_bid)
+            end
+          end
+
+          def complete(bid, status, parent_bid)
+            pending, children, success = Batch.redis do |r|
+              r.multi do
+                r.hincrby("BID-#{bid}", "pending", 0)
+                r.hincrby("BID-#{bid}", "children", 0)
+                r.scard("BID-#{bid}-success")
+              end
+            end
+
+            # if we batch was successful run success callback
+            if pending.to_i.zero? && children == success
+              Batch.enqueue_callbacks(:success, bid)
+
+            elsif parent_bid
+              # if batch was not successfull check and see if its parent is complete
+              # if the parent is complete we trigger the complete callback
+              # We don't want to run this if the batch was successfull because the success
+              # callback may add more jobs to the parent batch
+
+              Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
+              _, complete, pending, children, failure = Batch.redis do |r|
+                r.multi do
+                  r.sadd("BID-#{parent_bid}-complete", bid)
+                  r.scard("BID-#{parent_bid}-complete")
+                  r.hincrby("BID-#{parent_bid}", "pending", 0)
+                  r.hincrby("BID-#{parent_bid}", "children", 0)
+                  r.scard("BID-#{parent_bid}-failed")
+                end
+              end
+              if complete == children && pending == failure
+                Batch.enqueue_callbacks(:complete, parent_bid)
+              end
+            end
+          end
+
+          def cleanup_redis bid, callback_bid=nil
+            Batch.cleanup_redis bid
+            Batch.cleanup_redis callback_bid if callback_bid
+          end
+        end
+      end
+    end
+  end
+end