canvas_sync 0.16.2 → 0.17.0.beta3

data/lib/canvas_sync/concerns/api_syncable.rb

@@ -29,6 +29,33 @@ module CanvasSync::Concerns
         end
       end
 
+      def bulk_sync_from_api_result(api_array, conflict_target: :canvas_id, import_args: {}, all_pages: true, batch_size: 1000)
+        columns = api_sync_options.keys
+
+        update_conditions = {
+          condition: Importers::BulkImporter.condition_sql(self, columns),
+          columns: columns,
+        }
+        update_conditions[:conflict_target] = conflict_target if conflict_target.present?
+        options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
+
+        if all_pages
+          batcher = BatchProcessor.new(of: batch_size) do |batch|
+            import(columns, batch, options)
+          end
+          api_array.all_pages_each do |api_item|
+            item = new.assign_from_api_params(api_items)
+            batcher << item
+          end
+          batcher.flush
+        else
+          items = api_array.map do |api_item|
+            new.assign_from_api_params(api_items)
+          end
+          import(columns, batch, options)
+        end
+      end
+
       def api_sync_options=(opts)
         @api_sync_options = opts
       end

data/lib/canvas_sync/job.rb

@@ -3,6 +3,8 @@ require "active_job"
 module CanvasSync
   # Inherit from this class to build a Job that will log to the canvas_sync_job_logs table
   class Job < ActiveJob::Base
+    attr_reader :job_log
+
     before_enqueue do |job|
       create_job_log(job)
     end
@@ -13,8 +15,6 @@ module CanvasSync
       @job_log.started_at = Time.now
       @job_log.save
 
-      @job_chain = job.arguments[0] if job.arguments[0].is_a?(Hash) && job.arguments[0].include?(:jobs)
-
       begin
         block.call
         @job_log.status = JobLog::SUCCESS_STATUS
@@ -22,11 +22,11 @@ module CanvasSync
         @job_log.exception = "#{e.class}: #{e.message}"
         @job_log.backtrace = e.backtrace.join('\n')
         @job_log.status = JobLog::ERROR_STATUS
-        if @job_chain&.[](:global_options)&.[](:on_failure)&.present?
+        if batch_context&.[](:on_failure)&.present?
           begin
-            class_name, method = @job_chain[:global_options][:on_failure].split('.')
+            class_name, method = batch_context[:on_failure].split('.')
             klass = class_name.constantize
-            klass.send(method.to_sym, e, job_chain: @job_chain, job_log: @job_log)
+            klass.send(method.to_sym, e, batch_context: batch_context, job_log: @job_log)
           rescue => e2
             @job_log.backtrace += "\n\nError Occurred while handling an Error: #{e2.class}: #{e2.message}"
             @job_log.backtrace += "\n" + e2.backtrace.join('\n')

data/lib/canvas_sync/job_batches/batch.rb

@@ -0,0 +1,399 @@
+
+begin
+  require 'sidekiq'
+rescue LoadError
+end
+
+require_relative './batch_aware_job'
+require_relative "./callback"
+require_relative "./context_hash"
+require_relative "./status"
+Dir[File.dirname(__FILE__) + "/jobs/*.rb"].each { |file| require file }
+require_relative "./chain_builder"
+
+# Implement Job Batching similar to Sidekiq::Batch. Supports ActiveJob and Sidekiq, or a mix thereof.
+# Much of this code is modifed/extended from https://github.com/breamware/sidekiq-batch
+
+module CanvasSync
+  module JobBatches
+    class Batch
+      class NoBlockGivenError < StandardError; end
+
+      def self.batch_attr(key, read_only: true)
+        class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          def #{key}=(value)
+            raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
+            @#{key} = value
+            persist_bid_attr('#{key}', value)
+          end
+
+          def #{key}
+            return @#{key} if defined?(@#{key})
+            if (@initialized || @existing)
+              @#{key} = read_bid_attr('#{key}')
+            end
+          end
+        RUBY
+      end
+
+      delegate :redis, to: :class
+
+      BID_EXPIRE_TTL = 2_592_000
+
+      attr_reader :bid
+
+      def initialize(existing_bid = nil)
+        @bid = existing_bid || SecureRandom.urlsafe_base64(10)
+        @existing = !(!existing_bid || existing_bid.empty?)  # Basically existing_bid.present?
+        @initialized = false
+        @bidkey = "BID-" + @bid.to_s
+        @pending_attrs = {}
+        @ready_to_queue = []
+        self.created_at = Time.now.utc.to_f unless @existing
+      end
+
+      batch_attr :description
+      batch_attr :created_at
+      batch_attr :callback_queue, read_only: false
+      batch_attr :callback_batch, read_only: false
+      batch_attr :allow_context_changes
+
+      def context
+        return @context if defined?(@context)
+
+        if (@initialized || @existing)
+          @context = ContextHash.new(bid)
+        else
+          @context = ContextHash.new(bid, {})
+        end
+      end
+
+      def context=(value)
+        raise "context is read-only once the batch has been started" if (@initialized || @existing) # && !allow_context_changes
+        raise "context must be a Hash" unless value.is_a?(Hash) || value.nil?
+        return nil if value.nil? && @context.nil?
+
+        value = {} if value.nil?
+        value = value.local if value.is_a?(ContextHash)
+
+        @context ||= ContextHash.new(bid, {})
+        @context.set_local(value)
+        # persist_bid_attr('context', JSON.unparse(@context.local))
+      end
+
+      def save_context_changes
+        @context&.save!
+      end
+
+      def on(event, callback, options = {})
+        return unless Callback::VALID_CALLBACKS.include?(event.to_s)
+        callback_key = "#{@bidkey}-callbacks-#{event}"
+        redis do |r|
+          r.multi do
+            r.sadd(callback_key, JSON.unparse({
+              callback: callback,
+              opts: options
+            }))
+            r.expire(callback_key, BID_EXPIRE_TTL)
+          end
+        end
+      end
+
+      def jobs
+        raise NoBlockGivenError unless block_given?
+
+        if !@existing && !@initialized
+          parent_bid = Thread.current[:batch]&.bid
+
+          redis do |r|
+            r.multi do
+              r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
+              r.expire(@bidkey, BID_EXPIRE_TTL)
+            end
+          end
+
+          flush_pending_attrs
+          @context&.save!
+
+          @initialized = true
+        end
+
+        @ready_to_queue = []
+
+        begin
+          parent = Thread.current[:batch]
+          Thread.current[:batch] = self
+          yield
+        ensure
+          Thread.current[:batch] = parent
+        end
+
+        redis do |r|
+          r.multi do
+            if parent_bid
+              r.hincrby("BID-#{parent_bid}", "children", 1)
+              r.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
+              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            end
+
+            r.hincrby(@bidkey, "pending", @ready_to_queue.size)
+            r.hincrby(@bidkey, "total", @ready_to_queue.size)
+            r.expire(@bidkey, BID_EXPIRE_TTL)
+
+            if @ready_to_queue.size > 0
+              r.sadd(@bidkey + "-jids", @ready_to_queue)
+              r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
+            end
+          end
+        end
+
+        @ready_to_queue
+      end
+
+      def increment_job_queue(jid)
+        @ready_to_queue << jid
+      end
+
+      def invalidate_all
+        redis do |r|
+          r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
+        end
+      end
+
+      def parent_bid
+        redis do |r|
+          r.hget(@bidkey, "parent_bid")
+        end
+      end
+
+      def parent
+        if parent_bid
+          Batch.new(parent_bid)
+        end
+      end
+
+      def valid?(batch = self)
+        valid = !redis { |r| r.exists?("invalidated-bid-#{batch.bid}") }
+        batch.parent ? valid && valid?(batch.parent) : valid
+      end
+
+      # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
+      def self.without_batch
+        parent = Thread.current[:batch]
+        Thread.current[:batch] = nil
+        yield
+      ensure
+        Thread.current[:batch] = parent
+      end
+
+      private
+
+      def persist_bid_attr(attribute, value)
+        if @initialized || @existing
+          redis do |r|
+            r.multi do
+              r.hset(@bidkey, attribute, value)
+              r.expire(@bidkey, BID_EXPIRE_TTL)
+            end
+          end
+        else
+          @pending_attrs[attribute] = value
+        end
+      end
+
+      def read_bid_attr(attribute)
+        redis do |r|
+          r.hget(@bidkey, attribute)
+        end
+      end
+
+      def flush_pending_attrs
+        redis do |r|
+          r.mapped_hmset(@bidkey, @pending_attrs)
+        end
+        @pending_attrs = {}
+      end
+
+      class << self
+        def process_failed_job(bid, jid)
+          _, pending, failed, children, complete, parent_bid = redis do |r|
+            r.multi do
+              r.sadd("BID-#{bid}-failed", jid)
+
+              r.hincrby("BID-#{bid}", "pending", 0)
+              r.scard("BID-#{bid}-failed")
+              r.hincrby("BID-#{bid}", "children", 0)
+              r.scard("BID-#{bid}-complete")
+              r.hget("BID-#{bid}", "parent_bid")
+
+              r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
+            end
+          end
+
+          # if the batch failed, and has a parent, update the parent to show one pending and failed job
+          if parent_bid
+            redis do |r|
+              r.multi do
+                r.hincrby("BID-#{parent_bid}", "pending", 1)
+                r.sadd("BID-#{parent_bid}-failed", jid)
+                r.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL)
+              end
+            end
+          end
+
+          if pending.to_i == failed.to_i && children == complete
+            enqueue_callbacks(:complete, bid)
+          end
+        end
+
+        def process_dead_job(bid, jid)
+          _, failed, children, complete, parent_bid = redis do |r|
+            r.multi do
+              r.sadd("BID-#{bid}-dead", jid)
+
+              r.scard("BID-#{bid}-dead")
+              r.hincrby("BID-#{bid}", "children", 0)
+              r.scard("BID-#{bid}-complete")
+              r.hget("BID-#{bid}", "parent_bid")
+
+              r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
+            end
+          end
+
+          if parent_bid
+            redis do |r|
+              r.multi do
+                r.sadd("BID-#{parent_bid}-dead", jid)
+                r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
+              end
+            end
+          end
+
+          enqueue_callbacks(:dead, bid)
+        end
+
+        def process_successful_job(bid, jid)
+          failed, pending, children, complete, success, total, parent_bid = redis do |r|
+            r.multi do
+              r.scard("BID-#{bid}-failed")
+              r.hincrby("BID-#{bid}", "pending", -1)
+              r.hincrby("BID-#{bid}", "children", 0)
+              r.scard("BID-#{bid}-complete")
+              r.scard("BID-#{bid}-success")
+              r.hget("BID-#{bid}", "total")
+              r.hget("BID-#{bid}", "parent_bid")
+
+              r.srem("BID-#{bid}-failed", jid)
+              r.srem("BID-#{bid}-jids", jid)
+              r.expire("BID-#{bid}", BID_EXPIRE_TTL)
+            end
+          end
+
+          all_success = pending.to_i.zero? && children == success
+          # if complete or successfull call complete callback (the complete callback may then call successful)
+          if (pending.to_i == failed.to_i && children == complete) || all_success
+            enqueue_callbacks(:complete, bid)
+            enqueue_callbacks(:success, bid) if all_success
+          end
+        end
+
+        def enqueue_callbacks(event, bid)
+          batch_key = "BID-#{bid}"
+          callback_key = "#{batch_key}-callbacks-#{event}"
+          already_processed, _, callbacks, queue, parent_bid, callback_batch = redis do |r|
+            r.multi do
+              r.hget(batch_key, event)
+              r.hset(batch_key, event, true)
+              r.smembers(callback_key)
+              r.hget(batch_key, "callback_queue")
+              r.hget(batch_key, "parent_bid")
+              r.hget(batch_key, "callback_batch")
+            end
+          end
+
+          return if already_processed == 'true'
+
+          queue ||= "default"
+          parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
+          callback_args = callbacks.reduce([]) do |memo, jcb|
+            cb = JSON.load(jcb)
+            memo << [cb['callback'], event.to_s, cb['opts'], bid, parent_bid]
+          end
+
+          opts = {"bid" => bid, "event" => event}
+
+          # Run callback batch finalize synchronously
+          if callback_batch
+            # Extract opts from cb_args or use current
+            # Pass in stored event as callback finalize is processed on complete event
+            cb_opts = callback_args.first&.at(2) || opts
+
+            logger.debug {"Run callback batch bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+            # Finalize now
+            finalizer = Batch::Callback::Finalize.new
+            status = Status.new bid
+            finalizer.dispatch(status, cb_opts)
+
+            return
+          end
+
+          logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+
+          if callback_args.empty?
+            # Finalize now
+            finalizer = Batch::Callback::Finalize.new
+            status = Status.new bid
+            finalizer.dispatch(status, opts)
+          else
+            # Otherwise finalize in sub batch complete callback
+            cb_batch = self.new
+            cb_batch.callback_batch = true
+            logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
+            cb_batch.on(:complete, "#{Batch::Callback::Finalize.to_s}#dispatch", opts)
+            cb_batch.jobs do
+              push_callbacks callback_args, queue
+            end
+          end
+        end
+
+        def cleanup_redis(bid)
+          logger.debug {"Cleaning redis of batch #{bid}"}
+          redis do |r|
+            r.del(
+              "BID-#{bid}",
+              "BID-#{bid}-callbacks-complete",
+              "BID-#{bid}-callbacks-success",
+              "BID-#{bid}-failed",
+
+              "BID-#{bid}-success",
+              "BID-#{bid}-complete",
+              "BID-#{bid}-jids",
+            )
+          end
+        end
+
+        def redis(*args, &blk)
+          defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
+        end
+
+        def logger
+          defined?(::Sidekiq) ? ::Sidekiq.logger : Rails.logger
+        end
+
+        private
+
+        def push_callbacks(args, queue)
+          Batch::Callback::Worker.enqueue_all(args, queue)
+        end
+      end
+    end
+
+    ActiveJob::Base.include BatchAwareJob
+  end
+end
+
+# Automatically integrate with Sidekiq if it is present.
+if defined?(::Sidekiq)
+  require_relative './sidekiq'
+  CanvasSync::JobBatches::Sidekiq.configure
+end