canvas_sync 0.17.0.beta14 → 0.17.3.beta1

data/lib/canvas_sync/job_batches/callback.rb

@@ -2,6 +2,7 @@ module CanvasSync
   module JobBatches
     class Batch
       module Callback
+        mattr_accessor :worker_class
 
         VALID_CALLBACKS = %w[success complete dead].freeze
 
@@ -47,41 +48,28 @@ module CanvasSync
           end
         end
 
-        if defined?(::Sidekiq)
-          class SidekiqCallbackWorker
-            include ::Sidekiq::Worker
-            include CallbackWorkerCommon
-
-            def self.enqueue_all(args, queue)
-              return if args.empty?
-
-              ::Sidekiq::Client.push_bulk(
-                'class' => self,
-                'args' => args,
-                'queue' => queue
-              )
-            end
-          end
-          Worker = SidekiqCallbackWorker
-        else
-          Worker = ActiveJobCallbackWorker
-        end
+        worker_class = ActiveJobCallbackWorker
 
         class Finalize
-          def dispatch status, opts
+          def dispatch(status, opts)
+            is_callback_batch = opts['origin'].present?
+            has_callback_batch = opts['callback_bid'].present?
+
             bid = opts["bid"]
-            callback_bid = status.bid
             event = opts["event"].to_sym
-            callback_batch = bid != callback_bid
 
-            Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
+            Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{is_callback_batch}"}
 
             batch_status = Status.new bid
             send(event, bid, batch_status, batch_status.parent_bid)
 
-            # Different events are run in different callback batches
-            Batch.cleanup_redis callback_bid if callback_batch
-            Batch.cleanup_redis bid if event == :success
+            if event == :success && !has_callback_batch
+              Batch.cleanup_redis(bid)
+            end
+
+            if event == :success && is_callback_batch && opts['origin']['event'].to_sym == :success
+              Batch.cleanup_redis(opts['origin']['for_bid'])
+            end
           end
 
           def success(bid, status, parent_bid)
@@ -102,8 +90,8 @@ module CanvasSync
                 r.scard("BID-#{parent_bid}-failed")
               end
             end
-            # if job finished successfully and parent batch completed call parent complete callback
-            # Success callback is called after complete callback
+            # If the job finished successfully and parent batch is completed, call parent :complete callback
+            # Parent :success callback will be called by its :complete callback
             if complete == children && pending == failure
               Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
               Batch.enqueue_callbacks(:complete, parent_bid)
@@ -119,10 +107,12 @@ module CanvasSync
               end
             end
 
-            # if we batch was successful run success callback
+            # If the batch was successful run :success callback, which will call the parent's :complete callback (if necessary)
+            # Also, only trigger the success callback if the :complete callback_batch was successful
             if pending.to_i.zero? && children == success
               Batch.enqueue_callbacks(:success, bid)
 
+            # otherwise check for a parent and call its :complete if needed
             elsif parent_bid
               # if batch was not successfull check and see if its parent is complete
               # if the parent is complete we trigger the complete callback

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -17,7 +17,7 @@ module CanvasSync
       def local_bid
         bid = @bid_stack[-1]
         while bid.present?
-          bhash = reolve_hash(bid)
+          bhash = resolve_hash(bid)
           return bid if bhash
           bid = get_parent_bid(bid)
         end
@@ -49,7 +49,7 @@ module CanvasSync
       def [](key)
         bid = @bid_stack[-1]
         while bid.present?
-          bhash = reolve_hash(bid)
+          bhash = resolve_hash(bid)
           return bhash[key] if bhash&.key?(key)
           bid = get_parent_bid(bid)
         end
@@ -94,7 +94,7 @@ module CanvasSync
       private
 
       def get_parent_hash(bid)
-        reolve_hash(get_parent_bid(bid)).freeze
+        resolve_hash(get_parent_bid(bid)).freeze
       end
 
       def get_parent_bid(bid)
@@ -105,13 +105,15 @@ module CanvasSync
         if index >= 0
           @bid_stack[index]
         else
-          pbid = Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
+          pbid = Batch.redis do |r|
+            r.hget("BID-#{bid}", "parent_bid") || r.hget("BID-#{bid}", "callback_for")
+          end
           @bid_stack.unshift(pbid)
           pbid
         end
       end
 
-      def reolve_hash(bid)
+      def resolve_hash(bid)
         return nil unless bid.present?
         return @hash_map[bid] if @hash_map.key?(bid)
 
@@ -137,6 +139,7 @@ module CanvasSync
       end
 
       def load_all
+        resolve_hash(@bid_stack[0]).freeze
         while @bid_stack[0].present?
           get_parent_hash(@bid_stack[0])
         end

data/lib/canvas_sync/job_batches/hincr_max.lua

@@ -0,0 +1,5 @@
+local r=redis.call('HGET', KEYS[1], ARGV[1])
+if r == false or r < ARGV[2] then
+    redis.call('HINCRBY', KEYS[1], ARGV[1], 1)
+end
+return r or 0

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -0,0 +1,99 @@
+require_relative './base_job'
+
+module CanvasSync
+  module JobBatches
+    class ManagedBatchJob < BaseJob
+      def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
+        man_batch_id = SecureRandom.urlsafe_base64(10)
+
+        if concurrency == 0 || concurrency == nil || concurrency == true
+          concurrency = sub_jobs.count
+        elsif concurrency == false
+          concurrency = 1
+        end
+
+        root_batch = Batch.new
+
+        Batch.redis do |r|
+          r.multi do
+            r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
+            r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
+            r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
+            r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
+
+            mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
+              j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
+              j = ActiveJob::Arguments.serialize([j])
+              JSON.unparse(j)
+            end
+            if ordered
+              r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+            else
+              r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+            end
+            r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
+          end
+        end
+
+        root_batch.description = "Managed Batch Root (#{man_batch_id})"
+        root_batch.allow_context_changes = (concurrency == 1)
+        root_batch.context = context
+        root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
+        root_batch.jobs {}
+
+        concurrency.times do
+          self.class.perform_next_sequence_job(man_batch_id)
+        end
+      end
+
+      def self.cleanup_redis(status, options)
+        man_batch_id = options['managed_batch_id']
+        Batch.redis do |r|
+          r.del(
+            "MNGBID-#{man_batch_id}",
+            "MNGBID-#{man_batch_id}-jobs",
+          )
+        end
+      end
+
+      def self.job_succeeded_callback(status, options)
+        man_batch_id = options['managed_batch_id']
+        perform_next_sequence_job(man_batch_id)
+      end
+
+      protected
+
+      def self.perform_next_sequence_job(man_batch_id)
+        root_bid, ordered = Batch.redis do |r|
+          r.multi do
+            r.hget("MNGBID-#{man_batch_id}", "root_bid")
+            r.hget("MNGBID-#{man_batch_id}", "ordered")
+          end
+        end
+
+        next_job_json = Batch.redis do |r|
+          if ordered
+            r.lpop("MNGBID-#{man_batch_id}-jobs")
+          else
+            r.spop("MNGBID-#{man_batch_id}-jobs")
+          end
+        end
+
+        return unless next_job_json.present?
+
+        next_job = JSON.parse(next_job_json)
+        next_job = ActiveJob::Arguments.deserialize(next_job)[0]
+
+        Batch.new(root_bid).jobs do
+          Batch.new.tap do |batch|
+            batch.description = "Managed Batch Fiber (#{man_batch_id})"
+            batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
+            batch.jobs do
+              ChainBuilder.enqueue_job(next_job)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -4,71 +4,12 @@ module CanvasSync
   module JobBatches
     class SerialBatchJob < BaseJob
       def perform(sub_jobs, context: nil)
-        serial_id = SecureRandom.urlsafe_base64(10)
-
-        root_batch = Batch.new
-
-        Batch.redis do |r|
-          r.multi do
-            mapped_sub_jobs = sub_jobs.map do |j|
-              j = ActiveJob::Arguments.serialize([j])
-              JSON.unparse(j)
-            end
-            r.hset("SERBID-#{serial_id}", "root_bid", root_batch.bid)
-            r.expire("SERBID-#{serial_id}", Batch::BID_EXPIRE_TTL)
-            r.rpush("SERBID-#{serial_id}-jobs", mapped_sub_jobs)
-            r.expire("SERBID-#{serial_id}-jobs", Batch::BID_EXPIRE_TTL)
-          end
-        end
-
-        root_batch.description = "Serial Batch Root (#{serial_id})"
-        root_batch.allow_context_changes = true
-        root_batch.context = context
-        root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", serial_batch_id: serial_id)
-        root_batch.jobs {}
-
-        self.class.perform_next_sequence_job(serial_id)
-      end
-
-      def self.cleanup_redis(status, options)
-        serial_id = options['serial_batch_id']
-        Batch.redis do |r|
-          r.del(
-            "SERBID-#{serial_id}",
-            "SERBID-#{serial_id}-jobs",
-          )
-        end
-      end
-
-      def self.job_succeeded_callback(status, options)
-        serial_id = options['serial_batch_id']
-        perform_next_sequence_job(serial_id)
-      end
-
-      protected
-
-      def self.perform_next_sequence_job(serial_id)
-        root_bid, next_job_json = Batch.redis do |r|
-          r.multi do
-            r.hget("SERBID-#{serial_id}", "root_bid")
-            r.lpop("SERBID-#{serial_id}-jobs")
-          end
-        end
-
-        return unless next_job_json.present?
-
-        next_job = JSON.parse(next_job_json)
-        next_job = ActiveJob::Arguments.deserialize(next_job)[0]
-
-        Batch.new(root_bid).jobs do
-          Batch.new.tap do |batch|
-            batch.description = "Serial Batch Fiber (#{serial_id})"
-            batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
-            batch.jobs do
-              ChainBuilder.enqueue_job(next_job)
-            end
-          end
-        end
+        ManagedBatchJob.new.perform(
+          sub_jobs,
+          context: context,
+          ordered: true,
+          concurrency: false,
+        )
       end
     end
   end

data/lib/canvas_sync/job_batches/pool.rb

@@ -0,0 +1,209 @@
+module CanvasSync
+  module JobBatches
+    class Pool
+      include RedisModel
+
+      HINCR_MAX = RedisScript.new(Pathname.new(__FILE__) + "../hincr_max.lua")
+
+      attr_reader :pid
+      redis_attr :description
+      redis_attr :created_at
+      redis_attr :concurrency, :int
+      redis_attr :order
+      redis_attr :on_failed_job
+      redis_attr :clean_when_empty, :bool
+
+      def initialize(pooolid = nil, **kwargs)
+        if pooolid
+          @existing = true
+          @pid = pooolid
+        else
+          @pid = SecureRandom.urlsafe_base64(10)
+          initialize_new(**kwargs)
+        end
+      end
+
+      def self.from_pid(pid)
+        new(pid)
+      end
+
+      def <<(job_desc)
+        add_job(job_desc)
+      end
+
+      def add_job(job_desc)
+        add_jobs([job_desc])
+      end
+
+      def add_jobs(job_descs)
+        job_descs.each do |job_desc|
+          wrapper = Batch.new
+          wrapper.description = "Pool Job Wrapper"
+          wrapper.on(:complete, "#{self.class.to_s}.job_completed_callback", pool_id: pid)
+          wrapper.on(:success, "#{self.class.to_s}.job_success_callback", pool_id: pid)
+          wrapper.jobs {}
+
+          job_desc = job_desc.with_indifferent_access
+          job_desc = job_desc.merge!(
+            job: job_desc[:job].to_s,
+            pool_wrapper_batch: wrapper.bid,
+          )
+
+          push_job_to_pool(job_desc)
+        end
+        refill_allotment
+      end
+
+      def cleanup_redis
+        Batch.logger.debug {"Cleaning redis of pool #{pid}"}
+        redis do |r|
+          r.del(
+            "#{redis_key}",
+            "#{redis_key}-jobs",
+          )
+        end
+      end
+
+      def handle_event(event, options)
+        queue_next = false
+        if event == :success && on_failed_job.to_sym == :wait # We trigger on the complete event, unless on_failed_job is :wait
+          queue_next = true
+        elsif event == :complete
+          if on_failed_job.to_sym != :wait
+            queue_next = true
+          end
+        end
+
+        if queue_next
+          active_count = redis do |r|
+            r.hincrby(redis_key, "active_count", -1)
+          end
+          added_count = refill_allotment
+
+          if active_count == 0 && added_count == 0
+            cleanup_redis if clean_when_empty
+          end
+        end
+      end
+
+      def self.job_completed_callback(status, options)
+        pid = options['pool_id']
+        from_pid(pid).handle_event(:complete, options)
+      end
+
+      def self.job_success_callback(status, options)
+        pid = options['pool_id']
+        from_pid(pid).handle_event(:success, options)
+      end
+
+      protected
+
+      def redis_key
+        "POOLID-#{pid}"
+      end
+
+      def refill_allotment
+        jobs_added = 0
+        limit = concurrency.to_i
+        redis do |r|
+          current_count = 0
+          while true
+            current_count = HINCR_MAX.call(r, [redis_key], ["active_count", limit]).to_i
+            if current_count < limit
+              job_desc = pop_job_from_pool
+              if job_desc.present?
+                Batch.new(job_desc['pool_wrapper_batch']).jobs do
+                  ChainBuilder.enqueue_job(job_desc)
+                end
+                jobs_added += 1
+              else
+                r.hincrby(redis_key, "active_count", -1)
+                break
+              end
+            else
+              break
+            end
+          end
+          r.expire(redis_key, Batch::BID_EXPIRE_TTL)
+          r.expire("#{redis_key}-jobs", Batch::BID_EXPIRE_TTL)
+        end
+        jobs_added
+      end
+
+      def push_job_to_pool(job_desc)
+        jobs_key = "#{redis_key}-jobs"
+        # This allows duplicate jobs when a Redis Set is used
+        job_desc['_pool_random_key_'] = SecureRandom.urlsafe_base64(10)
+        job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
+        order = self.order
+
+        redis do |r|
+          r.multi do
+            case order.to_sym
+            when :fifo, :lifo
+              r.rpush(jobs_key, job_json)
+            when :random
+              r.sadd(jobs_key, job_json)
+            when :priority
+              r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
+            end
+            r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
+          end
+        end
+      end
+
+      def pop_job_from_pool
+        jobs_key = "#{redis_key}-jobs"
+        order = self.order
+
+        job_json = nil
+        redis do |r|
+          job_json = case order.to_sym
+          when :fifo
+            r.lpop(jobs_key)
+          when :lifo
+            r.rpop(jobs_key)
+          when :random
+            r.spop(jobs_key)
+          when :priority
+            r.zpopmax(jobs_key)
+          end
+        end
+
+        return nil unless job_json.present?
+
+        ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
+      end
+
+      def pending_count
+        order = self.order
+        redis do |r|
+          case order.to_sym
+          when :fifo, :lifo
+            r.llen(jobs_key)
+          when :random
+            r.scard(jobs_key)
+          when :priority
+            r.zcard(jobs_key)
+          end
+        end
+      end
+
+      def self.redis(&blk)
+        Batch.redis &blk
+      end
+      delegate :redis, to: :class
+
+      private
+
+      def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait)
+        self.created_at = Time.now.utc.to_f
+        self.order = order
+        self.concurrency = concurrency
+        self.clean_when_empty = clean_when_empty
+        self.on_failed_job = on_failed_job
+        flush_pending_attrs
+      end
+    end
+  end
+end