canvas_sync 0.17.3.beta1 → 0.17.5.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a67693c73d6b2bd34ffd9cf6ac17038c4d226eab30e1795ebbbd6a40d03e1bff
-  data.tar.gz: 2d42669d68de6ef9242e3d956ff15c82bb18fc07ee5d9e94c5eb1ac283928599
+  metadata.gz: ba2e2585837d6eefc1b0aa4e0f2f531598442618afc5b3d6d78ca9fda9596278
+  data.tar.gz: b198271129b9ff7623d203d30d3c206aea92f540a92fdd52258196745b88519e
 SHA512:
-  metadata.gz: 8d820e38c40e77e5199ef07af41a475ad7a35d36198ee315cb552324b1159592e329d6401779ad218ed7a7c30abdbe7df2995ad26b132a1da8c33f0dea83a516
-  data.tar.gz: 035e631264ef58221f1163e60e4e8b4167a1930800c4e54bf143f148a0e56a2447d81fa0d8150f6d2ade81a9dfb5eebec07f826413cae0fd958543fb68fe9e0f
+  metadata.gz: a88e9f513aac8a8b556c38a7e54d8fcc0209fecfa1e342d9873205677c8c27875d6a9a690115eabbb0aa549c8f9988b939581b897f889895bb59ed3e918ab4e8
+  data.tar.gz: 5438fe0b453dfbf394b6f778fcda09f423a3d7a5421b64b1252ecc090d0005b21a5d0301ac345cf8bcfe40ae87e8553e3678e63b2c0420694499f4070101d9a5

data/lib/canvas_sync/job_batches/batch.rb

@@ -19,6 +19,8 @@ require_relative "./chain_builder"
 
 module CanvasSync
   module JobBatches
+    class SuccessfulFailure < RuntimeError; end
+
     class Batch
       include RedisModel
 
@@ -35,7 +37,6 @@ module CanvasSync
         @existing = !(!existing_bid || existing_bid.empty?)  # Basically existing_bid.present?
         @initialized = false
         @bidkey = "BID-" + @bid.to_s
-        @ready_to_queue = nil
         self.created_at = Time.now.utc.to_f unless @existing
       end
 
@@ -100,7 +101,7 @@ module CanvasSync
               if parent_bid
                 r.hincrby("BID-#{parent_bid}", "children", 1)
                 r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
-                r.sadd("BID-#{parent_bid}-bids", bid)
+                r.zadd("BID-#{parent_bid}-bids", created_at, bid)
               end
             end
           end
@@ -113,28 +114,20 @@ module CanvasSync
           assert_batch_is_open
         end
 
-        job_queue = @ready_to_queue = []
-
         begin
           parent = Thread.current[:batch]
           Thread.current[:batch] = self
           yield
         ensure
-          @ready_to_queue = nil
-          append_jobs(job_queue, parent_bid)
           Thread.current[:batch] = parent
         end
 
-        job_queue
+        nil
       end
 
       def increment_job_queue(jid)
-        if @ready_to_queue
-          @ready_to_queue << jid
-        else
-          assert_batch_is_open
-          append_jobs([jid])
-        end
+        assert_batch_is_open
+        append_jobs([jid])
       end
 
       def invalidate_all
@@ -180,27 +173,36 @@ module CanvasSync
         @bidkey
       end
 
+      def flush_pending_attrs
+        super
+        redis do |r|
+          r.zadd("batches", created_at, bid)
+        end
+      end
+
       private
 
       def assert_batch_is_open
         unless defined?(@closed)
           redis do |r|
-            @closed = r.hget(@bidkey, 'closed') == 'true'
+            @closed = r.hget(@bidkey, 'success') == 'true'
           end
         end
         raise "Cannot add jobs to Batch #{} bid - it has already entered the callback-stage" if @closed
       end
 
-      def append_jobs(jids, parent_bid = self.parent_bid)
+      def append_jobs(jids)
+        jids = jids.uniq
+        return unless jids.size > 0
+
         redis do |r|
+          tme = Time.now.utc.to_f
+          added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
           r.multi do
-            r.hincrby(@bidkey, "pending", jids.size)
+            r.hincrby(@bidkey, "pending", added)
+            r.hincrby(@bidkey, "job_count", added)
             r.expire(@bidkey, BID_EXPIRE_TTL)
-
-            if jids.size > 0
-              r.sadd(@bidkey + "-jids", jids)
-              r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
-            end
+            r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
           end
         end
       end
@@ -208,6 +210,8 @@ module CanvasSync
       class << self
         def process_failed_job(bid, jid)
           _, pending, failed, children, complete, parent_bid = redis do |r|
+            return unless r.exists?("BID-#{bid}")
+
             r.multi do
               r.sadd("BID-#{bid}-failed", jid)
 
@@ -228,6 +232,8 @@ module CanvasSync
 
         def process_dead_job(bid, jid)
           _, failed, children, complete, parent_bid = redis do |r|
+            return unless r.exists?("BID-#{bid}")
+
             r.multi do
               r.sadd("BID-#{bid}-dead", jid)
 
@@ -254,6 +260,8 @@ module CanvasSync
 
         def process_successful_job(bid, jid)
           _, failed, pending, children, complete, success, parent_bid = redis do |r|
+            return unless r.exists?("BID-#{bid}")
+
             r.multi do
               r.srem("BID-#{bid}-failed", jid)
 
@@ -264,7 +272,8 @@ module CanvasSync
               r.scard("BID-#{bid}-batches-success")
               r.hget("BID-#{bid}", "parent_bid")
 
-              r.srem("BID-#{bid}-jids", jid)
+              r.hincrby("BID-#{bid}", "successful-jobs", 1)
+              r.zrem("BID-#{bid}-jids", jid)
               r.expire("BID-#{bid}", BID_EXPIRE_TTL)
             end
           end
@@ -277,29 +286,13 @@ module CanvasSync
           end
         end
 
-        def possibly_enqueue_callbacks(bid, types: [:complete, :success])
-          pending_jobs, failed_jobs, total_batches, completed_batches, successful_batches = redis do |r|
-            r.multi do
-
-            end
-          end
-
-          if (types.include?(:complete)) && (pending_jobs.to_i == failed_jobs.to_i && total_batches.to_i == completed_batches.to_i)
-            enqueue_callbacks(:complete, bid)
-          end
-          if (types.include?(:success)) && (pending_jobs.to_i.zero? && total_batches.to_i == successful_batches.to_i)
-            enqueue_callbacks(:success, bid)
-          end
-        end
-
         def enqueue_callbacks(event, bid)
           batch_key = "BID-#{bid}"
           callback_key = "#{batch_key}-callbacks-#{event}"
-          already_processed, _, _, callbacks, queue, parent_bid, callback_params = redis do |r|
+          already_processed, _, callbacks, queue, parent_bid, callback_params = redis do |r|
             return unless r.exists?(batch_key)
             r.multi do
               r.hget(batch_key, event)
-              r.hset(batch_key, "closed", true)
               r.hset(batch_key, event, true)
               r.smembers(callback_key)
               r.hget(batch_key, "callback_queue")
@@ -323,6 +316,11 @@ module CanvasSync
           if callback_args.present? && !callback_params.present?
             logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
 
+            redis do |r|
+              r.sadd("#{batch_key}-pending_callbacks", event)
+              r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
+            end
+
             with_batch(parent_bid) do
               cb_batch = self.new
               cb_batch.callback_params = {
@@ -351,7 +349,8 @@ module CanvasSync
         def cleanup_redis(bid)
           logger.debug {"Cleaning redis of batch #{bid}"}
           redis do |r|
-            r.del(
+            r.zrem("batches", bid)
+            r.unlink(
               "BID-#{bid}",
               "BID-#{bid}-callbacks-complete",
               "BID-#{bid}-callbacks-success",
@@ -362,10 +361,21 @@ module CanvasSync
               "BID-#{bid}-batches-failed",
               "BID-#{bid}-bids",
               "BID-#{bid}-jids",
+              "BID-#{bid}-pending_callbacks",
             )
           end
         end
 
+        def delete_prematurely!(bid)
+          child_bids = redis do |r|
+            r.zrange("BID-#{bid}-bids", 0, -1)
+          end
+          child_bids.each do |cbid|
+            delete_prematurely!(cbid)
+          end
+          cleanup_redis(bid)
+        end
+
         def redis(*args, &blk)
           defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
         end
@@ -374,8 +384,6 @@ module CanvasSync
           defined?(::Sidekiq) ? ::Sidekiq.logger : Rails.logger
         end
 
-        private
-
         def push_callbacks(args, queue)
           Batch::Callback::worker_class.enqueue_all(args, queue)
         end

data/lib/canvas_sync/job_batches/batch_aware_job.rb

@@ -10,8 +10,12 @@ module CanvasSync
             begin
               Thread.current[:batch] = Batch.new(@bid)
               block.call
-              batch&.save_context_changes
+              Thread.current[:batch].save_context_changes
               Batch.process_successful_job(@bid, job_id)
+            rescue SuccessfulFailure => err
+              Thread.current[:batch].save_context_changes
+              Batch.process_successful_job(@bid, job_id)
+              raise
             rescue
               Batch.process_failed_job(@bid, job_id)
               raise

data/lib/canvas_sync/job_batches/callback.rb

@@ -52,9 +52,6 @@ module CanvasSync
 
         class Finalize
           def dispatch(status, opts)
-            is_callback_batch = opts['origin'].present?
-            has_callback_batch = opts['callback_bid'].present?
-
             bid = opts["bid"]
             event = opts["event"].to_sym
 
@@ -63,19 +60,28 @@ module CanvasSync
             batch_status = Status.new bid
             send(event, bid, batch_status, batch_status.parent_bid)
 
-            if event == :success && !has_callback_batch
-              Batch.cleanup_redis(bid)
-            end
-
-            if event == :success && is_callback_batch && opts['origin']['event'].to_sym == :success
-              Batch.cleanup_redis(opts['origin']['for_bid'])
+            if event == :success
+              if opts['origin'].present?
+                origin_bid = opts['origin']['for_bid']
+                _, pending, success_ran = Batch.redis do |r|
+                  r.multi do
+                    r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
+                    r.scard("BID-#{origin_bid}-pending_callbacks")
+                    r.hget("BID-#{origin_bid}", "success")
+                  end
+                end
+                Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
+              end
+              if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
+                Batch.cleanup_redis(bid)
+              end
             end
           end
 
           def success(bid, status, parent_bid)
             return unless parent_bid
 
-            _, _, success, _, _, complete, pending, children, failure = Batch.redis do |r|
+            _, _, success, _, _, complete, pending, children, success, failure = Batch.redis do |r|
               r.multi do
                 r.sadd("BID-#{parent_bid}-batches-success", bid)
                 r.expire("BID-#{parent_bid}-batches-success", Batch::BID_EXPIRE_TTL)
@@ -87,15 +93,21 @@ module CanvasSync
 
                 r.hincrby("BID-#{parent_bid}", "pending", 0)
                 r.hincrby("BID-#{parent_bid}", "children", 0)
+                r.scard("BID-#{parent_bid}-batches-success")
                 r.scard("BID-#{parent_bid}-failed")
               end
             end
+
             # If the job finished successfully and parent batch is completed, call parent :complete callback
             # Parent :success callback will be called by its :complete callback
             if complete == children && pending == failure
               Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
               Batch.enqueue_callbacks(:complete, parent_bid)
             end
+            if pending.to_i.zero? && children == success
+              Batch.logger.debug {"Finalize parent success bid: #{parent_bid}"}
+              Batch.enqueue_callbacks(:success, parent_bid)
+            end
           end
 
           def complete(bid, status, parent_bid)
@@ -107,13 +119,7 @@ module CanvasSync
               end
             end
 
-            # If the batch was successful run :success callback, which will call the parent's :complete callback (if necessary)
-            # Also, only trigger the success callback if the :complete callback_batch was successful
-            if pending.to_i.zero? && children == success
-              Batch.enqueue_callbacks(:success, bid)
-
-            # otherwise check for a parent and call its :complete if needed
-            elsif parent_bid
+            if parent_bid && !(pending.to_i.zero? && children == success)
               # if batch was not successfull check and see if its parent is complete
               # if the parent is complete we trigger the complete callback
               # We don't want to run this if the batch was successfull because the success

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -28,6 +28,10 @@ module CanvasSync
         @hash_map[local_bid]
       end
 
+      def own
+        resolve_hash(@bid_stack[-1]) || {}
+      end
+
       def set_local(new_hash)
         @dirty = true
         local.clear.merge!(new_hash)
@@ -106,7 +110,8 @@ module CanvasSync
           @bid_stack[index]
         else
           pbid = Batch.redis do |r|
-            r.hget("BID-#{bid}", "parent_bid") || r.hget("BID-#{bid}", "callback_for")
+            callback_params = JSON.parse(r.hget("BID-#{bid}", "callback_params") || "{}")
+            callback_params['for_bid'] || r.hget("BID-#{bid}", "parent_bid")
           end
           @bid_stack.unshift(pbid)
           pbid

data/lib/canvas_sync/job_batches/pool.rb

@@ -10,7 +10,7 @@ module CanvasSync
       redis_attr :created_at
       redis_attr :concurrency, :int
       redis_attr :order
-      redis_attr :on_failed_job
+      redis_attr :on_failed_job, :symbol
       redis_attr :clean_when_empty, :bool
 
       def initialize(pooolid = nil, **kwargs)
@@ -24,6 +24,7 @@ module CanvasSync
       end
 
       def self.from_pid(pid)
+        raise "PID must be given" unless pid.present?
         new(pid)
       end
 
@@ -38,9 +39,9 @@ module CanvasSync
       def add_jobs(job_descs)
         job_descs.each do |job_desc|
           wrapper = Batch.new
-          wrapper.description = "Pool Job Wrapper"
-          wrapper.on(:complete, "#{self.class.to_s}.job_completed_callback", pool_id: pid)
-          wrapper.on(:success, "#{self.class.to_s}.job_success_callback", pool_id: pid)
+          wrapper.description = "Pool Job Wrapper (PID: #{pid})"
+          checkin_event = (on_failed_job == :wait) ? :success : :complete
+          wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
           wrapper.jobs {}
 
           job_desc = job_desc.with_indifferent_access
@@ -57,43 +58,50 @@ module CanvasSync
       def cleanup_redis
         Batch.logger.debug {"Cleaning redis of pool #{pid}"}
         redis do |r|
-          r.del(
+          r.zrem("pools", pid)
+          r.unlink(
             "#{redis_key}",
             "#{redis_key}-jobs",
           )
         end
       end
 
-      def handle_event(event, options)
-        queue_next = false
-        if event == :success && on_failed_job.to_sym == :wait # We trigger on the complete event, unless on_failed_job is :wait
-          queue_next = true
-        elsif event == :complete
-          if on_failed_job.to_sym != :wait
-            queue_next = true
-          end
+      def active_count
+        redis do |r|
+          r.hincrby(redis_key, "active_count", 0)
         end
+      end
 
-        if queue_next
-          active_count = redis do |r|
-            r.hincrby(redis_key, "active_count", -1)
-          end
-          added_count = refill_allotment
-
-          if active_count == 0 && added_count == 0
-            cleanup_redis if clean_when_empty
+      def pending_count
+        jobs_key = "#{redis_key}-jobs"
+        order = self.order || 'fifo'
+        redis do |r|
+          case order.to_sym
+          when :fifo, :lifo
+            r.llen(jobs_key)
+          when :random
+            r.scard(jobs_key)
+          when :priority
+            r.zcard(jobs_key)
           end
         end
       end
 
-      def self.job_completed_callback(status, options)
-        pid = options['pool_id']
-        from_pid(pid).handle_event(:complete, options)
+      def job_checked_in(status, options)
+        active_count = redis do |r|
+          return unless r.exists?(redis_key)
+          r.hincrby(redis_key, "active_count", -1)
+        end
+
+        added_count = refill_allotment
+        if active_count == 0 && added_count == 0
+          cleanup_redis if clean_when_empty
+        end
       end
 
-      def self.job_success_callback(status, options)
+      def self.job_checked_in(status, options)
         pid = options['pool_id']
-        from_pid(pid).handle_event(:success, options)
+        from_pid(pid).job_checked_in(status, options)
       end
 
       protected
@@ -175,29 +183,25 @@ module CanvasSync
         ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
       end
 
-      def pending_count
-        order = self.order
-        redis do |r|
-          case order.to_sym
-          when :fifo, :lifo
-            r.llen(jobs_key)
-          when :random
-            r.scard(jobs_key)
-          when :priority
-            r.zcard(jobs_key)
-          end
-        end
-      end
-
       def self.redis(&blk)
         Batch.redis &blk
       end
       delegate :redis, to: :class
 
+      protected
+
+      def flush_pending_attrs
+        super
+        redis do |r|
+          r.zadd("pools", created_at, pid)
+        end
+      end
+
       private
 
-      def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait)
+      def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait, description: nil)
         self.created_at = Time.now.utc.to_f
+        self.description = description
         self.order = order
         self.concurrency = concurrency
         self.clean_when_empty = clean_when_empty