canvas_sync 0.17.0.beta15 → 0.17.3.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f242a2d6641d3d51bff9b73eaf2165588405a468a1806183d20e68d7c324d976
-  data.tar.gz: ca2dc6bf47c7c861c42f6550707028d15d1b54469e1a12e3ce7fedead6204102
+  metadata.gz: 416545a03622711eba78ef420fd07aff9eb1e56ee9536b9e12369406885079c5
+  data.tar.gz: e9cabdc870f0903bcffc60f711c0ed34a5b471d1bbdf1d8a2a4d83ce9ac745a4
 SHA512:
-  metadata.gz: 4d25fa9c0d9155780a087ac94b9a72cadddd9805c60760f5bdf719870245b8f1b8c23e14650646565b7a03b6a7983e329049fb1057ccfd990f662ee479322ffd
-  data.tar.gz: f551da63e671e43194c8a2bd2cfe8f95499417d74599d14307f1470ed678a712492f7ad6a70d7208f57a6b3a22e9fb5e5ced209afb292b93b2f0ce88db02d609
+  metadata.gz: 020ba0e260823c40a21273c3bda767c9774207fa36bf7e639d0927db19d544a27b1fefcebfdd64e2b4445ba5ca526098229d95e4c3ee7f5f6e336ef42da8a1f6
+  data.tar.gz: 1840797de37257526d002507fe1849c84cd5cc71ed5bbb176a93061dce8d4c25e85528ec7312a3215e9520bcf4f708ac8cf5834539fdac4a7810919700844d33

data/README.md

@@ -246,6 +246,64 @@ A batch can be created using `Sidekiq::Batch` or `CanvasSync::JobBatching::Batch
 
 Also see `canvas_sync/jobs/begin_sync_chain_job`, `canvas_sync/Job_batches/jobs/serial_batch_job`, or `canvas_sync/Job_batches/jobs/concurrent_batch_job` for example usage.
 
+Example:
+```ruby
+batch = CanvasSync::JobBatches::Batch.new
+batch.description = "Some Batch" # Optional, but can be useful for debugging
+
+batch.on(:complete, "SomeClass.on_complete", kw_param: 1)
+batch.on(:success, "SomeClass.on_success", some_param: 'foo')
+
+# Add context to the batch. Can be accessed as batch_context on any jobs within the batch.
+# Nested Batches will have their contexts merged
+batch.context = {
+  some_value: 'blah',
+}
+
+batch.jobs do
+  # Enqueue jobs like normal
+end
+```
+
+#### Job Pools
+A job pool is like a custom Sidekiq Queue. You can add jobs to it and it will empty itself out into one of the actual queues.
+However, it adds some options for tweaking the logic:
+- `concurrency` (default: `nil`) - Define how many jobs from the pool can run at once.
+- `order` (default: `fifo`) - Define how the pool will empty itself
+  - `fifo` - First-In First-Out, a traditional queue
+  - `lifo` - Last-In First-Out
+  - `random` - Pluck and run jobs in random order
+  - `priority` - Execute jobs in a priority order (NB: Due to Redis, this priority-random, meaning that items with the same priority will be run in random order, not fifo)
+- `clean_when_empty` (default: `true`) - Automatically clean the pool when it is empty
+- `on_failed_job` (default `:wait`) - If a Job fails, should the pool `:continue` and still enqueue the next job or `:wait` for the job to succeed
+
+Example:
+```ruby
+pool = CanvasSync::JobBatches::Pool.new(concurrency: 4, order: :priority, clean_when_empty: false)
+pool_id = pool.pid
+
+# Add a job to the pool
+pool << {
+  job: SomeJob, # The Class of a ActiveJob Job or Sidekiq Worker
+  parameters: [1, 2, 3], # Array of params to pass th e Job
+  priority: 100, # Only effective if order=:priority, higher is higher
+}
+
+# Add many jobs to the pool
+pool.add_jobs([
+  {
+    job: SomeJob, # The Class of a ActiveJob Job or Sidekiq Worker
+    parameters: [1, 2, 3], # Array of params to pass th e Job
+    priority: 100, # Only effective if order=:priority, higher is higher
+  },
+  # ...
+])
+
+# ...Later
+CanvasSync::JobBatches::Pool.from_pid(pool_id).cleanup_redis
+
+```
+
 ## Legacy Support
 
 ### Legacy Mappings

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -64,13 +64,12 @@ module CanvasSync
         columns = columns.dup
 
         update_conditions = {
-          condition: condition_sql(klass, columns),
+          condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
         update_conditions[:conflict_target] = conflict_target if conflict_target
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
-
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
         klass.import(columns, rows, options)
       end
@@ -85,10 +84,14 @@ module CanvasSync
       # started_at = Time.now
       # run_the_users_sync!
       # changed = User.where("updated_at >= ?", started_at)
-      def self.condition_sql(klass, columns)
+      def self.condition_sql(klass, columns, report_start)
         columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
         excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
-        "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
+        if klass.column_names.include?("updated_at") && report_start
+          condition_sql += " AND #{klass.quoted_table_name}.updated_at < '#{report_start}'"
+        end
+        condition_sql
       end
 
       def self.batch_size

data/lib/canvas_sync/job_batches/batch.rb

@@ -4,10 +4,13 @@ begin
 rescue LoadError
 end
 
+require_relative './redis_model'
+require_relative './redis_script'
 require_relative './batch_aware_job'
 require_relative "./callback"
 require_relative "./context_hash"
 require_relative "./status"
+require_relative "./pool"
 Dir[File.dirname(__FILE__) + "/jobs/*.rb"].each { |file| require file }
 require_relative "./chain_builder"
 
@@ -17,24 +20,9 @@ require_relative "./chain_builder"
 module CanvasSync
   module JobBatches
     class Batch
-      class NoBlockGivenError < StandardError; end
-
-      def self.batch_attr(key, read_only: true)
-        class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def #{key}=(value)
-            raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
-            @#{key} = value
-            persist_bid_attr('#{key}', value)
-          end
+      include RedisModel
 
-          def #{key}
-            return @#{key} if defined?(@#{key})
-            if (@initialized || @existing)
-              @#{key} = read_bid_attr('#{key}')
-            end
-          end
-        RUBY
-      end
+      class NoBlockGivenError < StandardError; end
 
       delegate :redis, to: :class
 
@@ -47,16 +35,14 @@ module CanvasSync
         @existing = !(!existing_bid || existing_bid.empty?)  # Basically existing_bid.present?
         @initialized = false
         @bidkey = "BID-" + @bid.to_s
-        @pending_attrs = {}
-        @ready_to_queue = nil
         self.created_at = Time.now.utc.to_f unless @existing
       end
 
-      batch_attr :description
-      batch_attr :created_at
-      batch_attr :callback_queue, read_only: false
-      batch_attr :callback_batch, read_only: false
-      batch_attr :allow_context_changes
+      redis_attr :description
+      redis_attr :created_at
+      redis_attr :callback_queue, read_only: false
+      redis_attr :callback_params, :json
+      redis_attr :allow_context_changes
 
       def context
         return @context if defined?(@context)
@@ -122,29 +108,24 @@ module CanvasSync
           @context&.save!
 
           @initialized = true
+        else
+          assert_batch_is_open
         end
 
-        job_queue = @ready_to_queue = []
-
         begin
           parent = Thread.current[:batch]
           Thread.current[:batch] = self
           yield
         ensure
-          @ready_to_queue = nil
-          append_jobs(job_queue, parent_bid)
           Thread.current[:batch] = parent
         end
 
-        job_queue
+        nil
       end
 
       def increment_job_queue(jid)
-        if @ready_to_queue
-          @ready_to_queue << jid
-        else
-          append_jobs([jid])
-        end
+        assert_batch_is_open
+        append_jobs([jid])
       end
 
       def invalidate_all
@@ -170,52 +151,42 @@ module CanvasSync
         batch.parent ? valid && valid?(batch.parent) : valid
       end
 
-      # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
-      def self.without_batch
+      def self.with_batch(batch)
+        batch = self.new(batch) if batch.is_a?(String)
         parent = Thread.current[:batch]
-        Thread.current[:batch] = nil
+        Thread.current[:batch] = batch
         yield
       ensure
         Thread.current[:batch] = parent
       end
 
-      private
-
-      def persist_bid_attr(attribute, value)
-        if @initialized || @existing
-          redis do |r|
-            r.multi do
-              r.hset(@bidkey, attribute, value)
-              r.expire(@bidkey, BID_EXPIRE_TTL)
-            end
-          end
-        else
-          @pending_attrs[attribute] = value
-        end
+      # Any Batches or Jobs created in the given block won't be assocaiated to the current batch
+      def self.without_batch(&blk)
+        with_batch(nil, &blk)
       end
 
-      def read_bid_attr(attribute)
-        redis do |r|
-          r.hget(@bidkey, attribute)
-        end
+      protected
+
+      def redis_key
+        @bidkey
       end
 
-      def flush_pending_attrs
-        redis do |r|
-          r.mapped_hmset(@bidkey, @pending_attrs)
+      private
+
+      def assert_batch_is_open
+        unless defined?(@closed)
+          redis do |r|
+            @closed = r.hget(@bidkey, 'closed') == 'true'
+          end
         end
-        @pending_attrs = {}
+        raise "Cannot add jobs to Batch #{} bid - it has already entered the callback-stage" if @closed
       end
 
-      def append_jobs(jids, parent_bid = self.parent_bid)
+      def append_jobs(jids)
+        jids = jids.uniq
         redis do |r|
           r.multi do
-            if parent_bid
-              r.hincrby("BID-#{parent_bid}", "total", jids.size)
-            end
-
             r.hincrby(@bidkey, "pending", jids.size)
-            r.hincrby(@bidkey, "total", jids.size)
             r.expire(@bidkey, BID_EXPIRE_TTL)
 
             if jids.size > 0
@@ -242,17 +213,6 @@ module CanvasSync
             end
           end
 
-          # if the batch failed, and has a parent, update the parent to show one pending and failed job
-          if parent_bid
-            redis do |r|
-              r.multi do
-                r.hincrby("BID-#{parent_bid}", "pending", 1)
-                r.sadd("BID-#{parent_bid}-failed", jid)
-                r.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL)
-              end
-            end
-          end
-
           if pending.to_i == failed.to_i && children == complete
             enqueue_callbacks(:complete, bid)
           end
@@ -285,7 +245,7 @@ module CanvasSync
         end
 
         def process_successful_job(bid, jid)
-          _, failed, pending, children, complete, success, total, parent_bid = redis do |r|
+          _, failed, pending, children, complete, success, parent_bid = redis do |r|
             r.multi do
               r.srem("BID-#{bid}-failed", jid)
 
@@ -294,7 +254,6 @@ module CanvasSync
               r.hincrby("BID-#{bid}", "children", 0)
               r.scard("BID-#{bid}-batches-complete")
               r.scard("BID-#{bid}-batches-success")
-              r.hget("BID-#{bid}", "total")
               r.hget("BID-#{bid}", "parent_bid")
 
               r.srem("BID-#{bid}-jids", jid)
@@ -302,6 +261,8 @@ module CanvasSync
             end
           end
 
+          # TODO - There seems to be an issue where a :complete callback batch will occasionally run its job, but won't enqueue_callbacks()
+
           all_success = pending.to_i.zero? && children == success
           # if complete or successfull call complete callback (the complete callback may then call successful)
           if (pending.to_i == failed.to_i && children == complete) || all_success
@@ -310,17 +271,34 @@ module CanvasSync
           end
         end
 
+        def possibly_enqueue_callbacks(bid, types: [:complete, :success])
+          pending_jobs, failed_jobs, total_batches, completed_batches, successful_batches = redis do |r|
+            r.multi do
+
+            end
+          end
+
+          if (types.include?(:complete)) && (pending_jobs.to_i == failed_jobs.to_i && total_batches.to_i == completed_batches.to_i)
+            enqueue_callbacks(:complete, bid)
+          end
+          if (types.include?(:success)) && (pending_jobs.to_i.zero? && total_batches.to_i == successful_batches.to_i)
+            enqueue_callbacks(:success, bid)
+          end
+        end
+
         def enqueue_callbacks(event, bid)
           batch_key = "BID-#{bid}"
           callback_key = "#{batch_key}-callbacks-#{event}"
-          already_processed, _, callbacks, queue, parent_bid, callback_batch = redis do |r|
+          already_processed, _, _, callbacks, queue, parent_bid, callback_params = redis do |r|
+            return unless r.exists?(batch_key)
             r.multi do
               r.hget(batch_key, event)
+              r.hset(batch_key, "closed", true)
               r.hset(batch_key, event, true)
               r.smembers(callback_key)
               r.hget(batch_key, "callback_queue")
               r.hget(batch_key, "parent_bid")
-              r.hget(batch_key, "callback_batch")
+              r.hget(batch_key, "callback_params")
             end
           end
 
@@ -328,6 +306,7 @@ module CanvasSync
 
           queue ||= "default"
           parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
+          callback_params = JSON.parse(callback_params) if callback_params.present?
           callback_args = callbacks.reduce([]) do |memo, jcb|
             cb = JSON.load(jcb)
             memo << [cb['callback'], event.to_s, cb['opts'], bid, parent_bid]
@@ -335,38 +314,32 @@ module CanvasSync
 
           opts = {"bid" => bid, "event" => event}
 
-          # Run callback batch finalize synchronously
-          if callback_batch
-            # Extract opts from cb_args or use current
-            # Pass in stored event as callback finalize is processed on complete event
-            cb_opts = callback_args.first&.at(2) || opts
+          if callback_args.present? && !callback_params.present?
+            logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
 
-            logger.debug {"Run callback batch bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
-            # Finalize now
-            finalizer = Batch::Callback::Finalize.new
-            status = Status.new bid
-            finalizer.dispatch(status, cb_opts)
+            with_batch(parent_bid) do
+              cb_batch = self.new
+              cb_batch.callback_params = {
+                for_bid: bid,
+                event: event,
+              }
+              opts['callback_bid'] = cb_batch.bid
 
-            return
+              logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
+              cb_batch.jobs do
+                push_callbacks callback_args, queue
+              end
+            end
           end
 
-          logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
-
-          if callback_args.empty?
-            # Finalize now
-            finalizer = Batch::Callback::Finalize.new
-            status = Status.new bid
-            finalizer.dispatch(status, opts)
-          else
-            # Otherwise finalize in sub batch complete callback
-            cb_batch = self.new
-            cb_batch.callback_batch = true
-            logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
-            cb_batch.on(:complete, "#{Batch::Callback::Finalize.to_s}#dispatch", opts)
-            cb_batch.jobs do
-              push_callbacks callback_args, queue
-            end
+          if callback_params.present?
+            opts['origin'] = callback_params
           end
+
+          logger.debug {"Run batch finalizer bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
+          finalizer = Batch::Callback::Finalize.new
+          status = Status.new bid
+          finalizer.dispatch(status, opts)
         end
 
         def cleanup_redis(bid)
@@ -381,6 +354,7 @@ module CanvasSync
               "BID-#{bid}-batches-success",
               "BID-#{bid}-batches-complete",
               "BID-#{bid}-batches-failed",
+              "BID-#{bid}-bids",
               "BID-#{bid}-jids",
             )
           end
@@ -397,7 +371,7 @@ module CanvasSync
         private
 
         def push_callbacks(args, queue)
-          Batch::Callback::Worker.enqueue_all(args, queue)
+          Batch::Callback::worker_class.enqueue_all(args, queue)
         end
       end
     end

data/lib/canvas_sync/job_batches/callback.rb

@@ -2,6 +2,7 @@ module CanvasSync
   module JobBatches
     class Batch
       module Callback
+        mattr_accessor :worker_class
 
         VALID_CALLBACKS = %w[success complete dead].freeze
 
@@ -47,41 +48,28 @@ module CanvasSync
           end
         end
 
-        if defined?(::Sidekiq)
-          class SidekiqCallbackWorker
-            include ::Sidekiq::Worker
-            include CallbackWorkerCommon
-
-            def self.enqueue_all(args, queue)
-              return if args.empty?
-
-              ::Sidekiq::Client.push_bulk(
-                'class' => self,
-                'args' => args,
-                'queue' => queue
-              )
-            end
-          end
-          Worker = SidekiqCallbackWorker
-        else
-          Worker = ActiveJobCallbackWorker
-        end
+        worker_class = ActiveJobCallbackWorker
 
         class Finalize
-          def dispatch status, opts
+          def dispatch(status, opts)
+            is_callback_batch = opts['origin'].present?
+            has_callback_batch = opts['callback_bid'].present?
+
             bid = opts["bid"]
-            callback_bid = status.bid
             event = opts["event"].to_sym
-            callback_batch = bid != callback_bid
 
-            Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
+            Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{is_callback_batch}"}
 
             batch_status = Status.new bid
             send(event, bid, batch_status, batch_status.parent_bid)
 
-            # Different events are run in different callback batches
-            Batch.cleanup_redis callback_bid if callback_batch
-            Batch.cleanup_redis bid if event == :success
+            if event == :success && !has_callback_batch
+              Batch.cleanup_redis(bid)
+            end
+
+            if event == :success && is_callback_batch && opts['origin']['event'].to_sym == :success
+              Batch.cleanup_redis(opts['origin']['for_bid'])
+            end
           end
 
           def success(bid, status, parent_bid)
@@ -102,8 +90,9 @@ module CanvasSync
                 r.scard("BID-#{parent_bid}-failed")
               end
             end
-            # if job finished successfully and parent batch completed call parent complete callback
-            # Success callback is called after complete callback
+
+            # If the job finished successfully and parent batch is completed, call parent :complete callback
+            # Parent :success callback will be called by its :complete callback
             if complete == children && pending == failure
               Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
               Batch.enqueue_callbacks(:complete, parent_bid)
@@ -119,10 +108,12 @@ module CanvasSync
               end
             end
 
-            # if we batch was successful run success callback
+            # If the batch was successful run :success callback, which will call the parent's :complete callback (if necessary)
+            # Also, only trigger the success callback if the :complete callback_batch was successful
             if pending.to_i.zero? && children == success
               Batch.enqueue_callbacks(:success, bid)
 
+            # otherwise check for a parent and call its :complete if needed
             elsif parent_bid
               # if batch was not successfull check and see if its parent is complete
               # if the parent is complete we trigger the complete callback