canvas_sync 0.18.10 → 0.19.0.beta1

data/lib/canvas_sync/job_batches/pool.rb

@@ -3,7 +3,7 @@ module CanvasSync
     class Pool
       include RedisModel
 
-      HINCR_MAX = RedisScript.new(Pathname.new(__FILE__) + "../hincr_max.lua")
+      POOL_REFILL = RedisScript.new(Pathname.new(__FILE__) + "../pool_refill.lua")
 
       attr_reader :pid
       redis_attr :description
@@ -36,7 +36,7 @@ module CanvasSync
         add_jobs([job_desc])
       end
 
-      def add_jobs(job_descs)
+      def add_jobs(job_descs, skip_refill: false)
         job_descs.each do |job_desc|
           wrapper = Batch.new
           wrapper.description = "Pool Job Wrapper (PID: #{pid})"
@@ -52,7 +52,7 @@ module CanvasSync
 
           push_job_to_pool(job_desc)
         end
-        refill_allotment
+        refill_allotment unless skip_refill
       end
 
       def keep_open!
@@ -64,19 +64,13 @@ module CanvasSync
             let_close!
           end
         else
-          redis.hset(redis_key, 'keep_open', true)
+          redis.hset(redis_key, 'keep_open', 'true')
         end
       end
 
       def let_close!
-        _, active_count = redis.multi do |r|
-          r.hset(redis_key, 'keep_open', false)
-          r.hincrby(redis_key, "active_count", 0)
-        end
-
-        if active_count == 0 && pending_count == 0
-          cleanup_redis if clean_when_empty
-        end
+        redis.hset(redis_key, 'keep_open', 'false')
+        cleanup_if_empty
       end
 
       def cleanup_redis
@@ -90,8 +84,26 @@ module CanvasSync
         end
       end
 
-      def active_count
-        redis.hincrby(redis_key, "active_count", 0)
+      def cleanup_if_empty
+        self.order
+
+        activec, pactivec, pendingc, clean_when_empty, keep_open = redis.multi do |r|
+          r.scard("#{redis_key}-active")
+          r.hincrby(redis_key, "_active_count", 0)
+          pending_count(r)
+          r.hget(redis_key, 'clean_when_empty')
+          r.hget(redis_key, 'keep_open')
+        end
+
+        return if keep_open == 'true' || clean_when_empty == 'false'
+
+        if activec <= 0 && pactivec <= 0 && pendingc <= 0
+          cleanup_redis
+        end
+      end
+
+      def active_count(r = redis)
+        r.scard("#{redis_key}-active") + r.hincrby(redis_key, "_active_count", 0)
       end
 
       def pending_count(r = redis)
@@ -108,24 +120,9 @@ module CanvasSync
       end
 
       def job_checked_in(status, options)
-        active_count, pending_count = redis do |r|
-          return unless r.exists?(redis_key)
-
-          # Make sure this is loaded outside of the pipeline
-          self.order
-
-          redis.multi do |r|
-            r.hincrby(redis_key, "active_count", -1)
-            self.pending_count(r)
-          end
-        end
-
-        added_count = refill_allotment
-        if active_count == 0 && added_count == 0 && pending_count == 0
-          if clean_when_empty && redis.hget(redis_key, 'keep_open') != 'true'
-            cleanup_redis
-          end
-        end
+        redis.srem("#{redis_key}-active", status.bid)
+        active_count = refill_allotment
+        cleanup_if_empty unless active_count > 0
       end
 
       def self.job_checked_in(status, options)
@@ -133,6 +130,14 @@ module CanvasSync
         from_pid(pid).job_checked_in(status, options)
       end
 
+      # Administrative/console method to cleanup expired pools from the WebUI
+      def self.cleanup_redis_index!
+        suffixes = ["", "-active", "-jobs"]
+        r.zrangebyscore("pools", "0", Batch::BID_EXPIRE_TTL.seconds.ago.to_i).each do |pid|
+          r.zrem("pools", pid) if Batch.cleanup_redis_index_for("POOLID-#{pid}", suffixes)
+        end
+      end
+
       protected
 
       def redis_key
@@ -140,31 +145,52 @@ module CanvasSync
       end
 
       def refill_allotment
-        jobs_added = 0
-        limit = concurrency.to_i
-        redis do |r|
-          current_count = 0
-          while true
-            current_count = HINCR_MAX.call(r, [redis_key], ["active_count", limit]).to_i
-            if current_count < limit
-              job_desc = pop_job_from_pool
-              if job_desc.present?
-                Batch.new(job_desc[:pool_wrapper_batch]).jobs do
-                  ChainBuilder.enqueue_job(job_desc)
-                end
-                jobs_added += 1
-              else
-                r.hincrby(redis_key, "active_count", -1)
-                break
-              end
-            else
-              break
+        active_count, job_descs = POOL_REFILL.call(redis, [redis_key, "#{redis_key}-jobs", "#{redis_key}-active"], [])
+        return active_count if active_count < 0
+
+        pending_job_descs = job_descs.dup
+
+        added_jobs = []
+        failed_to_add_jobs = []
+        add_exception = nil
+
+        while pending_job_descs.count > 0
+          begin
+            job_json = pending_job_descs.shift
+            job_desc = ::ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]&.symbolize_keys
+
+            wbid = job_desc[:pool_wrapper_batch]
+
+            Batch.new(wbid).jobs do
+              ChainBuilder.enqueue_job(job_desc)
             end
+
+            added_jobs << wbid
+          rescue => ex
+            failed_to_add_jobs << job_json
+            add_exception = ex
           end
+        end
+
+        redis.multi do |r|
+          r.sadd("#{redis_key}-active", added_jobs) if added_jobs.count > 0
+          # Release reserved slots now that we've added the jobs to `-active`
+          r.hincrby(redis_key, "_active_count", -job_descs.count)
+
           r.expire(redis_key, Batch::BID_EXPIRE_TTL)
+          r.expire("#{redis_key}-active", Batch::BID_EXPIRE_TTL)
           r.expire("#{redis_key}-jobs", Batch::BID_EXPIRE_TTL)
         end
-        jobs_added
+
+        # If this happens, we end up in a bad state (as we don't try to re-add items to the pool or refill_allotment again), but
+        #   this should be a _really_ rare case that should only occur if we've lost connection to Redis or something, so we're
+        #   operating on the assumption that if we get here, any recovery logic will fail too
+        if add_exception.present?
+          Batch.logger.error {"Error popping jobs from Pool #{pid}: #{add_exception}"}
+          raise add_exception
+        end
+
+        active_count + added_jobs.count
       end
 
       def push_job_to_pool(job_desc)
@@ -183,13 +209,15 @@ module CanvasSync
           when :priority
             r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
           end
+          r.expire(redis_key, Batch::BID_EXPIRE_TTL)
           r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
         end
       end
 
+      # @deprecated
       def pop_job_from_pool
         jobs_key = "#{redis_key}-jobs"
-        order = self.order
+        order = self.order || 'fifo'
 
         job_json = case order.to_sym
         when :fifo
@@ -199,7 +227,7 @@ module CanvasSync
         when :random
           redis.spop(jobs_key)
         when :priority
-          redis.zpopmax(jobs_key)
+          redis.zpopmax(jobs_key)&.[](0)
         end
 
         return nil unless job_json.present?
@@ -214,6 +242,7 @@ module CanvasSync
 
       def flush_pending_attrs
         super
+        redis.expire(redis_key, Batch::BID_EXPIRE_TTL)
         redis.zadd("pools", created_at, pid)
       end
 

data/lib/canvas_sync/job_batches/pool_refill.lua

@@ -0,0 +1,40 @@
+
+local poolkey = KEYS[1]
+local qkey = KEYS[2]
+local activekey = KEYS[3]
+
+if redis.call('EXISTS', poolkey) == 0 then
+    return { -1, {} } -- pool doesn't exist
+end
+
+local pool_type = redis.call('HGET', poolkey, "order")
+local allotment = tonumber(redis.call("HGET", poolkey, "concurrency"))
+local active = redis.call("SCARD", activekey) + (redis.call("HGET", poolkey, "_active_count") or 0)
+
+local pop_count = allotment - active
+
+local popped_items = {}
+
+if pop_count > 0 then
+    if pool_type == "fifo" then
+        popped_items = redis.call("LPOP", qkey, pop_count) or {}
+    elseif pool_type == "lifo" then
+        popped_items = redis.call("RPOP", qkey, pop_count) or {}
+    elseif pool_type == "random" then
+        popped_items = redis.call("SPOP", qkey, pop_count) or {}
+    elseif pool_type == "priority" then
+        local temp_items = redis.call("ZPOPMAX", qkey, pop_count) or {}
+        for i,v in ipairs(temp_items) do
+            if i % 2 == 1 then
+                table.insert(popped_items, v)
+            end
+        end
+    end
+end
+
+-- Reserve slots for these jobs while we return to Ruby and deserialize them
+--  This could also be inlined by just storing a key in the queue and storing parameters
+--  in a Hash, but this seems more efficient.
+redis.call('HINCRBY', poolkey, "_active_count", #popped_items)
+
+return { active, popped_items }

data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/css/styles.less

@@ -1,6 +1,7 @@
 
 @color-green: #25c766;
-@color-red: #c7254e;
+@color-darkred: #8f0000;
+@color-red: #e03963;
 @color-yellow: #c4c725;
 
 .code-wrap.batch-context .args-extended {
@@ -30,6 +31,9 @@
             &.failed {
                 color: @color-red;
             }
+            &.dead {
+                color: @color-darkred;
+            }
             &.success {
                 color: @color-green;
             }
@@ -156,7 +160,7 @@
             }
 
             .status-block {
-                width: 10em;
+                width: 12em;
                 text-align: center;
             }
         }

data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/js/batch_tree.js

@@ -12,6 +12,8 @@ const StatusBlock = (props) => html`
     |
     <span class="tree-stat failed">${props.failed_count}</span>
     |
+    <span class="tree-stat dead">${props.dead_count}</span>
+    |
     <span class="tree-stat success">${props.successful_count}</span>
     /
     <span class="tree-stat total">${props.total_count}</span>
@@ -96,7 +98,7 @@ class TreeRoot extends Component {
     const tree_data = JSON.parse(document.querySelector('#batch-tree #initial-data').innerHTML);
     return html`
       <div class="tree-header">
-        <${StatusBlock} pending_count="pending" failed_count="failed" successful_count="successful" total_count="total" />
+        <${StatusBlock} pending_count="pending" failed_count="failed" dead_count="dead" successful_count="successful" total_count="total" />
       </div>
       <${TreeLevel} data=${tree_data} />
     `;

data/lib/canvas_sync/job_batches/sidekiq/web/views/_batches_table.erb

@@ -5,12 +5,13 @@
             <th rowspan="2"><%= t('BID') %></th>
             <th rowspan="2"><%= t('Description') %></th>
 
-            <th colspan="4"><%= t('Jobs') %></th>
+            <th colspan="5"><%= t('Jobs') %></th>
             <th colspan="4"><%= t('Sub-Batches') %></th>
         </tr>
         <tr>
             <th><%= t('Pending') %></th>
             <th><%= t('Failed') %></th>
+            <th><%= t('Dead') %></th>
             <th><%= t('Complete') %></th>
             <th><%= t('Total') %></th>
 
@@ -30,6 +31,7 @@
 
             <td><%= status.pending %></th>
             <td><%= status.failures %></th>
+            <td><%= status.dead %></th>
             <td><%= status.completed_count %></th>
             <td><%= status.job_count %></th>
 

data/lib/canvas_sync/job_batches/sidekiq/web/views/_jobs_table.erb

@@ -1,6 +1,7 @@
 <table class="table table-striped table-bordered table-hover">
     <thead>
         <tr>
+            <th><%= t('JID') %></th>
             <th><%= t('Job Class') %></th>
             <th><%= t('Parameters') %></th>
         </tr>
@@ -8,6 +9,7 @@
 
     <% @jobs.each do |job_desc| %>
         <tr>
+            <td><%= job_desc[:jid] %></td>
             <td><%= job_desc['job'] %></td>
             <td>
                 <code class="code-wrap">

data/lib/canvas_sync/job_batches/sidekiq/web/views/pool.erb

@@ -60,6 +60,9 @@
                 <th><%= t('Job Class') %></th>
                 <th><%= t('Parameters') %></th>
                 <th><%= t('Wrapper Batch BID') %></th>
+                <% if @pool.order == 'priority' %>
+                    <th><%= t('Priority') %></th>
+                <% end %>
             </tr>
         </thead>
 
@@ -68,10 +71,16 @@
                 <td><%= job_desc['job'] %></td>
                 <td>
                     <code class="code-wrap">
-                        <div class="args-extended"><%= job_desc['parameters'].to_json  %></div>
+                        <div class="args-extended">
+                            <%= job_desc['args']&.to_json %>
+                            <%= job_desc['kwargs']&.to_json %>
+                        </div>
                     </code>
                 </td>
                 <td><a href="<%= root_path %>batches/<%= job_desc['pool_wrapper_batch'] %>"><%= job_desc['pool_wrapper_batch'] %></a></td>
+                <% if @pool.order == 'priority' %>
+                    <td><%= job_desc['priority'] %></td>
+                <% end %>
             </tr>
         <% end %>
     </table>

data/lib/canvas_sync/job_batches/sidekiq/web.rb

@@ -45,7 +45,9 @@ module CanvasSync::JobBatches::Sidekiq
         @sub_batches = @sub_batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
 
         @current_jobs_page, @total_jobs_size, @jobs = page("BID-#{@batch.bid}-jids", params['job_page'], @count)
-        @jobs = @jobs.map {|jid, score| jid }
+        @jobs = @jobs.map do |jid, score|
+          { jid: jid, }
+        end
 
         erb(get_template(:batch))
       end
@@ -68,6 +70,7 @@ module CanvasSync::JobBatches::Sidekiq
               jobs_total = r.hget("BID-#{bid}", "job_count").to_i
               jobs_pending = r.hget("BID-#{bid}", 'pending').to_i
               jobs_failed = r.scard("BID-#{bid}-failed").to_i
+              jobs_dead = r.scard("BID-#{bid}-dead").to_i
               jobs_success = jobs_total - jobs_pending
 
               batches_total = r.hget("BID-#{bid}", 'children').to_i
@@ -90,6 +93,7 @@ module CanvasSync::JobBatches::Sidekiq
                   pending_count: jobs_pending,
                   successful_count: jobs_success,
                   failed_count: jobs_failed,
+                  dead_count: jobs_dead,
                   total_count: jobs_total,
                   # items: batches.map{|b| layer_data[b] },
                 },

data/lib/canvas_sync/job_batches/status.rb

@@ -21,6 +21,10 @@ module CanvasSync
           Batch.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
         end
 
+        def dead
+          Batch.redis { |r| r.scard("BID-#{bid}-dead") }.to_i
+        end
+
         def completed_count
           job_count - pending
         end

data/lib/canvas_sync/jobs/canvas_process_waiter.rb

@@ -7,9 +7,9 @@ module CanvasSync::Jobs
       response = canvas_sync_client.get(progress_url)
       status = kwargs[:status_key].present? ? response[kwargs[:status_key]] : response['workflow_state'] || response['status']
 
-      if %w[completed complete].include? status
+      if %w[completed complete imported imported_with_messages].include? status
         InvokeCallbackWorker.perform_later(build_next_job(next_job, kwargs, response)) if next_job
-      elsif %w[failed error].include? status
+      elsif %w[failed error failed_with_messages].include? status
         if kwargs[:on_failure].is_a?(Hash)
           InvokeCallbackWorker.perform_later(build_next_job(kwargs[:on_failure], kwargs, response))
         else
@@ -33,40 +33,8 @@ module CanvasSync::Jobs
 
     # This is a separate job so that, if it fails and a retry is triggered, it doesn't query the API needlessly
     class InvokeCallbackWorker < ActiveJob::Base
-      # rubocop:disable Metrics/PerceivedComplexity
       def perform(job)
-        job = job.symbolize_keys
-
-        params = job[:args] || []
-        params << job[:kwargs].symbolize_keys if job[:kwargs]
-        # params[-1] = params[-1].symbolize_keys if params[-1].is_a?(Hash)
-
-        if job[:model]
-          model_class = load_constant(job[:model])
-          find_by = job[:find_by]
-          target = find_by.is_a?(Hash) ? model_class.find_by(find_by) : model_class.find_by(id: find_by)
-          target.send(job[:method], *params)
-        elsif job[:class]
-          target = load_constant(job[:class])
-          target.send(job[:method], *params)
-        elsif job[:instance_of]
-          target = load_constant(job[:instance_of]).new
-          target.send(job[:method], *params)
-        elsif job[:job]
-          job_class = load_constant(job[:job])
-          job_class = job_class.set(job[:options]) if job[:options].present?
-          if job_class < ActiveJob::Base
-            job_class.perform_later(*params)
-          else
-            job_class.perform_async(*params)
-          end
-        end
-      end
-      # rubocop:enable Metrics/PerceivedComplexity
-
-      def load_constant(const)
-        const = const.constantize if const.is_a?(String)
-        const
+        CanvasSync::MiscHelper.invoke_task(job)
       end
     end
   end

data/lib/canvas_sync/misc_helper.rb

@@ -11,5 +11,53 @@ module CanvasSync
         ActiveRecord::Type::Boolean.new.deserialize(v)
       end
     end
+
+    def self.invoke_task(job)
+      job = job.symbolize_keys
+
+      job_args = job[:args] || job[:parameters] || []
+      job_kwargs = job[:kwargs] || {}
+
+      if (mthd = job[:method]) && !(job[:class] || job[:instance_of] || job[:model])
+        if mthd.include?('#')
+          clazz, method = clazz.split("#")
+          job[:instance_of] = clazz
+          job[:method] = method
+        elsif mthd.include?('.')
+          clazz, method = mthd.split(".")
+          job[:class] = clazz
+          job[:method] = method
+        end
+      end
+
+      if job[:model]
+        # TODO Support globalid
+        model_class = load_constant(job[:model])
+        find_by = job[:find_by]
+        target = find_by.is_a?(Hash) ? model_class.find_by(find_by) : model_class.find_by(id: find_by)
+        target.send(job[:method], *job_args, **job_kwargs)
+      elsif job[:class]
+        target = load_constant(job[:class])
+        target.send(job[:method], *job_args, **job_kwargs)
+      elsif job[:instance_of]
+        target = load_constant(job[:instance_of]).new
+        target.send(job[:method], *job_args, **job_kwargs)
+      elsif job[:job]
+        job_class = load_constant(job[:job])
+        job_class = job_class.set(job[:options]) if job[:options].present?
+        if job_class < ActiveJob::Base
+          job_class.perform_later(*job_args, **job_kwargs)
+        else
+          job_args << job_kwargs.symbolize_keys if job_kwargs
+          # job_args[-1] = job_args[-1].symbolize_keys if job_args[-1].is_a?(Hash)
+          job_class.perform_async(*job_args)
+        end
+      end
+    end
+
+    def self.load_constant(const)
+      const = const.constantize if const.is_a?(String)
+      const
+    end
   end
 end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.18.10".freeze
+  VERSION = "0.19.0.beta1".freeze
 end

data/lib/canvas_sync.rb

@@ -193,8 +193,7 @@ module CanvasSync
 
       term_parent_chain = current_chain
 
-      per_term_chain = JobBatches::ChainBuilder.new(model_job_map[:terms])
-      per_term_chain.params[:term_scope] = term_scope
+      per_term_chain = JobBatches::ChainBuilder.build(model_job_map[:terms], term_scope: term_scope)
       current_chain = per_term_chain
 
       term_scoped_models.each do |mdl|
@@ -206,7 +205,7 @@ module CanvasSync
       )
 
       # Skip syncing terms if not required
-      unless current_chain.empty? || (models & ['terms']).present?
+      if !current_chain.empty? || (models & ['terms']).present?
         term_parent_chain << per_term_chain
       end
 
@@ -226,8 +225,6 @@ module CanvasSync
       globals: {},
       &blk
     )
-      root_chain = JobBatches::ChainBuilder.new(CanvasSync::Jobs::BeginSyncChainJob)
-
       global_options = {
         legacy_support: legacy_support,
         updated_after: updated_after,
@@ -237,11 +234,7 @@ module CanvasSync
       global_options[:account_id] = account_id if account_id.present?
       global_options.merge!(globals) if globals
 
-      root_chain.params[1] = global_options
-
-      root_chain.apply_block(&blk)
-
-      root_chain
+      JobBatches::ChainBuilder.build(CanvasSync::Jobs::BeginSyncChainJob, [], global_options, &blk)
     end
 
     def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)