canvas_sync 0.17.30 → 0.17.33

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef8cd570185754998d8c14cd67f87c6d60e3507f7996925bdc4a425cd7d43f1d
-  data.tar.gz: 931416870261971aaeccfc59bae91972c8d7a7277aab9da4ac18610c3352a7fe
+  metadata.gz: e8585726d4c1e8cf1627897ed3b7e8af68a76f86d3ea35b9ba3ac0b7eaa56ff9
+  data.tar.gz: 1318778a8bbaadb0f5069a1b9bfdb106f0ae2e8ed5124079a8e9d7c4583be359
 SHA512:
-  metadata.gz: 7262c43e7395e4c9de96a9d5c4688ff85299fcab3f97abcea719b6f73c36fb519c23ccf7a224605a8552d0e547fe135ecc055ac7f664d240f0cc1dd8a932e2bf
-  data.tar.gz: 524b36471a7b135856c2e3bd935e8ab7e057616c1bd3ed4a6d5e7a6d66c98f3236cf87d041f946a5e4ac149bef9830100619ba9e61568971f0e17898ae2f44f3
+  metadata.gz: 0ef7f29ce974199f5f1f26d43e4827ee77d93245b7b1ac4339fd33d03cbce1f6707df45d97ea533f24c21a45678e22b75db611a5a770ef073bf70fd0f37aa797
+  data.tar.gz: cc8892a896e4950e251c202e39ad4f17f012d8a1d39e588201ff27d927cd7e9d7160836fd05fd6892838cc3466dda5fd3e840121392cdb86269c5908cf0b0cea

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -88,7 +88,7 @@ module CanvasSync::Concerns
       end
 
       def unlink_column(key)
-        @map_def.delete(key)
+        @map_def[:report_columns].delete(key)
       end
 
       def link_column(m, type: nil, &blk)

data/lib/canvas_sync/engine.rb

@@ -32,24 +32,28 @@ module CanvasSync
     initializer :integrate_pandapal do
       require 'panda_pal'
 
-      if PandaPal::Organization.respond_to?(:scheduled_task)
-        if PandaPal::Organization.respond_to?(:define_setting)
-          PandaPal::Organization.define_setting(:canvas_sync, {
-            type: 'Hash',
-            required: false,
-            properties: {
-              job_log_retention: { **RETENTION_TYPE },
-              sync_batch_retention: { **RETENTION_TYPE },
-            }
-          })
-        end
+      Rails.application.reloader.to_prepare do
+        if PandaPal::Organization.respond_to?(:scheduled_task)
+          if PandaPal::Organization.respond_to?(:define_setting)
+            PandaPal::Organization.define_setting(:canvas_sync, {
+              type: 'Hash',
+              required: false,
+              properties: {
+                job_log_retention: { **RETENTION_TYPE },
+                sync_batch_retention: { **RETENTION_TYPE },
+              }
+            })
+          end
 
-        PandaPal::Organization.scheduled_task '0 0 3 * * *', :clean_canvas_sync_logs do
-          job_log_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :job_log_retention) || '3 months', keep_zero: true).seconds.ago
-          JobLog.where('updated_at < ?', job_log_retention).delete_all
+          unless PandaPal::Organization.task_scheduled?(:clean_canvas_sync_logs)
+            PandaPal::Organization.scheduled_task '0 0 3 * * *', :clean_canvas_sync_logs do
+              job_log_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :job_log_retention) || '3 months', keep_zero: true).seconds.ago
+              JobLog.where('updated_at < ?', job_log_retention).delete_all
 
-          sync_batch_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :sync_batch_retention) || '6 months', keep_zero: true).seconds.ago
-          SyncBatch.where('updated_at < ?', sync_batch_retention).delete_all
+              sync_batch_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :sync_batch_retention) || '6 months', keep_zero: true).seconds.ago
+              SyncBatch.where('updated_at < ?', sync_batch_retention).delete_all
+            end
+          end
         end
       end
     rescue LoadError

data/lib/canvas_sync/job_batches/batch.rb

@@ -28,7 +28,7 @@ module CanvasSync
 
       delegate :redis, to: :class
 
-      BID_EXPIRE_TTL = 2_592_000
+      BID_EXPIRE_TTL = 90.days.to_i
       SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
       BID_HIERARCHY = RedisScript.new(Pathname.new(__FILE__) + "../hier_batch_ids.lua")
 
@@ -101,6 +101,8 @@ module CanvasSync
               r.hincrby("BID-#{parent_bid}", "children", 1)
               r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
               r.zadd("BID-#{parent_bid}-bids", created_at, bid)
+            else
+              r.zadd("BID-ROOT-bids", created_at, bid)
             end
           end
 
@@ -369,6 +371,7 @@ module CanvasSync
           logger.debug {"Cleaning redis of batch #{bid}"}
           redis do |r|
             r.zrem("batches", bid)
+            r.zrem("BID-ROOT-bids", bid)
             r.unlink(
               "BID-#{bid}",
               "BID-#{bid}-callbacks-complete",
@@ -450,6 +453,23 @@ module CanvasSync
           end
         end
 
+        def uget(key)
+          Batch.redis do |r|
+            case r.type(key)
+            when 'string'
+              r.get(key)
+            when 'list'
+              r.lrange(key, 0, -1)
+            when 'hash'
+              r.hgetall(key)
+            when 'set'
+              r.smembers(key)
+            when 'zset'
+              r.smembers(key, 0, -1)
+            end
+          end
+        end
+
         def method_missing(method_name, *arguments, &block)
           Batch.redis do |r|
             r.send(method_name, *arguments, &block)

data/lib/canvas_sync/job_batches/callback.rb

@@ -27,7 +27,7 @@ module CanvasSync
 
             if clazz && object = Object.const_get(clazz)
               target = target == :instance ? object.new : object
-              if target.respond_to?(method)
+              if target.respond_to?(method, true)
                 target.send(method, status, opts)
               else
                 Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")

data/lib/canvas_sync/job_batches/pool.rb

@@ -44,7 +44,7 @@ module CanvasSync
           wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
           wrapper.jobs {}
 
-          job_desc = job_desc.with_indifferent_access
+          job_desc = job_desc.symbolize_keys
           job_desc = job_desc.merge!(
             job: job_desc[:job].to_s,
             pool_wrapper_batch: wrapper.bid,
@@ -149,7 +149,7 @@ module CanvasSync
             if current_count < limit
               job_desc = pop_job_from_pool
               if job_desc.present?
-                Batch.new(job_desc['pool_wrapper_batch']).jobs do
+                Batch.new(job_desc[:pool_wrapper_batch]).jobs do
                   ChainBuilder.enqueue_job(job_desc)
                 end
                 jobs_added += 1
@@ -170,7 +170,7 @@ module CanvasSync
       def push_job_to_pool(job_desc)
         jobs_key = "#{redis_key}-jobs"
         # This allows duplicate jobs when a Redis Set is used
-        job_desc['_pool_random_key_'] = SecureRandom.urlsafe_base64(10)
+        job_desc[:_pool_random_key_] = SecureRandom.urlsafe_base64(10)
         job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
         order = self.order
 
@@ -204,7 +204,7 @@ module CanvasSync
 
         return nil unless job_json.present?
 
-        ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
+        ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]&.symbolize_keys
       end
 
       def self.redis(&blk)

data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb

@@ -12,7 +12,7 @@ module CanvasSync::JobBatches::Sidekiq
       end
 
       def drain_zset(key)
-        items, _ = Batch.redis do |r|
+        items, _ = CanvasSync::JobBatches::Batch.redis do |r|
           r.multi do |r|
             r.zrange(key, 0, -1)
             r.zremrangebyrank(key, 0, -1)

data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb

@@ -10,6 +10,16 @@
                 <th colspan="2" scope=row><%= t('Batch') %></td>
                 <td><%= @batch.bid %></td>
             </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Parent') %></td>
+                <td>
+                    <% if @batch.parent_bid.present? %>
+                        <a href="<%= root_path %>batches/<%= @batch.parent_bid %>"><%= @batch.parent_bid %></a>
+                    <% else %>
+                        ROOT
+                    <% end %>
+                </td>
+            </tr>
             <tr>
                 <th colspan="2" scope=row><%= t('Started') %></td>
                 <td><%= safe_relative_time(@batch.created_at.to_f) %></td>

data/lib/canvas_sync/job_batches/sidekiq/web.rb

@@ -10,6 +10,8 @@ require_relative "web/helpers"
 module CanvasSync::JobBatches::Sidekiq
   module Web
     DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
+    Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
+    Sidekiq::WebHelpers::SAFE_QPARAMS << 'count'
 
     def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       app.helpers do
@@ -24,7 +26,9 @@ module CanvasSync::JobBatches::Sidekiq
 
       app.get "/batches" do
         @count = (params['count'] || 25).to_i
-        @current_page, @total_size, @batches = page('batches', params['page'], @count)
+
+        source_key = params['all_batches'] ? "batches" : "BID-ROOT-bids"
+        @current_page, @total_size, @batches = page(source_key, params['page'], @count)
         @batches = @batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
 
         erb(get_template(:batches))

data/lib/canvas_sync/jobs/sync_submissions_job.rb

@@ -10,7 +10,7 @@ module CanvasSync
       def perform(options)
         super(
           "proserv_student_submissions_csv",
-          merge_report_params(options, { include_all: options[:include_all] }, {}),
+          merge_report_params(options, { include_all: options[:include_all], enrollment_batch_size: options[:enrollment_batch_size] }, {}),
           CanvasSync::Processors::SubmissionsProcessor.to_s,
           {},
         )

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.30".freeze
+  VERSION = "0.17.33".freeze
 end

data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe CanvasSync::Jobs::SyncSubmissionsJob do
     context "a term id is in the global_options" do
       it "enqueues a ReportStarter for the proserv_student_submissions_csv for the given term" do
         expect_any_instance_of(Bearcat::Client).to receive(:start_report)
-          .with("self", "proserv_student_submissions_csv", { parameters: { enrollment_term_id: 1, include_all: nil } })
+          .with("self", "proserv_student_submissions_csv", { parameters: { enrollment_term_id: 1, include_all: nil, enrollment_batch_size: nil } })
           .and_return("id" => 1)
 
         expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.17.30
+  version: 0.17.33
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-04-05 00:00:00.000000000 Z
+date: 2022-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails