canvas_sync 0.17.31 → 0.17.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09cebb9b63254bcdf9e96ae0133d882fcf800ad405f6d53080793e8f3f0c052f'
-  data.tar.gz: 40d27ea858ac931d006ccbab54f7ee1b725166ad3fec4ecb369f6e3f0fa611f1
+  metadata.gz: 8029fc55239810f8064f8d1c9fb4cd1f1756a7605581e7aced13e150e69c8b6b
+  data.tar.gz: 2deac9eb222ed8c636159c8100e8aa6cd59d2fa26d3b13b83c7be31cf7cbc0bd
 SHA512:
-  metadata.gz: de38e11de998bef7cca15c875d63b29984d2cdf4dd90436854484e5686c5aca4eb249fd864e24b66d532afcba63b3f7ddb62c5e5abf55d1e2c621fa6cb566af7
-  data.tar.gz: 8493bc7a6f100da697840f65aeaceacbacc95c224aa53c66129e19d5a88d61b1f59202252ac80b2e8d42fc4661f974b30bd1825b448e58b477ce9e33f71a9f2c
+  metadata.gz: b1d70b6a2493a49a9eeb48532e9438d02209d3de9609afa0a9bd09a5fe4c9816dcd9b3d7724ffa2385cd65f19b873901cce5fe5b733fa534e4c96ce785146f91
+  data.tar.gz: c985b3da6ca96c4120b852f0b4df938a6393a797dd8365ce3c9e61404b67623484930b9001e13172309399e745774bd63208097954d569e094f86c4a56e94e38

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -88,7 +88,7 @@ module CanvasSync::Concerns
       end
 
       def unlink_column(key)
-        @map_def.delete(key)
+        @map_def[:report_columns].delete(key)
       end
 
       def link_column(m, type: nil, &blk)

data/lib/canvas_sync/engine.rb

@@ -32,24 +32,28 @@ module CanvasSync
     initializer :integrate_pandapal do
       require 'panda_pal'
 
-      if PandaPal::Organization.respond_to?(:scheduled_task)
-        if PandaPal::Organization.respond_to?(:define_setting)
-          PandaPal::Organization.define_setting(:canvas_sync, {
-            type: 'Hash',
-            required: false,
-            properties: {
-              job_log_retention: { **RETENTION_TYPE },
-              sync_batch_retention: { **RETENTION_TYPE },
-            }
-          })
-        end
+      Rails.application.reloader.to_prepare do
+        if PandaPal::Organization.respond_to?(:scheduled_task)
+          if PandaPal::Organization.respond_to?(:define_setting)
+            PandaPal::Organization.define_setting(:canvas_sync, {
+              type: 'Hash',
+              required: false,
+              properties: {
+                job_log_retention: { **RETENTION_TYPE },
+                sync_batch_retention: { **RETENTION_TYPE },
+              }
+            })
+          end
 
-        PandaPal::Organization.scheduled_task '0 0 3 * * *', :clean_canvas_sync_logs do
-          job_log_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :job_log_retention) || '3 months', keep_zero: true).seconds.ago
-          JobLog.where('updated_at < ?', job_log_retention).delete_all
+          unless PandaPal::Organization.task_scheduled?(:clean_canvas_sync_logs)
+            PandaPal::Organization.scheduled_task '0 0 3 * * *', :clean_canvas_sync_logs do
+              job_log_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :job_log_retention) || '3 months', keep_zero: true).seconds.ago
+              JobLog.where('updated_at < ?', job_log_retention).delete_all
 
-          sync_batch_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :sync_batch_retention) || '6 months', keep_zero: true).seconds.ago
-          SyncBatch.where('updated_at < ?', sync_batch_retention).delete_all
+              sync_batch_retention = ChronicDuration.parse(settings.dig(:canvas_sync, :sync_batch_retention) || '6 months', keep_zero: true).seconds.ago
+              SyncBatch.where('updated_at < ?', sync_batch_retention).delete_all
+            end
+          end
         end
       end
     rescue LoadError

data/lib/canvas_sync/generators/templates/migrations/create_learning_outcomes.rb

@@ -0,0 +1,30 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateLearningOutcomes < ActiveRecord::Migration[5.1]
+  def change
+    create_table :learning_outcomes do |t|
+      t.bigint :canvas_id, null: false
+      t.integer :canvas_context_id
+      t.string :canvas_context_type
+      t.string :name
+      t.string :friendly_name
+      t.string :workflow_state
+      t.datetime :canvas_created_at
+      t.datetime :canvas_updated_at
+      t.string :migration_id
+      t.string :vendor_guid
+      t.string :low_grade
+      t.string :high_grade
+      t.string :calculation_method
+      t.string :calculation_int
+      t.integer :outcome_import_id
+      t.integer :root_account_ids, array: true, default: []
+      t.text :description
+
+      t.timestamps
+    end
+
+    add_index :learning_outcomes, :canvas_id, unique: true
+    add_index :learning_outcomes, [:canvas_context_id, :canvas_context_type], name: "index_learning_outcomes_on_context"
+  end
+end

data/lib/canvas_sync/generators/templates/models/learning_outcome.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# # <%= autogenerated_migration_warning %>
+
+class LearningOutcome < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  belongs_to :context, polymorphic: true, optional: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
+
+  api_syncable({
+    canvas_id: :id,
+    canvas_context_id: :context_id,
+    canvas_context_type: :context_type,
+    name: :title,
+    friendly_name: :display_name,
+    vendor_guid: :vendor_guid,
+    calculation_method: :calculation_method,
+    calculation_int: :calculation_int,
+    description: :description
+  }, ->(api) { api.get("/api/v1/outcomes/#{canvas_id}") })
+end

data/lib/canvas_sync/job_batches/batch.rb

@@ -28,7 +28,7 @@ module CanvasSync
 
       delegate :redis, to: :class
 
-      BID_EXPIRE_TTL = 2_592_000
+      BID_EXPIRE_TTL = 90.days.to_i
       SCHEDULE_CALLBACK = RedisScript.new(Pathname.new(__FILE__) + "../schedule_callback.lua")
       BID_HIERARCHY = RedisScript.new(Pathname.new(__FILE__) + "../hier_batch_ids.lua")
 
@@ -101,6 +101,8 @@ module CanvasSync
               r.hincrby("BID-#{parent_bid}", "children", 1)
               r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
               r.zadd("BID-#{parent_bid}-bids", created_at, bid)
+            else
+              r.zadd("BID-ROOT-bids", created_at, bid)
             end
           end
 
@@ -369,6 +371,7 @@ module CanvasSync
           logger.debug {"Cleaning redis of batch #{bid}"}
           redis do |r|
             r.zrem("batches", bid)
+            r.zrem("BID-ROOT-bids", bid)
             r.unlink(
               "BID-#{bid}",
               "BID-#{bid}-callbacks-complete",
@@ -450,6 +453,23 @@ module CanvasSync
           end
         end
 
+        def uget(key)
+          Batch.redis do |r|
+            case r.type(key)
+            when 'string'
+              r.get(key)
+            when 'list'
+              r.lrange(key, 0, -1)
+            when 'hash'
+              r.hgetall(key)
+            when 'set'
+              r.smembers(key)
+            when 'zset'
+              r.smembers(key, 0, -1)
+            end
+          end
+        end
+
         def method_missing(method_name, *arguments, &block)
           Batch.redis do |r|
             r.send(method_name, *arguments, &block)

data/lib/canvas_sync/job_batches/callback.rb

@@ -27,7 +27,7 @@ module CanvasSync
 
             if clazz && object = Object.const_get(clazz)
               target = target == :instance ? object.new : object
-              if target.respond_to?(method)
+              if target.respond_to?(method, true)
                 target.send(method, status, opts)
               else
                 Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")

data/lib/canvas_sync/job_batches/pool.rb

@@ -44,7 +44,7 @@ module CanvasSync
           wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
           wrapper.jobs {}
 
-          job_desc = job_desc.with_indifferent_access
+          job_desc = job_desc.symbolize_keys
           job_desc = job_desc.merge!(
             job: job_desc[:job].to_s,
             pool_wrapper_batch: wrapper.bid,
@@ -149,7 +149,7 @@ module CanvasSync
             if current_count < limit
               job_desc = pop_job_from_pool
               if job_desc.present?
-                Batch.new(job_desc['pool_wrapper_batch']).jobs do
+                Batch.new(job_desc[:pool_wrapper_batch]).jobs do
                   ChainBuilder.enqueue_job(job_desc)
                 end
                 jobs_added += 1
@@ -170,7 +170,7 @@ module CanvasSync
       def push_job_to_pool(job_desc)
         jobs_key = "#{redis_key}-jobs"
         # This allows duplicate jobs when a Redis Set is used
-        job_desc['_pool_random_key_'] = SecureRandom.urlsafe_base64(10)
+        job_desc[:_pool_random_key_] = SecureRandom.urlsafe_base64(10)
         job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
         order = self.order
 
@@ -204,7 +204,7 @@ module CanvasSync
 
         return nil unless job_json.present?
 
-        ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
+        ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]&.symbolize_keys
       end
 
       def self.redis(&blk)

data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb

@@ -12,7 +12,7 @@ module CanvasSync::JobBatches::Sidekiq
       end
 
       def drain_zset(key)
-        items, _ = Batch.redis do |r|
+        items, _ = CanvasSync::JobBatches::Batch.redis do |r|
           r.multi do |r|
             r.zrange(key, 0, -1)
             r.zremrangebyrank(key, 0, -1)

data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb

@@ -10,6 +10,16 @@
                 <th colspan="2" scope=row><%= t('Batch') %></td>
                 <td><%= @batch.bid %></td>
             </tr>
+            <tr>
+                <th colspan="2" scope=row><%= t('Parent') %></td>
+                <td>
+                    <% if @batch.parent_bid.present? %>
+                        <a href="<%= root_path %>batches/<%= @batch.parent_bid %>"><%= @batch.parent_bid %></a>
+                    <% else %>
+                        ROOT
+                    <% end %>
+                </td>
+            </tr>
             <tr>
                 <th colspan="2" scope=row><%= t('Started') %></td>
                 <td><%= safe_relative_time(@batch.created_at.to_f) %></td>

data/lib/canvas_sync/job_batches/sidekiq/web.rb

@@ -10,6 +10,8 @@ require_relative "web/helpers"
 module CanvasSync::JobBatches::Sidekiq
   module Web
     DEV_MODE = (defined?(Rails) && !Rails.env.production?) || !!ENV["SIDEKIQ_WEB_TESTING"]
+    Sidekiq::WebHelpers::SAFE_QPARAMS << 'all_batches'
+    Sidekiq::WebHelpers::SAFE_QPARAMS << 'count'
 
     def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       app.helpers do
@@ -24,7 +26,9 @@ module CanvasSync::JobBatches::Sidekiq
 
       app.get "/batches" do
         @count = (params['count'] || 25).to_i
-        @current_page, @total_size, @batches = page('batches', params['page'], @count)
+
+        source_key = params['all_batches'] ? "batches" : "BID-ROOT-bids"
+        @current_page, @total_size, @batches = page(source_key, params['page'], @count)
         @batches = @batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
 
         erb(get_template(:batches))

data/lib/canvas_sync/processors/model_mappings.yml

@@ -509,3 +509,58 @@ content_migrations:
     canvas_root_account_id:
       database_column_name: canvas_root_account_id
       type: integer
+
+learning_outcomes:
+  conflict_target: learning_outcome_id
+  report_columns:
+    learning_outcome_id:
+      database_column_name: canvas_id
+      type: integer
+    context_id:
+      database_column_name: canvas_context_id
+      type: integer
+    context_type:
+      database_column_name: canvas_context_type
+      type: string
+    name:
+      database_column_name: name
+      type: string
+    friendly_name:
+      database_column_name: friendly_name
+      type: string
+    workflow_state:
+      database_column_name: workflow_state
+      type: string
+    created_at:
+      database_column_name: canvas_created_at
+      type: datetime
+    updated_at:
+      database_column_name: canvas_updated_at
+      type: datetime
+    migration_id:
+      database_column_name: migration_id
+      type: string
+    vendor_guid:
+      database_column_name: vendor_guid
+      type: string
+    low_grade:
+      database_column_name: low_grade
+      type: string
+    high_grade:
+      database_column_name: high_grade
+      type: string
+    calculation_method:
+      database_column_name: calculation_method
+      type: string
+    calculation_int:
+      database_column_name: calculation_int
+      type: integer
+    outcome_import_id:
+      database_column_name: outcome_import_id
+      type: integer
+    root_account_ids:
+      database_column_name: root_account_ids
+      type: integer
+    description:
+      database_column_name: description
+      type: string

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -116,6 +116,13 @@ module CanvasSync
       def bulk_process_group_membership(report_file_path)
         do_bulk_import(report_file_path, GroupMembership, options: @options)
       end
+
+      def bulk_process_learning_outcomes(report_file_path)
+        do_bulk_import(report_file_path, LearningOutcome, options: @options) do |row|
+          row[:root_account_ids] = JSON.parse row[:root_account_ids]
+          row
+        end
+      end
     end
   end
 end

data/lib/canvas_sync/processors/report_processor.rb

@@ -12,14 +12,15 @@ module CanvasSync
         model.try(:get_sync_mapping, key) || mapping[key || CanvasSync::Concerns::SyncMapping::Mapping.normalize_model_name(model)]
       end
 
-      def do_bulk_import(report_file_path, model, options: {}, mapping_key: nil)
+      def do_bulk_import(report_file_path, model, options: {}, mapping_key: nil, &blk)
         m = mapping_for(model, mapping_key)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
           m[:report_columns],
           model,
           m[:conflict_target],
-          import_args: options
+          import_args: options,
+          &blk
         )
       end
     end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.31".freeze
+  VERSION = "0.17.34".freeze
 end

data/lib/canvas_sync.rb

@@ -45,6 +45,7 @@ module CanvasSync
     grading_periods
     grading_period_groups
     content_migrations
+    learning_outcomes
   ].freeze
 
   SUPPORTED_TERM_SCOPE_MODELS = %w[

data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb

@@ -104,6 +104,12 @@ RSpec.describe CanvasSync::Processors::ProvisioningReportProcessor do
       expect(obj.workflow_state).to eq 'active'
     end
 
+    it 'processes learning_outcomes' do
+      expect {
+        subject.process('spec/support/fixtures/reports/learning_outcomes.csv', { models: ['learning_outcomes'] }, 1)
+      }.to change { LearningOutcome.count }.by(2)
+    end
+
     it 'model with composite key behaves as expected' do
       expect {
         subject.process('spec/support/fixtures/reports/user_observers.csv', { models: ['user_observers'] }, 1)

data/spec/dummy/app/models/learning_outcome.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# # #
+# AUTO GENERATED MIGRATION
+# This migration was auto generated by the CanvasSync Gem.
+# You can add new columns to this table, but removing or
+# re-naming ones created here may break Canvas Syncing.
+#
+
+
+class LearningOutcome < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  belongs_to :context, polymorphic: true, optional: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
+
+  api_syncable({
+    canvas_id: :id,
+    canvas_context_id: :context_id,
+    canvas_context_type: :context_type,
+    name: :title,
+    friendly_name: :display_name,
+    vendor_guid: :vendor_guid,
+    calculation_method: :calculation_method,
+    calculation_int: :calculation_int,
+    description: :description
+  }, ->(api) { api.get("/api/v1/outcomes/#{canvas_id}") })
+end

data/spec/dummy/db/migrate/20220712210559_create_learning_outcomes.rb

@@ -0,0 +1,36 @@
+# #
+# AUTO GENERATED MIGRATION
+# This migration was auto generated by the CanvasSync Gem.
+# You can add new columns to this table, but removing or
+# re-naming ones created here may break Canvas Syncing.
+#
+
+
+class CreateLearningOutcomes < ActiveRecord::Migration[5.1]
+  def change
+    create_table :learning_outcomes do |t|
+      t.bigint :canvas_id, null: false
+      t.integer :canvas_context_id
+      t.string :canvas_context_type
+      t.string :name
+      t.string :friendly_name
+      t.string :workflow_state
+      t.datetime :canvas_created_at
+      t.datetime :canvas_updated_at
+      t.string :migration_id
+      t.string :vendor_guid
+      t.string :low_grade
+      t.string :high_grade
+      t.string :calculation_method
+      t.string :calculation_int
+      t.integer :outcome_import_id
+      t.integer :root_account_ids, array: true, default: []
+      t.text :description
+
+      t.timestamps
+    end
+
+    add_index :learning_outcomes, :canvas_id, unique: true
+    add_index :learning_outcomes, [:canvas_context_id, :canvas_context_type], name: "index_learning_outcomes_on_context"
+  end
+end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2022_03_08_072643) do
+ActiveRecord::Schema.define(version: 2022_07_12_210559) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -246,6 +246,30 @@ ActiveRecord::Schema.define(version: 2022_03_08_072643) do
     t.index ["canvas_id"], name: "index_groups_on_canvas_id", unique: true
   end
 
+  create_table "learning_outcomes", force: :cascade do |t|
+    t.bigint "canvas_id", null: false
+    t.integer "canvas_context_id"
+    t.string "canvas_context_type"
+    t.string "name"
+    t.string "friendly_name"
+    t.string "workflow_state"
+    t.datetime "canvas_created_at"
+    t.datetime "canvas_updated_at"
+    t.string "migration_id"
+    t.string "vendor_guid"
+    t.string "low_grade"
+    t.string "high_grade"
+    t.string "calculation_method"
+    t.string "calculation_int"
+    t.integer "outcome_import_id"
+    t.integer "root_account_ids", default: [], array: true
+    t.text "description"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["canvas_context_id", "canvas_context_type"], name: "index_learning_outcomes_on_context"
+    t.index ["canvas_id"], name: "index_learning_outcomes_on_canvas_id", unique: true
+  end
+
   create_table "pseudonyms", force: :cascade do |t|
     t.bigint "canvas_id", null: false
     t.bigint "canvas_user_id"