canvas_sync 0.17.26.beta1 → 0.17.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce544260a5cc94f9193cb99da3bc55e06ecd1078e0ef6a5afca282cc4d39169f
-  data.tar.gz: 30a5bccedebde1cb00aeb31219b1d9f605672fa8ea58cb50af483cf8f98d15fb
+  metadata.gz: 8359f6a5054489357a480d15019981f0ade50a145f714bcbd2274c6570689316
+  data.tar.gz: c98c3d8ecda4291e162e31d7c4dfdf9ddff65763f0bb7c345006585f53894cf4
 SHA512:
-  metadata.gz: e84bde52492c2276650bd662adff677c0aa6a1c511587886d45fc6d328389af242ade72fe872dda2093df9ebb151aa25a4a353559a31d74330849f0a0c93d159
-  data.tar.gz: 3c30eceedad806932b61abc66456973250c7247ba7431406f1c5440b09b05cc3b7174112e34e78392d8f3e9c89dfcd7eba43377a0f8ee30d6320a00a50303163
+  metadata.gz: 4ce90338e8e3751fd5767a09eaaf89e733532ffa59df5569bf908930929f34d5167bc01e2637a5eef6ba5ef515c2e6410cc17ce757542b4d0620bb08bd03fd38
+  data.tar.gz: e4456ad37bbf87b63a495d041f038ad6ece797b59c11d19cd7028a030aef452281105f2c53ab56bef3b4cf515f838366874b56838c87936d113c3cacf5570771

data/README.md

@@ -34,6 +34,7 @@ The following custom reports are required for the specified models:
 - assignment_groups = "Assignment Group Export" (proserv_assignment_group_export_csv)
 - context_modules = "Professional Services Context Modules Report" (proserv_context_modules_csv)
 - context_module_items = "Professional Services Context Module Items Report" (proserv_context_module_items_csv)
+- content_migrations = "Professional Services Content Migrations Report" (proserv_content_migrations_csv)
 
 ## Prerequisites
 

data/lib/canvas_sync/generators/templates/migrations/create_content_migrations.rb

@@ -0,0 +1,24 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateContentMigrations < ActiveRecord::Migration[5.1]
+  def change
+    create_table :content_migrations do |t|
+      t.bigint :canvas_id
+      t.bigint :canvas_context_id
+      t.string :canvas_context_type
+      t.string :workflow_state
+      t.text :migration_settings
+      t.datetime :started_at
+      t.datetime :finished_at
+      t.float :progress
+      t.bigint :canvas_source_course_id
+      t.string :migration_type
+      t.bigint :canvas_child_subscription_id
+      t.bigint :canvas_root_account_id
+
+      t.timestamps
+    end
+
+    add_index :content_migrations, :canvas_id, unique: true
+  end
+end

data/lib/canvas_sync/generators/templates/models/content_migration.rb

@@ -0,0 +1,10 @@
+# <%= autogenerated_model_warning %>
+
+class ContentMigration < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :context, polymorphic: true, optional: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
+
+end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -65,7 +65,7 @@ module CanvasSync
         }
 
         row_buffer = nil
-        if defined?(User) && klass == User && csv_column_names.include?(:user_id)
+        if defined?(User) && klass == User && csv_column_names.include?('user_id')
           row_buffer = UserRowBuffer.new(&row_buffer_out)
         else
           row_buffer = NullRowBuffer.new(&row_buffer_out)

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -95,6 +95,10 @@ module CanvasSync
         @flattened = flattened.with_indifferent_access
       end
 
+      def to_h
+        flatten
+      end
+
       private
 
       def get_parent_hash(bid)

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -3,16 +3,19 @@ require_relative './base_job'
 module CanvasSync
   module JobBatches
     class ConcurrentBatchJob < BaseJob
+      def self.make_batch(sub_jobs, context: nil, &blk)
+        ManagedBatchJob.make_batch(
+          sub_jobs,
+          ordered: false,
+          concurrency: true,
+          context: context,
+          desc_prefix: 'ConcurrentBatchJob',
+          &blk
+        )
+      end
+
       def perform(sub_jobs, context: nil)
-        Batch.new.tap do |b|
-          b.description = "Concurrent Batch Root"
-          b.context = context
-          b.jobs do
-            sub_jobs.each do |j|
-              ChainBuilder.enqueue_job(j)
-            end
-          end
-        end
+        self.class.make_batch(sub_jobs, context: context)
       end
     end
   end

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -3,8 +3,8 @@ require_relative './base_job'
 module CanvasSync
   module JobBatches
     class ManagedBatchJob < BaseJob
-      def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
-        man_batch_id = SecureRandom.urlsafe_base64(10)
+      def self.make_batch(sub_jobs, ordered: true, concurrency: nil, context: nil, desc_prefix: nil, &blk)
+        desc_prefix ||= ''
 
         if concurrency == 0 || concurrency == nil || concurrency == true
           concurrency = sub_jobs.count
@@ -14,38 +14,60 @@ module CanvasSync
 
         root_batch = Batch.new
 
-        Batch.redis do |r|
-          r.multi do
-            r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
-            r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
-            r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
-            r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
-
-            mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
-              j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
-              j = ActiveJob::Arguments.serialize([j])
-              JSON.unparse(j)
-            end
-            if ordered
-              r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
-            else
-              r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+        if concurrency < sub_jobs.count
+          man_batch_id = SecureRandom.urlsafe_base64(10)
+
+          Batch.redis do |r|
+            r.multi do
+              r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
+              r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
+              r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
+              r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
+
+              mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
+                j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
+                j = ActiveJob::Arguments.serialize([j])
+                JSON.unparse(j)
+              end
+              if ordered
+                r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+              else
+                r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
+              end
+              r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
             end
-            r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
           end
+
+          root_batch.allow_context_changes = (concurrency == 1)
+          root_batch.on(:success, "#{to_s}.cleanup_redis", managed_batch_id: man_batch_id)
+
+          desc_prefix = "MGD(#{man_batch_id}): #{desc_prefix}"
         end
 
-        root_batch.description = "Managed Batch Root (#{man_batch_id})"
-        root_batch.allow_context_changes = (concurrency == 1)
         root_batch.context = context
-        root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
-        root_batch.jobs {}
 
-        concurrency.times do
-          self.class.perform_next_sequence_job(man_batch_id)
+        blk.call(ManagedBatchProxy.new(root_batch)) if blk.present?
+
+        root_batch.description = "#{desc_prefix}: #{root_batch.description || 'Root'}"
+
+        if concurrency < sub_jobs.count
+          root_batch.jobs {}
+          concurrency.times do
+            perform_next_sequence_job(man_batch_id)
+          end
+        else
+          root_batch.jobs do
+            sub_jobs.each do |j|
+              ChainBuilder.enqueue_job(j)
+            end
+          end
         end
       end
 
+      def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
+        self.class.make_batch(sub_jobs, ordered: ordered, concurrency: concurrency, context: context)
+      end
+
       def self.cleanup_redis(status, options)
         man_batch_id = options['managed_batch_id']
         Batch.redis do |r|
@@ -94,6 +116,21 @@ module CanvasSync
           end
         end
       end
+
+      class ManagedBatchProxy
+        def initialize(real_batch)
+          @real_batch = real_batch
+        end
+
+        delegate_missing_to :real_batch
+
+        def jobs
+          raise "Managed Batches do not support calling .jobs directly!"
+        end
+
+        private
+        attr_reader :real_batch
+      end
     end
   end
 end

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -3,14 +3,20 @@ require_relative './base_job'
 module CanvasSync
   module JobBatches
     class SerialBatchJob < BaseJob
-      def perform(sub_jobs, context: nil)
-        ManagedBatchJob.new.perform(
+      def self.make_batch(sub_jobs, context: nil, &blk)
+        ManagedBatchJob.make_batch(
           sub_jobs,
-          context: context,
           ordered: true,
           concurrency: false,
+          context: context,
+          desc_prefix: 'SerialBatchJob',
+          &blk
         )
       end
+
+      def perform(sub_jobs, context: nil)
+        self.class.make_batch(sub_jobs, context: context)
+      end
     end
   end
 end

data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb

@@ -12,7 +12,7 @@ module CanvasSync::JobBatches::Sidekiq
       end
 
       def drain_zset(key)
-        items, _ = Sidekiq.redis do |r|
+        items, _ = Batch.redis do |r|
           r.multi do
             r.zrange(key, 0, -1)
             r.zremrangebyrank(key, 0, -1)

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -31,6 +31,8 @@ module CanvasSync
           status: 'processing',
         )
 
+        globals[:batch_genre] = genre
+        globals[:batch_start_time] = sync_batch.started_at.iso8601
         globals[:sync_batch_id] = sync_batch.id
 
         JobBatches::Batch.new.tap do |b|

data/lib/canvas_sync/jobs/sync_content_migrations_job.rb

@@ -0,0 +1,20 @@
+module CanvasSync
+  module Jobs
+    class SyncContentMigrationsJob < ReportStarter
+      # Syncs ContentMigrations
+      #
+      # Starts a report processor for the content migrations report
+      # (the proserv_content_migrations_csv report must be enabled)
+      #
+      # @param options [Hash]
+      def perform(options)
+        super(
+          "proserv_content_migrations_csv",
+          merge_report_params(options),
+          CanvasSync::Processors::ContentMigrationsProcessor.to_s,
+          {},
+        )
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -6,15 +6,42 @@ module CanvasSync
           context = options[:context] || {}
           if options[:term_scope]
             Term.send(options[:term_scope]).find_each.map do |term|
-              local_context = context.merge(canvas_term_id: get_term_id(term))
-              JobBatches::ConcurrentBatchJob.perform_now(jobs, context: local_context)
+              term_id = get_term_id(term)
+              local_context = context.merge(canvas_term_id: term_id)
+
+              # Override the delta-syncing date if:
+              # 1. the Term hasn't been synced before or
+              # 2. the Term underwent a period of not syncing
+              term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
+              if batch_context[:updated_after]
+                if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
+                  local_context[:updated_after] = term_last_sync.presence
+                end
+              end
+
+              JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
+                b.description = "TermBatchJob(#{term.canvas_id}) Root"
+                b.context = local_context
+                b.on(:success, "#{self.class.to_s}.batch_finished")
+              end
             end
           else
-            JobBatches::ConcurrentBatchJob.perform_now(jobs, context: context)
+            JobBatches::ConcurrentBatchJob.make_batch(jobs, context: context)
           end
         end
       end
 
+      def self.batch_finished(status, opts)
+        ctx = JobBatches::Batch.current_context
+        term_id = ctx[:canvas_term_id]
+        CanvasSync.redis.set(last_sync_key(term_id), ctx[:batch_start_time])
+      end
+
+      def self.last_sync_key(term_id)
+        ctx = JobBatches::Batch.current_context
+        "#{CanvasSync.redis_prefix}:#{ctx[:batch_genre]}:#{term_id}:last_sync"
+      end
+
       def get_term_id(term)
         term.try(:canvas_id) || term.canvas_term_id
       end

data/lib/canvas_sync/processors/content_migrations_processor.rb

@@ -0,0 +1,19 @@
+require_relative "./report_processor"
+
+module CanvasSync
+  module Processors
+    # Processes a content migrations report using the bulk importer.
+    #
+    # @param report_file_path [String]
+    # @param options [Hash]
+    class ContentMigrationsProcessor < ReportProcessor
+      def self.process(report_file_path, _options, report_id)
+        new(report_file_path, _options)
+      end
+
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, ContentMigration, options: options)
+      end
+    end
+  end
+end

data/lib/canvas_sync/processors/model_mappings.yml

@@ -469,3 +469,43 @@ grading_period_groups:
     status:
       database_column_name: workflow_state
       type: string
+
+content_migrations:
+  conflict_target: canvas_migration_id
+  report_columns:
+    canvas_migration_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_context_id:
+      database_column_name: canvas_context_id
+      type: integer
+    canvas_context_type:
+      database_column_name: canvas_context_type
+      type: string
+    workflow_state:
+      database_column_name: workflow_state
+      type: string
+    migration_settings:
+      database_column_name: migration_settings
+      type: text
+    started_at:
+      database_column_name: started_at
+      type: datetime
+    finished_at:
+      database_column_name: finished_at
+      type: datetime
+    progress:
+      database_column_name: progress
+      type: float
+    canvas_source_course_id:
+      database_column_name: canvas_source_course_id
+      type: integer
+    migration_type:
+      database_column_name: migration_type
+      type: string
+    canvas_child_subscription_id:
+      database_column_name: canvas_child_subscription_id
+      type: integer
+    canvas_root_account_id:
+      database_column_name: canvas_root_account_id
+      type: integer

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.26.beta1".freeze
+  VERSION = "0.17.28".freeze
 end

data/lib/canvas_sync.rb

@@ -44,6 +44,7 @@ module CanvasSync
     user_observers
     grading_periods
     grading_period_groups
+    content_migrations
   ].freeze
 
   SUPPORTED_TERM_SCOPE_MODELS = %w[
@@ -150,6 +151,7 @@ module CanvasSync
         assignment_groups: CanvasSync::Jobs::SyncAssignmentGroupsJob,
         context_modules: CanvasSync::Jobs::SyncContextModulesJob,
         context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
+        content_migrations: CanvasSync::Jobs::SyncContentMigrationsJob,
       }.with_indifferent_access
 
       root_chain = base_canvas_sync_chain(**kwargs, globals: options[:global] || kwargs[:globals])
@@ -320,5 +322,15 @@ module CanvasSync
       return if invalid.empty?
       raise "Invalid live event(s) specified: #{invalid.join(', ')}. Only #{SUPPORTED_LIVE_EVENTS.join(', ')} are supported."
     end
+
+    def redis(*args, &blk)
+      JobBatches::Batch.redis(*args, &blk)
+    end
+
+    def redis_prefix
+      pfx = "cs"
+      pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
+      pfx
+    end
   end
 end

data/spec/canvas_sync/jobs/sync_content_migrations_job_spec.rb

@@ -0,0 +1,30 @@
+require "spec_helper"
+
+RSpec.describe CanvasSync::Jobs::SyncContentMigrationsJob do
+  describe "#perform" do
+    context "no parameters is specified" do
+      it "enqueues a ReportStarter for the proserv_content_migrations_csv" do
+        expect_any_instance_of(Bearcat::Client).to receive(:start_report)
+          .with("self", "proserv_content_migrations_csv", { parameters: { } })
+          .and_return("id" => 1)
+
+        expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
+
+        CanvasSync::Jobs::SyncContentMigrationsJob.perform_now({})
+      end
+    end
+
+    context "updated_after parameters is specified" do
+      it "enqueues a ReportStarter for the proserv_content_migrations_csv and get data from given date" do
+        expect_any_instance_of(Bearcat::Client).to receive(:start_report)
+          .with("self", "proserv_content_migrations_csv", { parameters: { updated_after: 6.hours.ago.to_s } })
+          .and_return("id" => 1)
+
+        expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
+
+        set_batch_context(updated_after: 6.hours.ago.to_s)
+        CanvasSync::Jobs::SyncContentMigrationsJob.perform_now({})
+      end
+    end
+  end
+end

data/spec/canvas_sync/processors/content_migrations_processor_spec.rb

@@ -0,0 +1,13 @@
+require "spec_helper"
+
+RSpec.describe CanvasSync::Processors::ContentMigrationsProcessor do
+  let(:subject) { CanvasSync::Processors::ContentMigrationsProcessor }
+
+  describe "#process" do
+    it "inserts content migrations" do
+      expect {
+        subject.process("spec/support/fixtures/reports/content_migrations.csv", {}, 1)
+      }.to change { ContentMigration.count }.by(2)
+    end
+  end
+end

data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb

@@ -10,6 +10,11 @@ RSpec.describe CanvasSync::Processors::ProvisioningReportProcessor do
       }.to change { User.count }.by(2)
     end
 
+    it 'uses a User row with a sis_id' do
+      subject.process('spec/support/fixtures/reports/users.csv', { models: ['users'] }, 1)
+      expect(User.find_by(canvas_id: 2).sis_id).to eq 'sis_id_2'
+    end
+
     it 'processes courses' do
       expect {
         subject.process('spec/support/fixtures/reports/courses.csv', { models: ['courses'] }, 1)

data/spec/dummy/app/models/content_migration.rb

@@ -0,0 +1,16 @@
+# #
+# AUTO GENERATED MODEL
+# This model was auto generated by the CanvasSync Gem.
+# You can customize it as needed, but make sure you test
+# any changes you make to the auto generated methods.
+#
+
+
+class ContentMigration < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :context, polymorphic: true, optional: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
+
+end

data/spec/dummy/db/migrate/20220308072643_create_content_migrations.rb

@@ -0,0 +1,30 @@
+# #
+# AUTO GENERATED MIGRATION
+# This migration was auto generated by the CanvasSync Gem.
+# You can add new columns to this table, but removing or
+# re-naming ones created here may break Canvas Syncing.
+#
+
+
+class CreateContentMigrations < ActiveRecord::Migration[5.1]
+  def change
+    create_table :content_migrations do |t|
+      t.bigint :canvas_id
+      t.bigint :canvas_context_id
+      t.string :canvas_context_type
+      t.string :workflow_state
+      t.text :migration_settings
+      t.datetime :started_at
+      t.datetime :finished_at
+      t.float :progress
+      t.bigint :canvas_source_course_id
+      t.string :migration_type
+      t.bigint :canvas_child_subscription_id
+      t.bigint :canvas_root_account_id
+
+      t.timestamps
+    end
+
+    add_index :content_migrations, :canvas_id, unique: true
+  end
+end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_10_01_184920) do
+ActiveRecord::Schema.define(version: 2022_03_08_072643) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -110,6 +110,24 @@ ActiveRecord::Schema.define(version: 2021_10_01_184920) do
     t.string "batch_bid"
   end
 
+  create_table "content_migrations", force: :cascade do |t|
+    t.bigint "canvas_id"
+    t.bigint "canvas_context_id"
+    t.string "canvas_context_type"
+    t.string "workflow_state"
+    t.text "migration_settings"
+    t.datetime "started_at"
+    t.datetime "finished_at"
+    t.float "progress"
+    t.bigint "canvas_source_course_id"
+    t.string "migration_type"
+    t.bigint "canvas_child_subscription_id"
+    t.bigint "canvas_root_account_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["canvas_id"], name: "index_content_migrations_on_canvas_id", unique: true
+  end
+
   create_table "context_module_items", force: :cascade do |t|
     t.bigint "canvas_id"
     t.bigint "canvas_context_module_id"

data/spec/support/fixtures/reports/content_migrations.csv

@@ -0,0 +1,3 @@
+canvas_migration_id,canvas_context_id,canvas_context_type,workflow_state,migration_settings,started_at,finished_at,progress,canvas_source_course_id,migration_type,canvas_child_subscription_id,canvas_root_account_id
+101,1250,Course,imported,,2022-02-18 10:10:40,2022-02-18 10:11:40,100,1200,course_copy_importer,,1
+102,1251,Course,imported,,2022-02-18 10:10:40,2022-02-18 10:11:40,100,1120,master_course_import,3,1

data/spec/support/fixtures/reports/users.csv

@@ -1,4 +1,5 @@
 canvas_user_id,user_id,integration_id,authentication_provider_id,login_id,first_name,last_name,full_name,sortable_name,short_name,email,status,created_by_sis
 1,sis_id_1,,123,spiderdude1337,Patches,the Spider,Patches the Spider,"the Spider, Patches",,poorbastard@yharnam.com,active,FALSE
-2,sis_id_2,,1234,retireddude109,Hunter,Djura,Hunter Djura,"Djura, Hunter",,dreamer@yharnam.com,active,FALSE
-2,sis_id_2,,1234,retireddude110,Hunter,Djura,Hunter Djura,"Djura, Hunter",,dreamer@yharnam.com,active,FALSE
+2,,,1234,retireddude109,Hunter,Djura,Hunter Djura,"Djura, Hunter",,dreamer@yharnam.com,active,FALSE
+2,sis_id_2,,1234,retireddude110,Hunter,Djura,Hunter Djura,"Djura, Hunter",,dreamer@yharnam.com,active,FALSE
+2,,,1234,retireddude110,Hunter,Djura,Hunter Djura,"Djura, Hunter",,dreamer@yharnam.com,active,FALSE