canvas_sync 0.17.27 → 0.17.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b85b5374a289fbab25718bb03812c9623065dc5c8b4e35fff67c79d220d7fda9
-  data.tar.gz: 9a8d01f7a45bfe58339c8ba8a808a29d655711945ec7cf27a8e8fcd02205f343
+  metadata.gz: ef8cd570185754998d8c14cd67f87c6d60e3507f7996925bdc4a425cd7d43f1d
+  data.tar.gz: 931416870261971aaeccfc59bae91972c8d7a7277aab9da4ac18610c3352a7fe
 SHA512:
-  metadata.gz: 73bf2bd012676ff339ccb3f2c7a2802e0752bc5623ccd301fc808d15ce139673be955ae4aee2b9ee842b18c2235cdbc843f197090c00b4caf6f57c6975d5c81a
-  data.tar.gz: eed6e24b8f30396a2f09751d2b786527a7b039c8b6fa6fbdcc9bdf695449f6e90ca76e138aec9fefb3a905abdbefd45ca092a1e1b85b0f706f6ca0fc35ff9fde
+  metadata.gz: 7262c43e7395e4c9de96a9d5c4688ff85299fcab3f97abcea719b6f73c36fb519c23ccf7a224605a8552d0e547fe135ecc055ac7f664d240f0cc1dd8a932e2bf
+  data.tar.gz: 524b36471a7b135856c2e3bd935e8ab7e057616c1bd3ed4a6d5e7a6d66c98f3236cf87d041f946a5e4ac149bef9830100619ba9e61568971f0e17898ae2f44f3

data/README.md

@@ -34,6 +34,7 @@ The following custom reports are required for the specified models:
 - assignment_groups = "Assignment Group Export" (proserv_assignment_group_export_csv)
 - context_modules = "Professional Services Context Modules Report" (proserv_context_modules_csv)
 - context_module_items = "Professional Services Context Module Items Report" (proserv_context_module_items_csv)
+- content_migrations = "Professional Services Content Migrations Report" (proserv_content_migrations_csv)
 
 ## Prerequisites
 

data/lib/canvas_sync/generators/templates/migrations/create_content_migrations.rb

@@ -0,0 +1,24 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateContentMigrations < ActiveRecord::Migration[5.1]
+  def change
+    create_table :content_migrations do |t|
+      t.bigint :canvas_id
+      t.bigint :canvas_context_id
+      t.string :canvas_context_type
+      t.string :workflow_state
+      t.text :migration_settings
+      t.datetime :started_at
+      t.datetime :finished_at
+      t.float :progress
+      t.bigint :canvas_source_course_id
+      t.string :migration_type
+      t.bigint :canvas_child_subscription_id
+      t.bigint :canvas_root_account_id
+
+      t.timestamps
+    end
+
+    add_index :content_migrations, :canvas_id, unique: true
+  end
+end

data/lib/canvas_sync/generators/templates/models/content_migration.rb

@@ -0,0 +1,10 @@
+# <%= autogenerated_model_warning %>
+
+class ContentMigration < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :context, polymorphic: true, optional: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
+
+end

data/lib/canvas_sync/job_batches/batch.rb

@@ -242,7 +242,7 @@ module CanvasSync
           _, pending, failed, children, complete, parent_bid = redis do |r|
             return unless r.exists?("BID-#{bid}")
 
-            r.multi do
+            r.multi do |r|
               r.sadd("BID-#{bid}-failed", jid)
 
               r.hincrby("BID-#{bid}", "pending", 0)
@@ -260,29 +260,19 @@ module CanvasSync
           end
         end
 
+        # Dead jobs are a Sidekiq feature.
+        # If this is called for a job, process_failed_job was also called
         def process_dead_job(bid, jid)
-          _, failed, children, complete, parent_bid = redis do |r|
+          _, dead_count = redis do |r|
             return unless r.exists?("BID-#{bid}")
 
-            r.multi do
+            r.multi do |r|
               r.sadd("BID-#{bid}-dead", jid)
-
               r.scard("BID-#{bid}-dead")
-              r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-batches-complete")
-              r.hget("BID-#{bid}", "parent_bid")
-
               r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
             end
           end
 
-          if parent_bid
-            redis.multi do
-              r.sadd("BID-#{parent_bid}-dead", jid)
-              r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
-            end
-          end
-
           enqueue_callbacks(:death, bid)
         end
 
@@ -290,7 +280,7 @@ module CanvasSync
           _, failed, pending, children, complete, success, parent_bid, keep_open = redis do |r|
             return unless r.exists?("BID-#{bid}")
 
-            r.multi do
+            r.multi do |r|
               r.srem("BID-#{bid}-failed", jid)
 
               r.scard("BID-#{bid}-failed")
@@ -323,7 +313,7 @@ module CanvasSync
             return unless r.exists?(batch_key)
             return if r.hget(batch_key, 'keep_open') == 'true'
 
-            r.multi do
+            r.multi do |r|
               r.smembers(callback_key)
               r.hget(batch_key, "callback_queue")
               r.hget(batch_key, "parent_bid")
@@ -442,12 +432,8 @@ module CanvasSync
       class RedisProxy
         def multi(*args, &block)
           Batch.redis do |r|
-            if block.arity == 1
-              r.multi(*args) do
-                block.call(r)
-              end
-            else
-              r.multi(*args, &block)
+            r.multi(*args) do |r|
+              block.call(r)
             end
           end
         end

data/lib/canvas_sync/job_batches/callback.rb

@@ -69,7 +69,7 @@ module CanvasSync
                 # This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
                 origin_bid = opts['origin']['for_bid']
                 _, pending, success_ran = Batch.redis do |r|
-                  r.multi do
+                  r.multi do |r|
                     r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
                     r.scard("BID-#{origin_bid}-pending_callbacks")
                     r.hget("BID-#{origin_bid}", "success")
@@ -88,7 +88,7 @@ module CanvasSync
             return unless parent_bid
 
             _, _, success, _, _, complete, pending, children, success, failure = Batch.redis do |r|
-              r.multi do
+              r.multi do |r|
                 r.sadd("BID-#{parent_bid}-batches-success", bid)
                 r.expire("BID-#{parent_bid}-batches-success", Batch::BID_EXPIRE_TTL)
                 r.scard("BID-#{parent_bid}-batches-success")
@@ -116,7 +116,7 @@ module CanvasSync
 
           def complete(bid, status, parent_bid)
             pending, children, success = Batch.redis do |r|
-              r.multi do
+              r.multi do |r|
                 r.hincrby("BID-#{bid}", "pending", 0)
                 r.hincrby("BID-#{bid}", "children", 0)
                 r.scard("BID-#{bid}-batches-success")
@@ -131,7 +131,7 @@ module CanvasSync
 
               Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
               _, _, complete, pending, children, failure = Batch.redis do |r|
-                r.multi do
+                r.multi do |r|
                   r.sadd("BID-#{parent_bid}-batches-complete", bid)
                   r.sadd("BID-#{parent_bid}-batches-failed", bid)
                   r.scard("BID-#{parent_bid}-batches-complete")

data/lib/canvas_sync/job_batches/context_hash.rb

@@ -127,7 +127,7 @@ module CanvasSync
         return @hash_map[bid] if @hash_map.key?(bid)
 
         context_json, editable = Batch.redis do |r|
-          r.multi do
+          r.multi do |r|
             r.hget("BID-#{bid}", "context")
             r.hget("BID-#{bid}", "allow_context_changes")
           end

data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb

@@ -18,7 +18,7 @@ module CanvasSync
           man_batch_id = SecureRandom.urlsafe_base64(10)
 
           Batch.redis do |r|
-            r.multi do
+            r.multi do |r|
               r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
               r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
               r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
@@ -87,7 +87,7 @@ module CanvasSync
 
       def self.perform_next_sequence_job(man_batch_id)
         root_bid, ordered = Batch.redis do |r|
-          r.multi do
+          r.multi do |r|
             r.hget("MNGBID-#{man_batch_id}", "root_bid")
             r.hget("MNGBID-#{man_batch_id}", "ordered")
           end

data/lib/canvas_sync/job_batches/pool.rb

@@ -94,16 +94,16 @@ module CanvasSync
         redis.hincrby(redis_key, "active_count", 0)
       end
 
-      def pending_count
+      def pending_count(r = redis)
         jobs_key = "#{redis_key}-jobs"
         order = self.order || 'fifo'
         case order.to_sym
         when :fifo, :lifo
-          redis.llen(jobs_key)
+          r.llen(jobs_key)
         when :random
-          redis.scard(jobs_key)
+          r.scard(jobs_key)
         when :priority
-          redis.zcard(jobs_key)
+          r.zcard(jobs_key)
         end
       end
 
@@ -114,9 +114,9 @@ module CanvasSync
           # Make sure this is loaded outside of the pipeline
           self.order
 
-          redis.multi do
+          redis.multi do |r|
             r.hincrby(redis_key, "active_count", -1)
-            self.pending_count
+            self.pending_count(r)
           end
         end
 

data/lib/canvas_sync/job_batches/redis_model.rb

@@ -40,7 +40,7 @@ module CanvasSync
       def persist_bid_attr(attribute, value)
         if @initialized || @existing
           redis do |r|
-            r.multi do
+            r.multi do |r|
               r.hset(redis_key, attribute, value)
               r.expire(redis_key, Batch::BID_EXPIRE_TTL)
             end

data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb

@@ -13,7 +13,7 @@ module CanvasSync::JobBatches::Sidekiq
 
       def drain_zset(key)
         items, _ = Batch.redis do |r|
-          r.multi do
+          r.multi do |r|
             r.zrange(key, 0, -1)
             r.zremrangebyrank(key, 0, -1)
           end

data/lib/canvas_sync/jobs/sync_content_migrations_job.rb

@@ -0,0 +1,20 @@
+module CanvasSync
+  module Jobs
+    class SyncContentMigrationsJob < ReportStarter
+      # Syncs ContentMigrations
+      #
+      # Starts a report processor for the content migrations report
+      # (the proserv_content_migrations_csv report must be enabled)
+      #
+      # @param options [Hash]
+      def perform(options)
+        super(
+          "proserv_content_migrations_csv",
+          merge_report_params(options),
+          CanvasSync::Processors::ContentMigrationsProcessor.to_s,
+          {},
+        )
+      end
+    end
+  end
+end

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -20,9 +20,9 @@ module CanvasSync
               end
 
               JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
-                b.description = "TermBatchJob(#{term.canvas_id}) Root"
+                b.description = "TermBatchJob(#{term_id}) Root"
                 b.context = local_context
-                b.on(:success, "#{self.class.to_s}.batch_finished")
+                b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false
               end
             end
           else

data/lib/canvas_sync/processors/content_migrations_processor.rb

@@ -0,0 +1,19 @@
+require_relative "./report_processor"
+
+module CanvasSync
+  module Processors
+    # Processes a content migrations report using the bulk importer.
+    #
+    # @param report_file_path [String]
+    # @param options [Hash]
+    class ContentMigrationsProcessor < ReportProcessor
+      def self.process(report_file_path, _options, report_id)
+        new(report_file_path, _options)
+      end
+
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, ContentMigration, options: options)
+      end
+    end
+  end
+end

data/lib/canvas_sync/processors/model_mappings.yml

@@ -469,3 +469,43 @@ grading_period_groups:
     status:
       database_column_name: workflow_state
       type: string
+
+content_migrations:
+  conflict_target: canvas_migration_id
+  report_columns:
+    canvas_migration_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_context_id:
+      database_column_name: canvas_context_id
+      type: integer
+    canvas_context_type:
+      database_column_name: canvas_context_type
+      type: string
+    workflow_state:
+      database_column_name: workflow_state
+      type: string
+    migration_settings:
+      database_column_name: migration_settings
+      type: text
+    started_at:
+      database_column_name: started_at
+      type: datetime
+    finished_at:
+      database_column_name: finished_at
+      type: datetime
+    progress:
+      database_column_name: progress
+      type: float
+    canvas_source_course_id:
+      database_column_name: canvas_source_course_id
+      type: integer
+    migration_type:
+      database_column_name: migration_type
+      type: string
+    canvas_child_subscription_id:
+      database_column_name: canvas_child_subscription_id
+      type: integer
+    canvas_root_account_id:
+      database_column_name: canvas_root_account_id
+      type: integer

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.27".freeze
+  VERSION = "0.17.30".freeze
 end

data/lib/canvas_sync.rb

@@ -44,6 +44,7 @@ module CanvasSync
     user_observers
     grading_periods
     grading_period_groups
+    content_migrations
   ].freeze
 
   SUPPORTED_TERM_SCOPE_MODELS = %w[
@@ -150,6 +151,7 @@ module CanvasSync
         assignment_groups: CanvasSync::Jobs::SyncAssignmentGroupsJob,
         context_modules: CanvasSync::Jobs::SyncContextModulesJob,
         context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
+        content_migrations: CanvasSync::Jobs::SyncContentMigrationsJob,
       }.with_indifferent_access
 
       root_chain = base_canvas_sync_chain(**kwargs, globals: options[:global] || kwargs[:globals])

data/spec/canvas_sync/jobs/sync_content_migrations_job_spec.rb

@@ -0,0 +1,30 @@
+require "spec_helper"
+
+RSpec.describe CanvasSync::Jobs::SyncContentMigrationsJob do
+  describe "#perform" do
+    context "no parameters is specified" do
+      it "enqueues a ReportStarter for the proserv_content_migrations_csv" do
+        expect_any_instance_of(Bearcat::Client).to receive(:start_report)
+          .with("self", "proserv_content_migrations_csv", { parameters: { } })
+          .and_return("id" => 1)
+
+        expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
+
+        CanvasSync::Jobs::SyncContentMigrationsJob.perform_now({})
+      end
+    end
+
+    context "updated_after parameters is specified" do
+      it "enqueues a ReportStarter for the proserv_content_migrations_csv and get data from given date" do
+        expect_any_instance_of(Bearcat::Client).to receive(:start_report)
+          .with("self", "proserv_content_migrations_csv", { parameters: { updated_after: 6.hours.ago.to_s } })
+          .and_return("id" => 1)
+
+        expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
+
+        set_batch_context(updated_after: 6.hours.ago.to_s)
+        CanvasSync::Jobs::SyncContentMigrationsJob.perform_now({})
+      end
+    end
+  end
+end

data/spec/canvas_sync/processors/content_migrations_processor_spec.rb

@@ -0,0 +1,13 @@
+require "spec_helper"
+
+RSpec.describe CanvasSync::Processors::ContentMigrationsProcessor do
+  let(:subject) { CanvasSync::Processors::ContentMigrationsProcessor }
+
+  describe "#process" do
+    it "inserts content migrations" do
+      expect {
+        subject.process("spec/support/fixtures/reports/content_migrations.csv", {}, 1)
+      }.to change { ContentMigration.count }.by(2)
+    end
+  end
+end

data/spec/dummy/app/models/content_migration.rb

@@ -0,0 +1,16 @@
+# #
+# AUTO GENERATED MODEL
+# This model was auto generated by the CanvasSync Gem.
+# You can customize it as needed, but make sure you test
+# any changes you make to the auto generated methods.
+#
+
+
+class ContentMigration < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :context, polymorphic: true, optional: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type
+
+end

data/spec/dummy/db/migrate/20220308072643_create_content_migrations.rb

@@ -0,0 +1,30 @@
+# #
+# AUTO GENERATED MIGRATION
+# This migration was auto generated by the CanvasSync Gem.
+# You can add new columns to this table, but removing or
+# re-naming ones created here may break Canvas Syncing.
+#
+
+
+class CreateContentMigrations < ActiveRecord::Migration[5.1]
+  def change
+    create_table :content_migrations do |t|
+      t.bigint :canvas_id
+      t.bigint :canvas_context_id
+      t.string :canvas_context_type
+      t.string :workflow_state
+      t.text :migration_settings
+      t.datetime :started_at
+      t.datetime :finished_at
+      t.float :progress
+      t.bigint :canvas_source_course_id
+      t.string :migration_type
+      t.bigint :canvas_child_subscription_id
+      t.bigint :canvas_root_account_id
+
+      t.timestamps
+    end
+
+    add_index :content_migrations, :canvas_id, unique: true
+  end
+end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_10_01_184920) do
+ActiveRecord::Schema.define(version: 2022_03_08_072643) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -110,6 +110,24 @@ ActiveRecord::Schema.define(version: 2021_10_01_184920) do
     t.string "batch_bid"
   end
 
+  create_table "content_migrations", force: :cascade do |t|
+    t.bigint "canvas_id"
+    t.bigint "canvas_context_id"
+    t.string "canvas_context_type"
+    t.string "workflow_state"
+    t.text "migration_settings"
+    t.datetime "started_at"
+    t.datetime "finished_at"
+    t.float "progress"
+    t.bigint "canvas_source_course_id"
+    t.string "migration_type"
+    t.bigint "canvas_child_subscription_id"
+    t.bigint "canvas_root_account_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["canvas_id"], name: "index_content_migrations_on_canvas_id", unique: true
+  end
+
   create_table "context_module_items", force: :cascade do |t|
     t.bigint "canvas_id"
     t.bigint "canvas_context_module_id"