canvas_sync 0.22.3 → 0.22.5

data/lib/canvas_sync/jobs/sync_course_progresses_job.rb

@@ -0,0 +1,21 @@
+module CanvasSync
+  module Jobs
+    class SyncCourseProgressesJob < ReportStarter
+      # Syncs CourseProgresses
+      #
+      # Starts a report processor for the course completion report
+      # (the proserv_course_completion_csv report must be enabled)
+      #
+      # @param options [Hash]
+      def perform(options)
+        super(
+          'proserv_course_completion_csv',
+          merge_report_params(options),
+          CanvasSync::Processors::CourseCompletionReportProcessor.to_s,
+          {},
+        )
+      end
+    end
+  end
+
+end

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -12,14 +12,17 @@ module CanvasSync
               # Override the delta-syncing date if:
               # 1. the Term hasn't been synced before or
               # 2. the Term underwent a period of not syncing
-              term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
               if batch_context[:updated_after]
+                term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
                 if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
                   local_context[:updated_after] = term_last_sync.presence
                 end
               end
 
               JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
+                # TODO If we do a Chain UI, this will need to checkin somehow to indicate that the chain forked
+                #   Or chain steps just show a summary - eg "Started", "X Jobs Running", "Done" or "X Jobs Running, Y Jobs Done" - and not individual forks
+                #   For a step to be considered done, all previous sibling-level steps must be done and no batches pending
                 b.description = "TermBatchJob(#{term_id}) Root"
                 b.context = local_context
                 b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false

data/lib/canvas_sync/processors/course_completion_report_processor.rb

@@ -0,0 +1,20 @@
+require_relative "./report_processor"
+
+module CanvasSync
+  module Processors
+    # Processes a course completion report using the bulk importer.
+    #
+    # @param report_file_path [String]
+    # @param options [Hash]
+    class CourseCompletionReportProcessor < ReportProcessor
+      def self.process(report_file_path, _options, report_id)
+        new(report_file_path, _options)
+      end
+
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, CourseProgress, options: options)
+      end
+    end
+  end
+end
+

data/lib/canvas_sync/processors/model_mappings.yml

@@ -105,6 +105,27 @@ courses:
       database_column_name: grading_standard_id
       type: integer
 
+course_progresses:
+  conflict_target:
+    - student_id
+    - course_id
+  report_columns:
+    student_id:
+      database_column_name: canvas_user_id
+      type: integer
+    course_id:
+      database_column_name: canvas_course_id
+      type: integer
+    requirement_count:
+      database_column_name: requirement_count
+      type: integer
+    requirement_completed_count:
+      database_column_name: requirement_completed_count
+      type: integer
+    completion_date:
+      database_column_name: completion_date
+      type: datetime
+
 enrollments:
   conflict_target: canvas_enrollment_id
   report_columns:
@@ -324,6 +345,12 @@ context_modules:
     deleted_at:
       database_column_name: deleted_at
       type: datetime
+    completion_requirements:
+      database_column_name: completion_requirements
+      type: yaml
+    prerequisites:
+      database_column_name: prerequisites
+      type: yaml
 
 context_module_items:
   conflict_target: canvas_context_module_item_id

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.22.3".freeze
+  VERSION = "0.22.5".freeze
 end

data/lib/canvas_sync.rb

@@ -53,6 +53,7 @@ module CanvasSync
     rubrics
     rubric_associations
     rubric_assessments
+    course_progresses
   ].freeze
 
   SUPPORTED_TERM_SCOPE_MODELS = %w[
@@ -64,6 +65,7 @@ module CanvasSync
     rubrics
     rubric_associations
     rubric_assessments
+    course_progresses
   ].freeze
 
   DEFAULT_TERM_SCOPE_MODELS = %w[
@@ -75,6 +77,7 @@ module CanvasSync
     rubrics
     rubric_associations
     rubric_assessments
+    course_progresses
   ].freeze
 
   SUPPORTED_LIVE_EVENTS = %w[
@@ -166,6 +169,7 @@ module CanvasSync
         context_modules: CanvasSync::Jobs::SyncContextModulesJob,
         context_module_items: CanvasSync::Jobs::SyncContextModuleItemsJob,
         content_migrations: CanvasSync::Jobs::SyncContentMigrationsJob,
+        course_progresses: CanvasSync::Jobs::SyncCourseProgressesJob,
         rubrics: CanvasSync::Jobs::SyncRubricsJob,
         rubric_associations: CanvasSync::Jobs::SyncRubricAssociationsJob,
         rubric_assessments: CanvasSync::Jobs::SyncRubricAssessmentsJob,

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -2,18 +2,6 @@ require 'spec_helper'
 
 RSpec.describe CanvasSync do
   describe '.provisioning_sync' do
-    it 'invokes the first job in the queue and passes on the rest of the job chain' do
-      expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], term_scope: nil).normalize![:args][0]
-
-      expect(CanvasSync::Jobs::BeginSyncChainJob).to receive(:perform_later)
-        .with(
-          expected_job_chain,
-          anything
-        )
-
-      CanvasSync.provisioning_sync(['courses'])
-    end
-
     context 'an invalid model is passed in' do
       it 'raises a helpful exception' do
         expect {
@@ -30,22 +18,25 @@ RSpec.describe CanvasSync do
           provisioning: { c: 3 },
           global: { d: 4 },
         })
-        expect(chain.normalize!).to eq({
+        expect(chain.normalize!).to match({
           :job => "CanvasSync::Jobs::BeginSyncChainJob",
+          :chain_link=>String,
           :args => [
             [
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[
                   [
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"], :b=>2}},
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users"], :b=>2}},
                     {
                       :job=>"CanvasSync::Jobs::SyncTermsJob",
+                      :chain_link=>String,
                       :args=>[],
                       :kwargs=>{
                         :term_scope=>"active",
                         :sub_jobs=>[
-                          {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :c=>3}},
+                          {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"], :c=>3}},
                         ],
                       },
                     },
@@ -62,20 +53,23 @@ RSpec.describe CanvasSync do
       context 'we are syncing users with a term scope' do
         it 'syncs the users in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active)
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[
                   [
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"]}},
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users"]}},
                     {
                       :job=>"CanvasSync::Jobs::SyncTermsJob",
+                      :chain_link=>String,
                       :args=>[],
                       :kwargs=>{
                         :sub_jobs=>[
-                          {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}
+                          {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}
                         ],
                         :term_scope=>"active",
                       }
@@ -93,17 +87,20 @@ RSpec.describe CanvasSync do
       context 'we are syncing users without a term scope' do
         it 'syncs users along with the rest of the provisioning report' do
           chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'])
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users", "courses"]}}],
                       :term_scope=>nil,
                     }
                   }
@@ -119,18 +116,21 @@ RSpec.describe CanvasSync do
       context 'we are syncing roles with a term scope' do
         it 'syncs the roles in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], term_scope: :active)
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
-                  {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
+                  {:job=>"CanvasSync::Jobs::SyncRolesJob", :chain_link=>String, :options=>{}},
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}],
                       :term_scope=>"active",
                     }
                   },
@@ -146,18 +146,21 @@ RSpec.describe CanvasSync do
       context 'we are syncing roles without a term scope' do
         it 'syncs roles separately even with no term scope' do
           chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'])
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
-                  {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
+                  {:job=>"CanvasSync::Jobs::SyncRolesJob", :chain_link=>String, :options=>{}},
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}],
                       :term_scope=>nil,
                     }
                   }
@@ -173,18 +176,21 @@ RSpec.describe CanvasSync do
       context 'we are syncing admins with a term scope' do
         it 'syncs the admins in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], term_scope: :active)
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
-                  {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
+                  {:job=>"CanvasSync::Jobs::SyncAdminsJob", :chain_link=>String, :options=>{}},
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}],
                       :term_scope=>"active"
                     }
                   }
@@ -200,18 +206,21 @@ RSpec.describe CanvasSync do
       context 'we are syncing admins without a term scope' do
         it 'syncs admins separately even with no term scope' do
           chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'])
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
-                  {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
+                  {:job=>"CanvasSync::Jobs::SyncAdminsJob", :chain_link=>String, :options=>{}},
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}],
                       :term_scope=>nil
                     }
                   }
@@ -228,19 +237,22 @@ RSpec.describe CanvasSync do
         it "appends the SyncAssignmentsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignments])
 
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
                       :sub_jobs=>[
-                        {:job=>"CanvasSync::Jobs::SyncAssignmentsJob", :options=>{}},
-                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"]}}
+                        {:job=>"CanvasSync::Jobs::SyncAssignmentsJob", :chain_link=>String, :options=>{}},
+                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users", "enrollments"]}}
                       ],
                       :term_scope=>nil
                     }
@@ -258,19 +270,22 @@ RSpec.describe CanvasSync do
         it "appends the SyncSubmissionsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments submissions])
 
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
                       :sub_jobs=>[
-                        {:job=>"CanvasSync::Jobs::SyncSubmissionsJob", :options=>{}},
-                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"]}}
+                        {:job=>"CanvasSync::Jobs::SyncSubmissionsJob", :chain_link=>String, :options=>{}},
+                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users", "enrollments"]}}
                       ],
                       :term_scope=>nil
                     }
@@ -288,19 +303,22 @@ RSpec.describe CanvasSync do
         it "appends the SyncAssignmentGroupsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignment_groups])
 
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
                       :sub_jobs=>[
-                        {:job=>"CanvasSync::Jobs::SyncAssignmentGroupsJob", :options=>{}},
-                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"]}}
+                        {:job=>"CanvasSync::Jobs::SyncAssignmentGroupsJob", :chain_link=>String, :options=>{}},
+                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users", "enrollments"]}}
                       ],
                       :term_scope=>nil
                     }