canvas_sync 0.17.18 → 0.17.23.beta2

data/lib/canvas_sync/jobs/report_starter.rb

@@ -12,7 +12,7 @@ module CanvasSync
       def perform(report_name, report_params, processor, options, allow_redownloads: false)
         account_id = options[:account_id] || batch_context[:account_id] || "self"
         options[:sync_start_time] = DateTime.now.utc.iso8601
-
+        options[:report_params] = report_params
         report_id = start_report(account_id, report_name, report_params)
         # TODO: Restore report caching support (does nayone actually use it?)
         # report_id = if allow_redownloads
@@ -28,7 +28,7 @@ module CanvasSync
             report_name,
             report_id,
             processor,
-            options,
+            options
           )
         end
       end

data/lib/canvas_sync/processors/assignment_groups_processor.rb

@@ -12,13 +12,7 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:assignment_groups][:report_columns],
-          AssignmentGroup,
-          mapping[:assignment_groups][:conflict_target].to_sym,
-          import_args: options
-        )
+        do_bulk_import(report_file_path, AssignmentGroup, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/assignments_processor.rb

@@ -12,13 +12,7 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:assignments][:report_columns],
-          Assignment,
-          mapping[:assignments][:conflict_target].to_sym,
-          import_args: options
-        )
+        do_bulk_import(report_file_path, Assignment, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/context_module_items_processor.rb

@@ -12,13 +12,7 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:context_module_items][:report_columns],
-          ContextModuleItem,
-          mapping[:context_module_items][:conflict_target].to_sym,
-          import_args: options
-        )
+        do_bulk_import(report_file_path, ContextModuleItem, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/context_modules_processor.rb

@@ -12,13 +12,7 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:context_modules][:report_columns],
-          ContextModule,
-          mapping[:context_modules][:conflict_target].to_sym,
-          import_args: options
-        )
+        do_bulk_import(report_file_path, ContextModule, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/model_mappings.yml

@@ -407,3 +407,71 @@ group_memberships:
     status:
       database_column_name: workflow_state
       type: string
+
+user_observers:
+  conflict_target:
+    - canvas_observer_id
+    - canvas_student_id
+  report_columns:
+    canvas_observer_id:
+      database_column_name: observing_user_id
+      type: integer
+    canvas_student_id:
+      database_column_name: observed_user_id
+      type: integer
+    status:
+      database_column_name: workflow_state
+      type: string
+
+grading_periods:
+  conflict_target: grading_period_id
+  report_columns:
+    grading_period_id:
+      database_column_name: canvas_id
+      type: integer
+    title:
+      database_column_name: title
+      type: string
+    weight:
+      database_column_name: weight
+      type: float
+    start_date:
+      database_column_name: start_date
+      type: datetime
+    end_date:
+      database_column_name: end_date
+      type: datetime
+    close_date:
+      database_column_name: close_date
+      type: datetime
+    grading_period_group_id:
+      database_column_name: canvas_grading_period_group_id
+      type: integer
+    status:
+      database_column_name: workflow_state
+      type: string
+
+grading_period_groups:
+  conflict_target: grading_period_group_id
+  report_columns:
+    grading_period_group_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_course_id:
+      database_column_name: canvas_course_id
+      type: integer
+    canvas_account_id:
+      database_column_name: canvas_account_id
+      type: integer
+    title:
+      database_column_name: title
+      type: string
+    weighted:
+      database_column_name: weighted
+      type: boolean
+    display_totals_for_all_grading_periods:
+      database_column_name: display_totals_for_all_grading_periods
+      type: boolean
+    status:
+      database_column_name: workflow_state
+      type: string

data/lib/canvas_sync/processors/normal_processor.rb

@@ -13,12 +13,12 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        conflict_target =  mapping[options[:mapping].to_sym][:conflict_target]
+        m = mapping[options[:mapping].to_sym]
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
-          mapping[options[:mapping].to_sym][:report_columns],
+          m[:report_columns],
           options[:klass].constantize,
-          conflict_target ? conflict_target.to_sym : conflict_target,
+          m[:conflict_target],
           import_args: options
         )
       end

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -69,94 +69,52 @@ module CanvasSync
       end
 
       def bulk_process_users(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:users][:report_columns],
-          User,
-          mapping[:users][:conflict_target].to_sym,
-          import_args: @options
-        )
+        do_bulk_import(report_file_path, User, options: @options)
+      end
+
+      def bulk_process_user_observers(report_file_path)
+        do_bulk_import(report_file_path, UserObserver, options: @options)
       end
 
       def bulk_process_pseudonyms(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:pseudonyms][:report_columns],
-          Pseudonym,
-          mapping[:pseudonyms][:conflict_target].to_sym,
-          import_args: @options
-        )
+        do_bulk_import(report_file_path, Pseudonym, options: @options)
       end
 
       def bulk_process_accounts(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:accounts][:report_columns],
-          Account,
-          mapping[:accounts][:conflict_target].to_sym,
-          import_args: @options
-        )
+        do_bulk_import(report_file_path, Account, options: @options)
       end
 
       def bulk_process_courses(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:courses][:report_columns],
-          Course,
-          mapping[:courses][:conflict_target].to_sym,
-          import_args: @options
-        )
+        do_bulk_import(report_file_path, Course, options: @options)
       end
 
       def bulk_process_enrollments(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:enrollments][:report_columns],
-          Enrollment,
-          mapping[:enrollments][:conflict_target].to_sym,
-          import_args: @options
-        )
+        do_bulk_import(report_file_path, Enrollment, options: @options)
       end
 
       def bulk_process_sections(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:sections][:report_columns],
-          Section,
-          mapping[:sections][:conflict_target].to_sym,
-          import_args: @options
-        )
+        do_bulk_import(report_file_path, Section, options: @options)
       end
 
       def bulk_process_xlist(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:xlist][:report_columns],
-          Section,
-          mapping[:xlist][:conflict_target].to_sym,
-          import_args: @options
-        )
+        do_bulk_import(report_file_path, Section, options: @options, mapping_key: :xlist)
       end
 
       def bulk_process_groups(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:groups][:report_columns],
-          Group,
-          mapping[:groups][:conflict_target].to_sym,
-          import_args: @options
-        )
+        do_bulk_import(report_file_path, Group, options: @options)
+      end
+
+      def bulk_process_grading_periods(report_file_path)
+        do_bulk_import(report_file_path, GradingPeriod, options: @options)
+      end
+
+      def bulk_process_grading_period_groups(report_file_path)
+        do_bulk_import(report_file_path, GradingPeriodGroup, options: @options)
       end
 
       # Note that group membership is singular because we override the model name param in sync_provisioning_report_job
       def bulk_process_group_membership(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:group_memberships][:report_columns],
-          GroupMembership,
-          mapping[:group_memberships][:conflict_target].to_sym,
-          import_args: @options
-        )
+        do_bulk_import(report_file_path, GroupMembership, options: @options)
       end
     end
   end

data/lib/canvas_sync/processors/report_processor.rb

@@ -5,17 +5,22 @@ module CanvasSync
     # Base report processing class
     class ReportProcessor
       def mapping
-        @mapping ||= begin
-          mapping = YAML.load_file(File.join(__dir__, "model_mappings.yml")).deep_symbolize_keys!
-          override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
+        SyncMapping::Mapping.default_mappings
+      end
 
-          if File.file?(override_filepath)
-            override = YAML.load_file(override_filepath).deep_symbolize_keys!
-            mapping = mapping.merge(override)
-          end
+      def mapping_for(model, key = nil)
+        model.try(:get_sync_mapping, key) || mapping[key || SyncMapping::Mapping.normalize_model_name(model)]
+      end
 
-          mapping
-        end
+      def do_bulk_import(report_file_path, model, options: {}, mapping_key: nil)
+        m = mapping_for(model, mapping_key)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          m[:report_columns],
+          model,
+          m[:conflict_target],
+          import_args: options
+        )
       end
     end
   end

data/lib/canvas_sync/processors/submissions_processor.rb

@@ -12,13 +12,7 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:submissions][:report_columns],
-          Submission,
-          mapping[:submissions][:conflict_target].to_sym,
-          import_args: options
-        )
+        do_bulk_import(report_file_path, Submission, options: options)
       end
     end
   end

data/lib/canvas_sync/record.rb

@@ -1,6 +1,10 @@
+
+require_relative 'concerns/sync_mapping'
+
 module CanvasSync
   module Record
     extend ActiveSupport::Concern
+    include CanvasSync::Concerns::SyncMapping
 
     included do
       define_model_callbacks :sync_import

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.18".freeze
+  VERSION = "0.17.23.beta2".freeze
 end

data/lib/canvas_sync.rb

@@ -41,6 +41,9 @@ module CanvasSync
     context_modules
     context_module_items
     xlist
+    user_observers
+    grading_periods
+    grading_period_groups
   ].freeze
 
   SUPPORTED_TERM_SCOPE_MODELS = %w[
@@ -176,7 +179,7 @@ module CanvasSync
       # These Models use the provisioning report, but are not term-scoped,
       # so we sync them outside of the term scoping to ensure work is not duplicated
       if term_scope.present?
-        models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
+        models -= (first_provisioning_models = models & ['users', 'pseudonyms', 'user_observers'])
         current_chain.insert(generate_provisioning_jobs(first_provisioning_models, options))
       end
 

data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb

@@ -62,6 +62,46 @@ RSpec.describe CanvasSync::Processors::ProvisioningReportProcessor do
       expect(group.canvas_user_id).to eq 10
     end
 
+    it 'processes user_observers' do
+      expect {
+        subject.process('spec/support/fixtures/reports/user_observers.csv', { models: ['user_observers'] }, 1)
+      }.to change { UserObserver.count }.by(2)
+      obj = UserObserver.find_by(observed_user_id: 3)
+      expect(obj.workflow_state).to eq 'active'
+      expect(obj.observing_user_id).to eq 21
+    end
+
+    it 'processes grading_periods' do
+      expect {
+        subject.process('spec/support/fixtures/reports/grading_periods.csv', { models: ['grading_periods'] }, 1)
+      }.to change { GradingPeriod.count }.by(2)
+      obj = GradingPeriod.find_by(canvas_id: 1)
+      puts obj.inspect
+      expect(obj.title).to eq 'Period 1'
+      expect(obj.weight).to eq 0.2
+      expect(obj.workflow_state).to eq 'active'
+    end
+
+    it 'processes grading_period_groups' do
+      expect {
+        subject.process('spec/support/fixtures/reports/grading_period_groups.csv', { models: ['grading_period_groups'] }, 1)
+      }.to change { GradingPeriodGroup.count }.by(1)
+      obj = GradingPeriodGroup.find_by(canvas_id: 1)
+      puts obj.inspect
+      expect(obj.title).to eq 'Test Group'
+      expect(obj.workflow_state).to eq 'active'
+    end
+
+    it 'model with composite key behaves as expected' do
+      expect {
+        subject.process('spec/support/fixtures/reports/user_observers.csv', { models: ['user_observers'] }, 1)
+      }.to change { UserObserver.count }.by(2)
+
+      expect {
+        subject.process('spec/support/fixtures/reports/user_observers.csv', { models: ['user_observers'] }, 1)
+      }.to change { UserObserver.count }.by(0)
+    end
+
     context 'options[:models] is multiple models' do
       it 'extracts the ZIP and processes each model' do
         user_count = User.count

data/spec/dummy/app/models/grading_period.rb

@@ -0,0 +1,14 @@
+# #
+# AUTO GENERATED MODEL
+# This model was auto generated by the CanvasSync Gem.
+# You can customize it as needed, but make sure you test
+# any changes you make to the auto generated methods.
+#
+
+
+class GradingPeriod < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :canvas_grading_period_group_id, optional: true
+end

data/spec/dummy/app/models/grading_period_group.rb

@@ -0,0 +1,15 @@
+# #
+# AUTO GENERATED MODEL
+# This model was auto generated by the CanvasSync Gem.
+# You can customize it as needed, but make sure you test
+# any changes you make to the auto generated methods.
+#
+
+
+class GradingPeriodGroup < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
+  belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
+end