canvas_sync 0.16.4 → 0.21.0

data/lib/canvas_sync/processors/model_mappings.yml

@@ -7,6 +7,7 @@ users:
     user_id:
       database_column_name: sis_id
       type: string
+      deprecated: true
     email:
       database_column_name: email
       type: string
@@ -407,3 +408,422 @@ group_memberships:
     status:
       database_column_name: workflow_state
       type: string
+
+user_observers:
+  conflict_target:
+    - canvas_observer_id
+    - canvas_student_id
+  report_columns:
+    canvas_observer_id:
+      database_column_name: observing_user_id
+      type: integer
+    canvas_student_id:
+      database_column_name: observed_user_id
+      type: integer
+    status:
+      database_column_name: workflow_state
+      type: string
+
+grading_periods:
+  conflict_target: grading_period_id
+  report_columns:
+    grading_period_id:
+      database_column_name: canvas_id
+      type: integer
+    title:
+      database_column_name: title
+      type: string
+    weight:
+      database_column_name: weight
+      type: float
+    start_date:
+      database_column_name: start_date
+      type: datetime
+    end_date:
+      database_column_name: end_date
+      type: datetime
+    close_date:
+      database_column_name: close_date
+      type: datetime
+    grading_period_group_id:
+      database_column_name: canvas_grading_period_group_id
+      type: integer
+    status:
+      database_column_name: workflow_state
+      type: string
+
+grading_period_groups:
+  conflict_target: grading_period_group_id
+  report_columns:
+    grading_period_group_id:
+      database_column_name: canvas_id
+      type: integer
+    title:
+      database_column_name: title
+      type: string
+    weighted:
+      database_column_name: weighted
+      type: boolean
+    display_totals_for_all_grading_periods:
+      database_column_name: display_totals_for_all_grading_periods
+      type: boolean
+    status:
+      database_column_name: workflow_state
+      type: string
+
+content_migrations:
+  conflict_target: canvas_migration_id
+  report_columns:
+    canvas_migration_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_context_id:
+      database_column_name: canvas_context_id
+      type: integer
+    canvas_context_type:
+      database_column_name: canvas_context_type
+      type: string
+    workflow_state:
+      database_column_name: workflow_state
+      type: string
+    migration_settings:
+      database_column_name: migration_settings
+      type: text
+    started_at:
+      database_column_name: started_at
+      type: datetime
+    finished_at:
+      database_column_name: finished_at
+      type: datetime
+    progress:
+      database_column_name: progress
+      type: float
+    canvas_source_course_id:
+      database_column_name: canvas_source_course_id
+      type: integer
+    migration_type:
+      database_column_name: migration_type
+      type: string
+    canvas_child_subscription_id:
+      database_column_name: canvas_child_subscription_id
+      type: integer
+    canvas_root_account_id:
+      database_column_name: canvas_root_account_id
+      type: integer
+
+learning_outcomes:
+  conflict_target: learning_outcome_id
+  report_columns:
+    learning_outcome_id:
+      database_column_name: canvas_id
+      type: integer
+    context_id:
+      database_column_name: canvas_context_id
+      type: integer
+    context_type:
+      database_column_name: canvas_context_type
+      type: string
+    name:
+      database_column_name: name
+      type: string
+    friendly_name:
+      database_column_name: friendly_name
+      type: string
+    workflow_state:
+      database_column_name: workflow_state
+      type: string
+    created_at:
+      database_column_name: canvas_created_at
+      type: datetime
+    updated_at:
+      database_column_name: canvas_updated_at
+      type: datetime
+    migration_id:
+      database_column_name: migration_id
+      type: string
+    vendor_guid:
+      database_column_name: vendor_guid
+      type: string
+    low_grade:
+      database_column_name: low_grade
+      type: string
+    high_grade:
+      database_column_name: high_grade
+      type: string
+    calculation_method:
+      database_column_name: calculation_method
+      type: string
+    calculation_int:
+      database_column_name: calculation_int
+      type: integer
+    outcome_import_id:
+      database_column_name: outcome_import_id
+      type: integer
+    root_account_ids:
+      database_column_name: root_account_ids
+      type: integer
+    description:
+      database_column_name: description
+      type: string
+
+learning_outcome_results:
+  conflict_target: id
+  report_columns:
+    id:
+      database_column_name: canvas_id
+      type: integer
+    context_id:
+      database_column_name: canvas_context_id
+      type: integer
+    context_type:
+      database_column_name: canvas_context_type
+      type: string
+    association_id:
+      database_column_name: canvas_association_id
+      type: integer
+    association_type:
+      database_column_name: canvas_association_type
+      type: string
+    content_tag_id:
+      database_column_name: canvas_content_tag_id
+      type: integer
+    learning_outcome_id:
+      database_column_name: canvas_learning_outcome_id
+      type: integer
+    user_id:
+      database_column_name: canvas_user_id
+      type: integer
+    mastery:
+      database_column_name: mastery
+      type: boolean
+    original_mastery:
+      database_column_name: original_mastery
+      type: boolean
+    artifact_id:
+      database_column_name: canvas_artifact_id
+      type: integer
+    artifact_type:
+      database_column_name: canvas_artifact_type
+      type: string
+    assessed_at:
+      database_column_name: assessed_at
+      type: datetime
+    title:
+      database_column_name: title
+      type: string
+    percent:
+      database_column_name: percent
+      type: float
+    associated_asset_id:
+      database_column_name: canvas_associated_asset_id
+      type: integer
+    associated_asset_type:
+      database_column_name: canvas_associated_asset_type
+      type: string
+    attempt:
+      database_column_name: attempt
+      type: integer
+    score:
+      database_column_name: score
+      type: float
+    possible:
+      database_column_name: possible
+      type: float
+    original_score:
+      database_column_name: original_score
+      type: float
+    original_possible:
+      database_column_name: original_possible
+      type: float
+    workflow_state:
+      database_column_name: workflow_state
+      type: string
+
+course_nicknames:
+  conflict_target:
+    - canvas_user_id
+    - canvas_course_id
+  report_columns:
+    user_preference_value_id:
+      database_column_name: canvas_user_preference_value_id
+      type: integer
+    canvas_user_id:
+      database_column_name: canvas_user_id
+      type: integer
+    canvas_course_id:
+      database_column_name: canvas_course_id
+      type: integer
+    nickname:
+      database_column_name: nickname
+      type: string
+
+rubrics:
+  conflict_target: id
+  report_columns:
+    id:
+      database_column_name: canvas_id
+      type: integer
+    user_id:
+      database_column_name: canvas_user_id
+      type: integer
+    rubric_id:
+      database_column_name: canvas_rubric_id
+      type: integer
+    context_id:
+      database_column_name: canvas_context_id
+      type: integer
+    context_type:
+      database_column_name: canvas_context_type
+      type: string
+    data:
+      database_column_name: data
+      type: text
+    points_possible:
+      database_column_name: points_possible
+      type: float
+    title:
+      database_column_name: title
+      type: string
+    description:
+      database_column_name: description
+      type: text
+    created_at:
+      database_column_name: created_at
+      type: datetime
+    reusable:
+      database_column_name: reusable
+      type: boolean
+    public:
+      database_column_name: public
+      type: boolean
+    read_only:
+      database_column_name: read_only
+      type: boolean
+    association_count:
+      database_column_name: association_count
+      type: integer
+    free_form_criterion_comments:
+      database_column_name: free_form_criterion_comments
+      type: boolean
+    context_code:
+      database_column_name: context_code
+      type: string
+    migration_id:
+      database_column_name: migration_id
+      type: string
+    hide_score_total:
+      database_column_name: hide_score_total
+      type: boolean
+    workflow_state:
+      database_column_name: workflow_state
+      type: string
+    root_account_id:
+      database_column_name: canvas_root_account_id
+      type: integer
+
+rubric_associations:
+  conflict_target: id
+  report_columns:
+    id:
+      database_column_name: canvas_id
+      type: integer
+    rubric_id:
+      database_column_name: canvas_rubric_id
+      type: integer
+    association_id:
+      database_column_name: canvas_association_id
+      type: integer
+    association_type:
+      database_column_name: canvas_association_type
+      type: string
+    use_for_grading:
+      database_column_name: use_for_grading
+      type: boolean
+    created_at:
+      database_column_name: created_at
+      type: datetime
+    title:
+      database_column_name: title
+      type: string
+    summary_data:
+      database_column_name: summary_data
+      type: text
+    purpose:
+      database_column_name: purpose
+      type: string
+    url:
+      database_column_name: url
+      type: string
+    context_id:
+      database_column_name: canvas_context_id
+      type: integer
+    context_type:
+      database_column_name: canvas_context_type
+      type: string
+    hide_score_total:
+      database_column_name: hide_score_total
+      type: boolean
+    bookmarked:
+      database_column_name: bookmarked
+      type: boolean
+    context_code:
+      database_column_name: context_code
+      type: string
+    hide_points:
+      database_column_name: hide_points
+      type: boolean
+    hide_outcome_results:
+      database_column_name: hide_outcome_results
+      type: boolean
+    root_account_id:
+      database_column_name: canvas_root_account_id
+      type: integer
+    workflow_state:
+      database_column_name: workflow_state
+      type: string
+
+rubric_assessments:
+  conflict_target: id
+  report_columns:
+    id:
+      database_column_name: canvas_id
+      type: integer
+    user_id:
+      database_column_name: canvas_user_id
+      type: integer
+    rubric_id:
+      database_column_name: canvas_rubric_id
+      type: integer
+    rubric_association_id:
+      database_column_name: canvas_rubric_association_id
+      type: integer
+    score:
+      database_column_name: score
+      type: float
+    data:
+      database_column_name: data
+      type: text
+    created_at:
+      database_column_name: created_at
+      type: datetime
+    artifact_id:
+      database_column_name: canvas_artifact_id
+      type: integer
+    artifact_type:
+      database_column_name: canvas_artifact_type
+      type: string
+    assessment_type:
+      database_column_name: assessment_type
+      type: string
+    assessor_id:
+      database_column_name: canvas_assessor_id
+      type: integer
+    artifact_attempt:
+      database_column_name: artifact_attempt
+      type: integer
+    hide_points:
+      database_column_name: hide_points
+      type: boolean
+    root_account_id:
+      database_column_name: canvas_root_account_id
+      type: integer

data/lib/canvas_sync/processors/normal_processor.rb

@@ -13,12 +13,12 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        conflict_target =  mapping[options[:mapping].to_sym][:conflict_target]
+        m = mapping[options[:mapping].to_sym]
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
-          mapping[options[:mapping].to_sym][:report_columns],
+          m[:columns],
           options[:klass].constantize,
-          conflict_target ? conflict_target.to_sym : conflict_target
+          m[:conflict_target],
         )
       end
     end

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -21,7 +21,6 @@ module CanvasSync
 
       def initialize(report_file_path, options) # rubocop:disable Metrics/AbcSize
         @options = options
-
         if options[:models].length == 1
           run_import(options[:models][0], report_file_path)
         else
@@ -70,85 +69,73 @@ module CanvasSync
       end
 
       def bulk_process_users(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:users][:report_columns],
-          User,
-          mapping[:users][:conflict_target].to_sym,
-        )
+        do_bulk_import(report_file_path, User, options: @options)
+      end
+
+      def bulk_process_user_observers(report_file_path)
+        do_bulk_import(report_file_path, UserObserver, options: @options)
       end
 
       def bulk_process_pseudonyms(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:pseudonyms][:report_columns],
-          Pseudonym,
-          mapping[:pseudonyms][:conflict_target].to_sym,
-        )
+        do_bulk_import(report_file_path, Pseudonym, options: @options)
       end
 
       def bulk_process_accounts(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:accounts][:report_columns],
-          Account,
-          mapping[:accounts][:conflict_target].to_sym
-        )
+        do_bulk_import(report_file_path, Account, options: @options)
       end
 
       def bulk_process_courses(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:courses][:report_columns],
-          Course,
-          mapping[:courses][:conflict_target].to_sym,
-        )
+        do_bulk_import(report_file_path, Course, options: @options)
       end
 
       def bulk_process_enrollments(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:enrollments][:report_columns],
-          Enrollment,
-          mapping[:enrollments][:conflict_target].to_sym,
-        )
+        do_bulk_import(report_file_path, Enrollment, options: @options)
       end
 
       def bulk_process_sections(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:sections][:report_columns],
-          Section,
-          mapping[:sections][:conflict_target].to_sym,
-        )
+        do_bulk_import(report_file_path, Section, options: @options)
       end
 
       def bulk_process_xlist(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:xlist][:report_columns],
-          Section,
-          mapping[:xlist][:conflict_target].to_sym,
-        )
+        do_bulk_import(report_file_path, Section, options: @options, mapping_key: :xlist)
       end
 
       def bulk_process_groups(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:groups][:report_columns],
-          Group,
-          mapping[:groups][:conflict_target].to_sym,
-        )
+        do_bulk_import(report_file_path, Group, options: @options)
+      end
+
+      def bulk_process_grading_periods(report_file_path)
+        do_bulk_import(report_file_path, GradingPeriod, options: @options)
+      end
+
+      def bulk_process_grading_period_groups(report_file_path)
+        do_bulk_import(report_file_path, GradingPeriodGroup, options: @options)
       end
 
       # Note that group membership is singular because we override the model name param in sync_provisioning_report_job
       def bulk_process_group_membership(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:group_memberships][:report_columns],
-          GroupMembership,
-          mapping[:group_memberships][:conflict_target].to_sym,
-        )
+        do_bulk_import(report_file_path, GroupMembership, options: @options)
+      end
+
+      def bulk_process_learning_outcomes(report_file_path)
+        do_bulk_import(report_file_path, LearningOutcome, options: @options) do |row|
+          row[:root_account_ids] = JSON.parse row[:root_account_ids]
+          row
+        end
+      end
+
+      def bulk_process_learning_outcome_results(report_file_path)
+        do_bulk_import(report_file_path, LearningOutcomeResult, options: @options)
+      end
+
+      def bulk_process_course_nicknames(report_file_path)
+        exists = []
+        do_bulk_import(report_file_path, CourseNickname, options: @options) do |row|
+          exists << row[:user_preference_value_id]
+          row
+        end
+        # Canvas does not soft delete UserPreferenceValues so there's no point in doing the same
+        CourseNickname.where.not(canvas_user_preference_value_id: exists).or(CourseNickname.where(canvas_user_preference_value_id: nil)).delete_all
       end
     end
   end

data/lib/canvas_sync/processors/report_processor.rb

@@ -5,17 +5,23 @@ module CanvasSync
     # Base report processing class
     class ReportProcessor
       def mapping
-        @mapping ||= begin
-          mapping = YAML.load_file(File.join(__dir__, "model_mappings.yml")).deep_symbolize_keys!
-          override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
+        CanvasSync::Concerns::SyncMapping::Mapping.default_mappings
+      end
 
-          if File.file?(override_filepath)
-            override = YAML.load_file(override_filepath).deep_symbolize_keys!
-            mapping = mapping.merge(override)
-          end
+      def mapping_for(model, key = nil)
+        CanvasSync::Concerns::SyncMapping.mapping_for(model, key)
+      end
 
-          mapping
-        end
+      def do_bulk_import(report_file_path, model, options: {}, mapping_key: nil, &blk)
+        m = mapping_for(model, mapping_key)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          m[:columns],
+          model,
+          m[:conflict_target],
+          import_args: options,
+          &blk
+        )
       end
     end
   end

data/lib/canvas_sync/processors/rubric_assessments_processor.rb

@@ -0,0 +1,19 @@
+require_relative "./report_processor"
+
+module CanvasSync
+  module Processors
+    # Processes a rubric assessments report using the bulk importer.
+    #
+    # @param report_file_path [String]
+    # @param options [Hash]
+    class RubricAssessmentsProcessor < ReportProcessor
+      def self.process(report_file_path, _options, report_id)
+        new(report_file_path, _options)
+      end
+
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, RubricAssessment, options: options)
+      end
+    end
+  end
+end

data/lib/canvas_sync/processors/rubric_associations_processor.rb

@@ -0,0 +1,19 @@
+require_relative "./report_processor"
+
+module CanvasSync
+  module Processors
+    # Processes a rubric association report using the bulk importer.
+    #
+    # @param report_file_path [String]
+    # @param options [Hash]
+    class RubricAssociationsProcessor < ReportProcessor
+      def self.process(report_file_path, _options, report_id)
+        new(report_file_path, _options)
+      end
+
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, RubricAssociation, options: options)
+      end
+    end
+  end
+end

data/lib/canvas_sync/processors/rubrics_processor.rb

@@ -0,0 +1,19 @@
+require_relative "./report_processor"
+
+module CanvasSync
+  module Processors
+    # Processes a rubrics report using the bulk importer.
+    #
+    # @param report_file_path [String]
+    # @param options [Hash]
+    class RubricsProcessor < ReportProcessor
+      def self.process(report_file_path, _options, report_id)
+        new(report_file_path, _options)
+      end
+
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, Rubric, options: options)
+      end
+    end
+  end
+end

data/lib/canvas_sync/processors/submissions_processor.rb

@@ -8,16 +8,11 @@ module CanvasSync
     # @param options [Hash]
     class SubmissionsProcessor < ReportProcessor
       def self.process(report_file_path, _options, report_id)
-        new(report_file_path)
+        new(report_file_path, _options)
       end
 
-      def initialize(report_file_path)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:submissions][:report_columns],
-          Submission,
-          mapping[:submissions][:conflict_target].to_sym,
-        )
+      def initialize(report_file_path, options)
+        do_bulk_import(report_file_path, Submission, options: options)
       end
     end
   end