canvas_sync 0.17.20 → 0.17.23.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 761f46113fd5ade5684101672ca0fda352e0fbec602bb8fe0fdbcaf584e728ba
-  data.tar.gz: '0827e3a31d1e2bf7bf0afdece5a6a123f2221f2db9a4d239a4c543024641da57'
+  metadata.gz: 6fe925c97d18da5736681b55e51a1702d727ef2900097c9058de6f247748af9f
+  data.tar.gz: 27c0bd4eb8d57efc705cbe3affdfa6ce4af9580b1b2374e11a8e7e84ddf2b8bd
 SHA512:
-  metadata.gz: 0ae25ea9b6305902f220604a4d040737cca48aa6a7d89960357179ad852951bcb99a93693c3fd684ef3a12b3db89874023f48f5ac6d54dac35edb3fd7eefcc57
-  data.tar.gz: ed161a60791e7d326faabea6431d6352eee48ea2c23dd0ff224e902ac9e545c881f18a3d0d31bffa186e95d5d840f605e492f58ef7b68d9583da7afd011dde89
+  metadata.gz: 82f682b5e2b713ba2adc45483f215c1de8103dc16bdd3872a1656649527053a8819de35b50d1f4bc8e9961bedbc647fd7a03dbebbe4f2ad0fb3235d6754788be
+  data.tar.gz: 674159253800095ba66e15d9d14f58fd41d46f7fa70a33a8303160488544d721f7e8c83700c8a5e4978436e01d34e070b7bc22ff7433219e68fe00d4620dc38f

data/README.md

@@ -190,7 +190,30 @@ Overrides are useful for two scenarios:
 - You have an existing application where the column names do not match up with what CanvasSync expects
 - You want to sync some other column in the report that CanvasSync is not configured to sync
 
-In order to create an override, place a file called `canvas_sync_provisioning_mapping.yml` in your Rails `config` directory. Define the tables and columns you want to override using the following format:
+Mappings can be modified by editing the Model class like such:
+```ruby
+class User < ApplicationRecord
+  include CanvasSync::Record
+
+  sync_mapping(reset: false) do # `reset: false` is the default
+    # The mapping can be totally cleared with `reset: true` in the `sync_mapping` call, or like such:
+    reset_links
+
+    # Add a new column:
+    link_column :column_in_report => :column_in_database, type: :datetime
+
+    # If the column name on the report and in the DB are the same, a shorthand can be used:
+    link_column :omit_from_final_grade, type: :datetime
+
+    # If the defaults define a column you don't want synced, you can remove it from the mapping:
+    unlink_column :column_in_database
+  end
+
+  # ...
+end
+```
+
+You can also create a file called `canvas_sync_provisioning_mapping.yml` in your Rails `config` directory. However, this approach requires you to re-specify the complete table in order to modify a table. Define the tables and columns you want to override using the following format:
 
 ```ruby
 users:

data/lib/canvas_sync/concerns/sync_mapping.rb

@@ -0,0 +1,112 @@
+module CanvasSync::Concerns
+  module SyncMapping
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def sync_mapping(key = nil, reset: false, &blk)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        existing_map = get_sync_mapping(key)
+        mapper = Mapping.new(existing_map&.deep_dup || {}.with_indifferent_access)
+        mapper.reset_links if reset
+        mapper.instance_exec(&blk)
+        @sync_mappings[key] = mapper.map_def.freeze
+      end
+
+      def get_sync_mapping(key = nil)
+        key ||= Mapping.normalize_model_name(self)
+        key = key.to_s
+        @sync_mappings ||= {}
+        @sync_mappings[key] || superclass.try(:get_sync_mapping, key) || Mapping.default_for(key)
+      end
+    end
+
+    class Mapping
+      attr_reader :map_def
+
+      def initialize(map_def = {}, model: nil)
+        @map_def = map_def
+        @model = model
+      end
+
+      def self.normalize_model_name(model)
+        model = model.name unless model.is_a?(String)
+        model.pluralize.underscore
+      end
+
+      def self.default_for(key)
+        default_mappings[key]
+      end
+
+      def self.default_mappings
+        @mappings ||= begin
+          maps = {}
+          default_v1_mappings.each do |mname, legacy|
+            m = maps[mname] = {}
+
+            m[:conflict_target] = Array(legacy[:conflict_target]).map(&:to_sym).map do |lct|
+              legacy[:report_columns][lct][:database_column_name]
+            end
+
+            m[:report_columns] = {}
+            legacy[:report_columns].each do |rcol, opts|
+              m[:report_columns][opts[:database_column_name]] = opts.except(:database_column_name).merge!(
+                report_column: rcol,
+              ).freeze
+            end
+          end
+          maps.with_indifferent_access.freeze
+        end
+      end
+
+      def self.default_v1_mappings
+        @legacy_mappings ||= begin
+          mapping = YAML.load_file(File.join(__dir__, '../processors', "model_mappings.yml")).deep_symbolize_keys!
+          override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
+
+          if File.file?(override_filepath)
+            override = YAML.load_file(override_filepath).deep_symbolize_keys!
+            mapping = mapping.merge(override)
+          end
+
+          mapping.freeze
+        end
+      end
+
+      def conflict_target(*columns)
+        if columns.count == 0
+          @map_def[:conflict_target]
+        else
+          @map_def[:conflict_target] = columns.flatten.compact
+        end
+      end
+
+      def reset_links
+        @map_def = {}
+      end
+
+      def unlink_column(key)
+        @map_def.delete(key)
+      end
+
+      def link_column(m, type: nil, &blk)
+        if m.is_a?(Hash)
+          raise "Hash should have exactly 1 entry" if m && m.count != 1
+          @map_def[:report_columns][m.values[0]] = {
+            report_column: m.keys[0],
+            type: type,
+            transform: blk,
+          }
+        elsif m.is_a?(Symbol)
+          @map_def[:report_columns][m] = {
+            report_column: m,
+            type: type,
+            transform: blk,
+          }
+        else
+          raise "Cannot handle argument of type #{m.class}"
+        end
+      end
+    end
+  end
+end

data/lib/canvas_sync/generators/install_generator.rb

@@ -48,6 +48,7 @@ module CanvasSync
       models.each do |model|
         migration_template "migrations/create_#{model}.rb", "db/migrate/create_#{model}.rb"
         template "models/#{model.singularize}.rb", "app/models/#{model.singularize}.rb"
+      rescue
       end
     end
   end

data/lib/canvas_sync/generators/templates/migrations/create_grading_period_groups.rb

@@ -0,0 +1,22 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
+  def change
+    create_table :grading_period_groups do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_course_id
+      t.bigint :canvas_account_id
+      t.string :title
+      t.boolean :weighted
+      t.boolean :display_totals_for_all_grading_periods
+
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :grading_period_groups, :canvas_id, unique: true
+    add_index :grading_period_groups, :canvas_course_id
+    add_index :grading_period_groups, :canvas_account_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_grading_periods.rb

@@ -0,0 +1,22 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateGradingPeriods < ActiveRecord::Migration[5.1]
+  def change
+    create_table :grading_periods do |t|
+      t.bigint :canvas_id, null: false
+      t.string :title
+      t.float :weight
+      t.datetime :start_date
+      t.datetime :end_date
+      t.datetime :close_date
+      t.bigint :canvas_grading_period_group_id
+
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :grading_periods, :canvas_id, unique: true
+    add_index :grading_periods, :canvas_grading_period_group_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_user_observers.rb

@@ -0,0 +1,17 @@
+# <%= autogenerated_migration_warning %>
+
+class CreateUserObservers < ActiveRecord::Migration[5.1]
+  def change
+    create_table :user_observers do |t|
+      t.bigint :observing_user_id
+      t.bigint :observed_user_id
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :user_observers, [:observed_user_id, :observing_user_id], unique: true
+    add_index :user_observers, :observing_user_id
+    add_index :user_observers, :observed_user_id
+  end
+end

data/lib/canvas_sync/generators/templates/models/grading_period.rb

@@ -0,0 +1,8 @@
+# <%= autogenerated_model_warning %>
+
+class GradingPeriod < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :canvas_grading_period_group_id, optional: true
+end

data/lib/canvas_sync/generators/templates/models/grading_period_group.rb

@@ -0,0 +1,9 @@
+# <%= autogenerated_model_warning %>
+
+class GradingPeriodGroup < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
+  belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
+end

data/lib/canvas_sync/generators/templates/models/user_observer.rb

@@ -0,0 +1,11 @@
+# <%= autogenerated_model_warning %>
+
+class UserObserver < ApplicationRecord
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
+
+  validates :canvas_id, uniqueness: true, presence: true
+
+  belongs_to :observing_user, primary_key: :canvas_id, foreign_key: :observing_user_id, class_name: 'User', optional: true
+  belongs_to :observed_user, primary_key: :canvas_id, foreign_key: :observed_user_id, class_name: 'User', optional: true
+end

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -24,30 +24,41 @@ module CanvasSync
       end
 
       def self.perform_in_batches(report_file_path, mapping, klass, conflict_target, import_args: {})
-        csv_column_names = mapping.keys
-        database_column_names = mapping.values.map { |value| value[:database_column_name] }
-        database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
+        csv_column_names = mapping.values.map { |value| value[:report_column].to_s }
+        database_column_names = mapping.keys
+
+        conflict_target = Array(conflict_target).map(&:to_s)
+        conflict_target_indices = conflict_target.map{|ct| database_column_names.index(ct) }
 
         row_ids = {}
         batcher = CanvasSync::BatchProcessor.new(of: batch_size) do |batch|
           row_ids = {}
-          perform_import(klass, database_column_names, batch, database_conflict_column_name, import_args)
+          perform_import(klass, database_column_names, batch, conflict_target, import_args)
         end
 
         row_buffer_out = ->(row) {
-          if conflict_target
-            next if row_ids[row[conflict_target]]
-            row_ids[row[conflict_target]] = true
+          formatted_row = mapping.map do |db_col, col_def|
+            value = nil
+            value = row[col_def[:report_column]] if col_def[:report_column]
+
+            if col_def[:type]
+              if col_def[:type].to_sym == :datetime
+                # TODO: add some timezone config to the mapping.
+                # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
+                value = DateTime.parse(value).utc rescue nil # rubocop:disable Style/RescueModifier
+              end
+            end
+
+            value = col_def[:transform].call(value, row) if col_def[:transform]
+
+            value
           end
 
-          formatted_row = csv_column_names.map do |column|
-            if mapping[column][:type].to_sym == :datetime
-              # TODO: add some timezone config to the mapping.
-              # In cases where the timestamp or date doesn't include a timezone, you should be able to specify one
-              DateTime.parse(row[column]).utc rescue nil # rubocop:disable Style/RescueModifier
-            else
-              row[column]
-            end
+          if conflict_target.present?
+            key = conflict_target_indices.map{|ct| formatted_row[ct] }
+            next if row_ids[key]
+
+            row_ids[key] = true
           end
 
           batcher << formatted_row
@@ -79,7 +90,7 @@ module CanvasSync
           condition: condition_sql(klass, columns, import_args[:sync_start_time]),
           columns: columns
         }
-        update_conditions[:conflict_target] = conflict_target if conflict_target
+        update_conditions[:conflict_target] = conflict_target if conflict_target.present?
 
         options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
         options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -40,7 +40,7 @@ module CanvasSync
       def insert_at(position, new_jobs)
         chain = self.class.get_chain_parameter(base_job)
         new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
-        chain.insert(-1, *new_jobs)
+        chain.insert(position, *new_jobs)
       end
 
       def insert(new_jobs, **kwargs)

data/lib/canvas_sync/processors/assignment_groups_processor.rb

@@ -12,13 +12,7 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:assignment_groups][:report_columns],
-          AssignmentGroup,
-          mapping[:assignment_groups][:conflict_target].to_sym,
-          import_args: options
-        )
+        do_bulk_import(report_file_path, AssignmentGroup, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/assignments_processor.rb

@@ -12,13 +12,7 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:assignments][:report_columns],
-          Assignment,
-          mapping[:assignments][:conflict_target].to_sym,
-          import_args: options
-        )
+        do_bulk_import(report_file_path, Assignment, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/context_module_items_processor.rb

@@ -12,13 +12,7 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:context_module_items][:report_columns],
-          ContextModuleItem,
-          mapping[:context_module_items][:conflict_target].to_sym,
-          import_args: options
-        )
+        do_bulk_import(report_file_path, ContextModuleItem, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/context_modules_processor.rb

@@ -12,13 +12,7 @@ module CanvasSync
       end
 
       def initialize(report_file_path, options)
-        CanvasSync::Importers::BulkImporter.import(
-          report_file_path,
-          mapping[:context_modules][:report_columns],
-          ContextModule,
-          mapping[:context_modules][:conflict_target].to_sym,
-          import_args: options
-        )
+        do_bulk_import(report_file_path, ContextModule, options: options)
       end
     end
   end

data/lib/canvas_sync/processors/model_mappings.yml

@@ -407,3 +407,71 @@ group_memberships:
     status:
       database_column_name: workflow_state
       type: string
+
+user_observers:
+  conflict_target:
+    - canvas_observer_id
+    - canvas_student_id
+  report_columns:
+    canvas_observer_id:
+      database_column_name: observing_user_id
+      type: integer
+    canvas_student_id:
+      database_column_name: observed_user_id
+      type: integer
+    status:
+      database_column_name: workflow_state
+      type: string
+
+grading_periods:
+  conflict_target: grading_period_id
+  report_columns:
+    grading_period_id:
+      database_column_name: canvas_id
+      type: integer
+    title:
+      database_column_name: title
+      type: string
+    weight:
+      database_column_name: weight
+      type: float
+    start_date:
+      database_column_name: start_date
+      type: datetime
+    end_date:
+      database_column_name: end_date
+      type: datetime
+    close_date:
+      database_column_name: close_date
+      type: datetime
+    grading_period_group_id:
+      database_column_name: canvas_grading_period_group_id
+      type: integer
+    status:
+      database_column_name: workflow_state
+      type: string
+
+grading_period_groups:
+  conflict_target: grading_period_group_id
+  report_columns:
+    grading_period_group_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_course_id:
+      database_column_name: canvas_course_id
+      type: integer
+    canvas_account_id:
+      database_column_name: canvas_account_id
+      type: integer
+    title:
+      database_column_name: title
+      type: string
+    weighted:
+      database_column_name: weighted
+      type: boolean
+    display_totals_for_all_grading_periods:
+      database_column_name: display_totals_for_all_grading_periods
+      type: boolean
+    status:
+      database_column_name: workflow_state
+      type: string