canvas_sync 0.12.0 → 0.16.1

data/lib/canvas_sync/generators/templates/models/role.rb

@@ -1,7 +1,8 @@
 # <%= autogenerated_model_warning %>
 
 class Role < ApplicationRecord
-  include CanvasSync::ApiSyncable
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
 
   validates :canvas_id, uniqueness: true, presence: true
   has_many :admins, foreign_key: :canvas_role_id, primary_key: :canvas_id

data/lib/canvas_sync/generators/templates/models/section.rb

@@ -1,7 +1,8 @@
 # <%= autogenerated_model_warning %>
 
 class Section < ApplicationRecord
-  include CanvasSync::ApiSyncable
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
 
   validates :canvas_id, uniqueness: true, presence: true
   belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true

data/lib/canvas_sync/generators/templates/models/submission.rb

@@ -1,7 +1,8 @@
 # <%= autogenerated_model_warning %>
 
 class Submission < ApplicationRecord
-  include CanvasSync::ApiSyncable
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
 
   validates :canvas_id, uniqueness: true, presence: true
   belongs_to :assignment, primary_key: :canvas_id, foreign_key: :canvas_assignment_id, optional: true
@@ -13,6 +14,7 @@ class Submission < ApplicationRecord
     canvas_user_id: :user_id,
     submitted_at: :submitted_at,
     graded_at: :graded_at,
+    cached_due_date: :due_at,
     score: :score,
     excused: :excused,
     workflow_state: :workflow_state,

data/lib/canvas_sync/generators/templates/models/term.rb

@@ -1,7 +1,8 @@
 # <%= autogenerated_model_warning %>
 
 class Term < ApplicationRecord
-  include CanvasSync::ApiSyncable
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
 
   validates :canvas_id, uniqueness: true, presence: true
   has_many :courses, foreign_key: :canvas_term_id, primary_key: :canvas_id

data/lib/canvas_sync/generators/templates/models/user.rb

@@ -1,13 +1,16 @@
 # <%= autogenerated_model_warning %>
 
 class User < ApplicationRecord
-  include CanvasSync::ApiSyncable
+  include CanvasSync::Record
+  include CanvasSync::Concerns::ApiSyncable
 
   validates :canvas_id, uniqueness: true, presence: true
+  has_many :pseudonyms, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :enrollments, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :admins, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :admin_roles, through: :admins, source: :role
   has_many :submissions, primary_key: :canvas_id, foreign_key: :canvas_user_id
+  has_many :group_memberships, primary_key: :canvas_id, foreign_key: :canvas_user_id
 
   api_syncable({
     sis_id: :sis_user_id,

data/lib/canvas_sync/importers/bulk_importer.rb

@@ -17,7 +17,13 @@ module CanvasSync
       #        Note: passing the key [:on_duplicate_key_ignore] will override the default behavior of [:on_duplicate_key_update]
       # @yieldparam [Array] row if a block is passed in it will yield the current row from the CSV.
       #   This can be used if you need to filter or massage the data in any way.
-      def self.import(report_file_path, mapping, klass, conflict_target, import_args: {}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
+      def self.import(report_file_path, mapping, klass, conflict_target, import_args: {}, &blk) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/LineLength
+        ClassCallbackExecutor.run_if_defined(klass, :sync_import) do
+          perform_in_batches(report_file_path, mapping, klass, conflict_target, import_args: import_args, &blk)
+        end
+      end
+
+      def self.perform_in_batches(report_file_path, mapping, klass, conflict_target, import_args: {})
         csv_column_names = mapping.keys
         database_column_names = mapping.values.map { |value| value[:database_column_name] }
         rows = []

data/lib/canvas_sync/importers/legacy_importer.rb

@@ -8,8 +8,10 @@ module CanvasSync
       # @param report_file_path [String]
       # @param klass [Object]
       def self.import(report_file_path, klass, account_id, options)
-        CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
-          klass.create_or_update_from_csv(row, account_id, options)
+        ClassCallbackExecutor.run_if_defined(klass, :sync_import) do
+          CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
+            klass.create_or_update_from_csv(row, account_id, options)
+          end
         end
       end
     end

data/lib/canvas_sync/job.rb

@@ -61,7 +61,9 @@ module CanvasSync
       if model.respond_to? :create_or_update
         model.create_or_update(params)
       elsif model.method_defined? :update_from_api_params!
-        model.find_or_initialize_by(canvas_id: params['id']).update_from_api_params!(params)
+        instance = model.find_or_initialize_by(canvas_id: params['id'])
+        instance.update_from_api_params!(params)
+        instance
       else
         raise "Could not create/update #{model.name}. It must have a create_or_update(params) ClassMethod or implement ApiSyncable."
       end

data/lib/canvas_sync/job_chain.rb

@@ -0,0 +1,57 @@
+module CanvasSync
+  class JobChain
+    attr_reader :chain_data
+
+    delegate_missing_to :chain_data
+
+    VALID_PLACEMENT_PARAMETERS = %i[before after].freeze
+
+    def initialize(chain_data)
+      @chain_data = chain_data
+    end
+
+    def jobs
+      chain_data[:jobs]
+    end
+
+    def global_options
+      chain_data[:global_options]
+    end
+
+    def merge_options(job, options)
+      jobs.each do |j|
+        j[:options].deep_merge!(options) if job_matches_pattern(j, job)
+      end
+    end
+
+    def insert(new_job, **kwargs)
+      invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
+      raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
+      raise "Exactly one placement parameter may be provided" if kwargs.values.compact!.length > 1
+
+      if !kwargs.present?
+        jobs << new_job
+      else
+        placement = kwargs.keys[0]
+        relative_to = kwargs.values[0]
+
+        index = jobs.index { |job| job_matches_pattern(job, relative_to) }
+        raise "Could not find a \"#{relative_to}\" job in the chain" if index.nil?
+
+        index += 1 if placement == :after
+        new_job[:job] = new_job[:job].to_s
+        jobs.insert(index, new_job)
+      end
+    end
+
+    def process!(extra_options: {})
+      CanvasSync::invoke_next(self, extra_options: extra_options)
+    end
+
+    private
+
+    def job_matches_pattern(job_entry, pattern)
+      job_entry[:job].to_s == pattern.to_s
+    end
+  end
+end

data/lib/canvas_sync/jobs/{sync_users_job.rb → sync_accounts_job.rb}

@@ -1,24 +1,29 @@
 module CanvasSync
   module Jobs
-    class SyncUsersJob < ReportStarter
-      # Starts a provisioning report for just users.
+    class SyncAccountsJob < ReportStarter
+      # Starts a provisioning report for just accounts.
       #
-      # Provisioning reports do not scope users by term, so when we are
+      # Provisioning reports do not scope accounts by term, so when we are
       # running provisioning by term we sync users first so we don't duplicate
-      # the work of syncing all users for each term.
+      # the work of syncing all accounts for each term.
       #
       # @param job_chain [Hash]
       # @param options [Hash]
       def perform(job_chain, options)
+        unless options[:root_account] == false
+          acc_params = CanvasSync.get_canvas_sync_client(job_chain[:global_options]).account("self")
+          update_or_create_model(Account, acc_params)
+        end
+
         super(
           job_chain,
           "proservices_provisioning_csv",
           merge_report_params(job_chain, options, {
-            users: true,
+            accounts: true,
             include_deleted: true,
           }, term_scope: false),
           CanvasSync::Processors::ProvisioningReportProcessor.to_s,
-          { models: ["users"] },
+          { models: ["accounts"] },
         )
       end
     end

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -49,6 +49,8 @@ module CanvasSync
         }
 
         options[:models].each do |model|
+          # group_membership is the only model param that is singular :(
+          model = 'group_membership' if model == 'group_memberships'
           params[model] = true
         end
 

data/lib/canvas_sync/processors/model_mappings.yml

@@ -29,6 +29,25 @@ users:
       database_column_name: login_id
       type: string
 
+pseudonyms:
+  conflict_target: pseudonym_id
+  report_columns:
+    pseudonym_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_user_id:
+      database_column_name: canvas_user_id
+      type: integer
+    sis_user_id:
+      database_column_name: sis_id
+      type: string
+    unique_id:
+      database_column_name: unique_id
+      type: string
+    status:
+      database_column_name: workflow_state
+      type: string
+
 accounts:
   conflict_target: canvas_account_id
   report_columns:
@@ -230,6 +249,9 @@ submissions:
     submission_date:
       database_column_name: submitted_at
       type: datetime
+    due_at:
+      database_column_name: due_at
+      type: datetime
     graded_date:
       database_column_name: graded_at
       type: datetime
@@ -326,3 +348,62 @@ context_module_items:
     assignment_id:
       database_column_name: canvas_assignment_id
       type: integer
+
+groups:
+  conflict_target: canvas_group_id
+  report_columns:
+    canvas_group_id:
+      database_column_name: canvas_id
+      type: integer
+    group_id:
+      database_column_name: sis_id
+      type: string
+    canvas_group_category_id:
+      database_column_name: canvas_group_category_id
+      type: integer
+    group_category_id:
+      database_column_name: group_category_sis_id
+      type: integer
+    canvas_account_id:
+      database_column_name: canvas_account_id
+      type: integer
+    canvas_course_id:
+      database_column_name: canvas_course_id
+      type: integer
+    name:
+      database_column_name: name
+      type: string
+    status:
+      database_column_name: workflow_state
+      type: string
+    context_id:
+      database_column_name: context_id
+      type: integer
+    context_type:
+      database_column_name: context_type
+      type: string
+    max_membership:
+      database_column_name: max_membership
+      type: integer
+
+group_memberships:
+  conflict_target: canvas_group_membership_id
+  report_columns:
+    canvas_group_membership_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_group_id:
+      database_column_name: canvas_group_id
+      type: integer
+    group_id:
+      database_column_name: group_sis_id
+      type: string
+    canvas_user_id:
+      database_column_name: canvas_user_id
+      type: integer
+    user_id:
+      database_column_name: user_sis_id
+      type: string
+    status:
+      database_column_name: workflow_state
+      type: string

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -78,6 +78,15 @@ module CanvasSync
         )
       end
 
+      def bulk_process_pseudonyms(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          mapping[:pseudonyms][:report_columns],
+          Pseudonym,
+          mapping[:pseudonyms][:conflict_target].to_sym,
+        )
+      end
+
       def bulk_process_accounts(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,
@@ -122,6 +131,25 @@ module CanvasSync
           mapping[:xlist][:conflict_target].to_sym,
         )
       end
+
+      def bulk_process_groups(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          mapping[:groups][:report_columns],
+          Group,
+          mapping[:groups][:conflict_target].to_sym,
+        )
+      end
+
+      # Note that group membership is singular because we override the model name param in sync_provisioning_report_job
+      def bulk_process_group_membership(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          mapping[:group_memberships][:report_columns],
+          GroupMembership,
+          mapping[:group_memberships][:conflict_target].to_sym,
+        )
+      end
     end
   end
 end

data/lib/canvas_sync/record.rb

@@ -0,0 +1,9 @@
+module CanvasSync
+  module Record
+    extend ActiveSupport::Concern
+
+    included do
+      define_model_callbacks :sync_import
+    end
+  end
+end

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.12.0".freeze
+  VERSION = "0.16.1".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 RSpec.describe CanvasSync do
   describe '.provisioning_sync' do
     it 'invokes the first job in the queue and passes on the rest of the job chain' do
-      expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], nil)
+      expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], nil).chain_data
       first_job = expected_job_chain[:jobs].shift
 
       expect(CanvasSync::Jobs::SyncTermsJob).to receive(:perform_later)
@@ -31,10 +31,10 @@ RSpec.describe CanvasSync do
           provisioning: { c: 3 },
           global: { d: 4 },
         })
-        expect(chain).to eq({
+        expect(chain.chain_data).to eq({
           jobs: [
             { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: { a: 1 } },
-            { job: CanvasSync::Jobs::SyncUsersJob.to_s, options: { b: 2 } },
+            { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'], b: 2 } },
             { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'], c: 3 } }
           ],
           global_options: { legacy_support: false, d: 4 }
@@ -44,10 +44,10 @@ RSpec.describe CanvasSync do
       context 'we are syncing users with a term scope' do
         it 'syncs the users in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], :active)
-          expect(chain).to eq({
+          expect(chain.chain_data).to eq({
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
-              { job: CanvasSync::Jobs::SyncUsersJob.to_s, options: {} },
+              { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'] } },
               { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
             ],
             global_options: { legacy_support: false }
@@ -58,7 +58,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing users without a term scope' do
         it 'syncs users along with the rest of the provisioning report' do
           chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'])
-          expect(chain).to eq({
+          expect(chain.chain_data).to eq({
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: ['users', 'courses'] } }
@@ -71,7 +71,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing roles with a term scope' do
         it 'syncs the roles in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], :active)
-          expect(chain).to eq({
+          expect(chain.chain_data).to eq({
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncRolesJob.to_s, options: {} },
@@ -85,7 +85,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing roles without a term scope' do
         it 'syncs roles separately even with no term scope' do
           chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'])
-          expect(chain).to eq({
+          expect(chain.chain_data).to eq({
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncRolesJob.to_s, options: {} },
@@ -99,7 +99,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing admins with a term scope' do
         it 'syncs the admins in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], :active)
-          expect(chain).to eq({
+          expect(chain.chain_data).to eq({
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncAdminsJob.to_s, options: {} },
@@ -113,7 +113,7 @@ RSpec.describe CanvasSync do
       context 'we are syncing admins without a term scope' do
         it 'syncs admins separately even with no term scope' do
           chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'])
-          expect(chain).to eq({
+          expect(chain.chain_data).to eq({
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncAdminsJob.to_s, options: {} },
@@ -128,7 +128,7 @@ RSpec.describe CanvasSync do
         it "appends the SyncAssignmentsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignments])
 
-          expect(chain).to eq(
+          expect(chain.chain_data).to eq(
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: %w[users enrollments] } },
@@ -143,7 +143,7 @@ RSpec.describe CanvasSync do
         it "appends the SyncSubmissionsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments submissions])
 
-          expect(chain).to eq(
+          expect(chain.chain_data).to eq(
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: %w[users enrollments] } },
@@ -158,7 +158,7 @@ RSpec.describe CanvasSync do
         it "appends the SyncAssignmentGroupsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignment_groups])
 
-          expect(chain).to eq(
+          expect(chain.chain_data).to eq(
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
               { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: nil, models: %w[users enrollments] } },
@@ -185,7 +185,7 @@ RSpec.describe CanvasSync do
             }
           }
         ]
-      )
+      ).chain_data
       first_job = expected_job_chain[:jobs].shift
 
       expect(CanvasSync::Jobs::SyncSimpleTableJob).to receive(:perform_later)
@@ -242,9 +242,12 @@ RSpec.describe CanvasSync do
         global_options: {}
       }
 
-      expect(chain).to eq(expected_job_chain)
+      expect(chain.chain_data).to eq(expected_job_chain)
 
     end
   end
-  
+
+  describe ".sync_scope" do
+
+  end
 end