canvas_sync 0.14.0 → 0.16.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6dc939cb0559d835965e4b61416fa79f19fdb0a4
-  data.tar.gz: 9e075f746f42ae230ed6219c8e16b00b77526748
+  metadata.gz: 8a38d324ba9fe5a06ad6cd7f9bc4fa3686ad8389
+  data.tar.gz: 9f67e4ad9dd7dc8be237871dd2224ee19a6e6e48
 SHA512:
-  metadata.gz: c4acb86d39b2aaf09ee9de7ae33d689bc28570babd3086098634fa85e0cca9f8144b22193e3b0d1e21c659e31c807e00899b670d0e52f8549e214b899aae17e2
-  data.tar.gz: a18ac0d166e5e61b24f0a00a0284db3c73ea9bf5ea642f4ac11ccbab0a7b436222d68e4930ed2c3f75d517fe0ef9ebb1bb4752019ea29c988f9ab27e92388f70
+  metadata.gz: 500266ca331f28db32e08c8e5d4f780f9e303299f07c3497f22a43236d50901b1f482dd08cd593338f11760aeb91693a1b43880809b024fb7ff6fe0bffe6fcfd
+  data.tar.gz: 18add737517d8ee376c4905b953dfdd3519cefebd7787530ed46f8c33fd78f58c21794c8853a8bc3d392a85aecadca536a29ca4479b4fda569a256fcc9a43d6f

data/lib/canvas_sync.rb

@@ -22,6 +22,7 @@ Dir[File.dirname(__FILE__) +  "/canvas_sync/concerns/**/*.rb"].each { |file| req
 module CanvasSync
   SUPPORTED_MODELS = %w[
     users
+    pseudonyms
     courses
     groups
     group_memberships
@@ -131,7 +132,7 @@ module CanvasSync
       duped_job_chain[:global_options][:fork_path] ||= []
       duped_job_chain[:global_options][:fork_keys] ||= []
       duped_job_chain[:global_options][:fork_path] << job_log.job_id
-      duped_job_chain[:global_options][:fork_keys] << ['canvas_term_id']
+      duped_job_chain[:global_options][:fork_keys] << keys
       duped_job_chain[:global_options][:on_failure] ||= 'CanvasSync::Jobs::ForkGather.handle_branch_error'
       sub_items = yield duped_job_chain
       sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
@@ -145,6 +146,10 @@ module CanvasSync
       terms.each do |t|
         return scope.send(t) if scope.respond_to?(t)
       end
+      model = scope.try(:model) || scope
+      if model.try(:column_names)&.include?(:workflow_state)
+        return scope.where.not(workflow_state: %w[deleted])
+      end
       Rails.logger.warn("Could not filter Syncable Scope for model '#{scope.try(:model)&.name || scope.name}'")
       scope
     end
@@ -195,7 +200,6 @@ module CanvasSync
       model_job_map = {
         terms: CanvasSync::Jobs::SyncTermsJob,
         accounts: CanvasSync::Jobs::SyncAccountsJob,
-        users: CanvasSync::Jobs::SyncUsersJob,
         roles: CanvasSync::Jobs::SyncRolesJob,
         admins: CanvasSync::Jobs::SyncAdminsJob,
 
@@ -223,35 +227,41 @@ module CanvasSync
 
       # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term
       try_add_model_job.call('accounts')
-      try_add_model_job.call('users') if term_scope.present?
+
+      # These Models use the provisioning report, but are not term-scoped,
+      # so we sync them before to ensure work is not duplicated
+      if term_scope.present?
+        models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
+        jobs.concat(
+          generate_provisioning_jobs(first_provisioning_models, options)
+        )
+      end
+
       try_add_model_job.call('roles')
       try_add_model_job.call('admins')
-
       pre_provisioning_jobs = jobs
-      jobs = []
 
       ###############################
       # Post provisioning report jobs
       ###############################
 
+      jobs = []
       try_add_model_job.call('assignments')
       try_add_model_job.call('submissions')
       try_add_model_job.call('assignment_groups')
       try_add_model_job.call('context_modules')
       try_add_model_job.call('context_module_items')
-
       post_provisioning_jobs = jobs
 
-      jobs = pre_provisioning_jobs
-      if models.present?
-        provisioning_job = {
-          job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s,
-          options: { term_scope: term_scope, models: models },
-        }
-        provisioning_job[:options].merge!(options[:provisioning]) if options[:provisioning].present?
-        jobs += Array.wrap(provisioning_job)
-      end
-      jobs += post_provisioning_jobs
+      ###############################
+      # Main provisioning job and queueing
+      ###############################
+
+      jobs = [
+        *pre_provisioning_jobs,
+        *generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
+        *post_provisioning_jobs,
+      ]
 
       global_options = { legacy_support: legacy_support }
       global_options[:account_id] = account_id if account_id.present?
@@ -260,6 +270,48 @@ module CanvasSync
       JobChain.new(jobs: jobs, global_options: global_options)
     end
 
+    def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
+      dup_models = [ *model_list ]
+      unique_option_models = {}
+
+      filtered_models = only_split ? (only_split & model_list) : model_list
+      filtered_models.each do |m|
+        mopts = options_hash[m.to_sym] || options_hash[default_key]
+        unique_option_models[mopts] ||= []
+        unique_option_models[mopts] << m
+        dup_models.delete(m)
+      end
+
+      if dup_models.present?
+        mopts = options_hash[default_key]
+        unique_option_models[mopts] ||= []
+        unique_option_models[mopts].concat(dup_models)
+      end
+
+      unique_option_models
+    end
+
+    def generate_provisioning_jobs(model_list, options_hash, job_options: {}, only_split: nil, default_key: :provisioning)
+      # Group the model options as best we can.
+      # This is mainly for backwards compatibility, since 'users' was previously it's own job
+      unique_option_models = group_by_job_options(
+        model_list,
+        options_hash,
+        only_split: only_split,
+        default_key: default_key,
+      )
+
+      unique_option_models.map do |mopts, models|
+        opts = { models: models }
+        opts.merge!(job_options)
+        opts.merge!(mopts) if mopts.present?
+        {
+          job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s,
+          options: opts,
+        }
+      end
+    end
+
     # Calls the canvas_sync_client in your app. If you have specified an account
     # ID when starting the job it will pass the account ID to your canvas_sync_client method.
     #

data/lib/canvas_sync/concerns/api_syncable.rb

@@ -146,7 +146,7 @@ module CanvasSync::Concerns
     # Apply a response Hash from the API to this model's attributes and save if changed?
     # @param [Hash] api_params API-format Hash
     # @return [self] self
-    def update_from_api_params(api_params, **kwargs)
+    def update_from_api_params(*args)
       assign_from_api_params(*args)
       save if changed?
     end

data/lib/canvas_sync/concerns/legacy_columns.rb

@@ -15,13 +15,14 @@ module CanvasSync::Concerns
       private
 
       def legacy_column_apply(cls)
-        cid_column = "canvas_#{subclass.name.downcase}_id"
-        column_names = subclass.columns.map(&:name)
+        return if cls.abstract_class
+        cid_column = "canvas_#{cls.name.downcase}_id"
+        column_names = cls.columns.map(&:name)
         return if column_names.include?('canvas_id') && column_names.include?(cid_column)
         if column_names.include?('canvas_id')
-          subclass.alias_attribute(cid_column.to_sym, :canvas_id)
+          cls.alias_attribute(cid_column.to_sym, :canvas_id)
         elsif column_names.include?(cid_column)
-          subclass.alias_attribute(:canvas_id, cid_column.to_sym)
+          cls.alias_attribute(:canvas_id, cid_column.to_sym)
         end
       rescue ActiveRecord::StatementInvalid
       end

data/lib/canvas_sync/generators/templates/migrations/create_pseudonyms.rb

@@ -0,0 +1,18 @@
+# <%= autogenerated_migration_warning %>
+
+class CreatePseudonyms < ActiveRecord::Migration[5.1]
+  def change
+    create_table :pseudonyms do |t|
+      t.bigint :canvas_id, null: false
+      t.bigint :canvas_user_id
+      t.string :sis_id
+      t.string :unique_id
+      t.string :workflow_state
+
+      t.timestamps
+    end
+
+    add_index :pseudonyms, :canvas_id, unique: true
+    add_index :pseudonyms, :canvas_user_id
+  end
+end

data/lib/canvas_sync/generators/templates/migrations/create_submissions.rb

@@ -8,6 +8,7 @@ class CreateSubmissions < ActiveRecord::Migration[5.1]
       t.bigint :canvas_assignment_id
       t.bigint :canvas_user_id
       t.datetime :submitted_at
+      t.datetime :due_at
       t.datetime :graded_at
       t.float :score
       t.float :points_possible

data/lib/canvas_sync/generators/templates/models/account.rb

@@ -14,6 +14,9 @@ class Account < ApplicationRecord
                           primary_key: :canvas_id, foreign_key: :canvas_parent_account_id
   has_many :groups, primary_key: :canvas_id, foreign_key: :canvas_account_id
 
+  scope :active, -> { where.not(workflow_state: 'deleted') }
+  # scope :should_canvas_sync, -> { active } # Optional - uses .active if not given
+
   api_syncable({
     name: :name,
     workflow_state: :workflow_state,

data/lib/canvas_sync/generators/templates/models/pseudonym.rb

@@ -0,0 +1,8 @@
+# <%= autogenerated_model_warning %>
+
+class Pseudonym < ApplicationRecord
+  include CanvasSync::Record
+
+  validates :canvas_id, uniqueness: true, presence: true
+  belongs_to :user, primary_key: :canvas_id, foreign_key: :canvas_user_id
+end

data/lib/canvas_sync/generators/templates/models/submission.rb

@@ -14,6 +14,7 @@ class Submission < ApplicationRecord
     canvas_user_id: :user_id,
     submitted_at: :submitted_at,
     graded_at: :graded_at,
+    cached_due_date: :due_at,
     score: :score,
     excused: :excused,
     workflow_state: :workflow_state,

data/lib/canvas_sync/generators/templates/models/user.rb

@@ -5,6 +5,7 @@ class User < ApplicationRecord
   include CanvasSync::Concerns::ApiSyncable
 
   validates :canvas_id, uniqueness: true, presence: true
+  has_many :pseudonyms, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :enrollments, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :admins, primary_key: :canvas_id, foreign_key: :canvas_user_id
   has_many :admin_roles, through: :admins, source: :role

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -31,7 +31,7 @@ module CanvasSync
         end
       end
 
-      private
+      protected
 
       def start_report(report_params, job_chain, options)
         CanvasSync::Jobs::ReportStarter.perform_later(

data/lib/canvas_sync/processors/model_mappings.yml

@@ -29,6 +29,25 @@ users:
       database_column_name: login_id
       type: string
 
+pseudonyms:
+  conflict_target: pseudonym_id
+  report_columns:
+    pseudonym_id:
+      database_column_name: canvas_id
+      type: integer
+    canvas_user_id:
+      database_column_name: canvas_user_id
+      type: integer
+    sis_user_id:
+      database_column_name: sis_id
+      type: string
+    unique_id:
+      database_column_name: unique_id
+      type: string
+    status:
+      database_column_name: workflow_state
+      type: string
+
 accounts:
   conflict_target: canvas_account_id
   report_columns:
@@ -230,6 +249,9 @@ submissions:
     submission_date:
       database_column_name: submitted_at
       type: datetime
+    due_at:
+      database_column_name: due_at
+      type: datetime
     graded_date:
       database_column_name: graded_at
       type: datetime

data/lib/canvas_sync/processors/provisioning_report_processor.rb

@@ -78,6 +78,15 @@ module CanvasSync
         )
       end
 
+      def bulk_process_pseudonyms(report_file_path)
+        CanvasSync::Importers::BulkImporter.import(
+          report_file_path,
+          mapping[:pseudonyms][:report_columns],
+          Pseudonym,
+          mapping[:pseudonyms][:conflict_target].to_sym,
+        )
+      end
+
       def bulk_process_accounts(report_file_path)
         CanvasSync::Importers::BulkImporter.import(
           report_file_path,

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.14.0".freeze
+  VERSION = "0.16.3".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -34,7 +34,7 @@ RSpec.describe CanvasSync do
         expect(chain.chain_data).to eq({
           jobs: [
             { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: { a: 1 } },
-            { job: CanvasSync::Jobs::SyncUsersJob.to_s, options: { b: 2 } },
+            { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'], b: 2 } },
             { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'], c: 3 } }
           ],
           global_options: { legacy_support: false, d: 4 }
@@ -47,7 +47,7 @@ RSpec.describe CanvasSync do
           expect(chain.chain_data).to eq({
             jobs: [
               { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
-              { job: CanvasSync::Jobs::SyncUsersJob.to_s, options: {} },
+              { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'] } },
               { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
             ],
             global_options: { legacy_support: false }
@@ -247,4 +247,7 @@ RSpec.describe CanvasSync do
     end
   end
 
+  describe ".sync_scope" do
+
+  end
 end

data/spec/dummy/db/migrate/20190702203627_create_submissions.rb

@@ -14,6 +14,7 @@ class CreateSubmissions < ActiveRecord::Migration[5.1]
       t.bigint :canvas_assignment_id
       t.bigint :canvas_user_id
       t.datetime :submitted_at
+      t.datetime :due_at
       t.datetime :graded_at
       t.float :score
       t.float :points_possible

data/spec/dummy/db/schema.rb

@@ -225,6 +225,7 @@ ActiveRecord::Schema.define(version: 2020_04_16_214248) do
     t.bigint "canvas_assignment_id"
     t.bigint "canvas_user_id"
     t.datetime "submitted_at"
+    t.datetime "due_at"
     t.datetime "graded_at"
     t.float "score"
     t.float "points_possible"

data/spec/support/fixtures/reports/submissions.csv

@@ -1,3 +1,3 @@
-"canvas user id","sis user id","user name","canvas course id","sis course id","course name","assignment id","assignment name","submission date","graded date","score","points possible","submission id","workflow state","excused"
-1,1,userName,1,1,courseName,1,assignmentName,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,1,graded,true
-1,1,userName,1,1,courseName,2,assignment2Name,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,2,submitted,false
+"canvas user id","sis user id","user name","canvas course id","sis course id","course name","assignment id","assignment name","submission date","graded date","score","points possible","submission id","workflow state","excused","due at"
+1,1,userName,1,1,courseName,1,assignmentName,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,1,graded,true,2017-09-27 18:28:15 UTC
+1,1,userName,1,1,courseName,2,assignment2Name,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,2,submitted,false,2017-09-28 18:28:15 UTC

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: canvas_sync
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.16.3
 platform: ruby
 authors:
 - Nate Collings
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-21 00:00:00.000000000 Z
+date: 2020-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -340,6 +340,7 @@ files:
 - lib/canvas_sync/generators/templates/migrations/create_enrollments.rb
 - lib/canvas_sync/generators/templates/migrations/create_group_memberships.rb
 - lib/canvas_sync/generators/templates/migrations/create_groups.rb
+- lib/canvas_sync/generators/templates/migrations/create_pseudonyms.rb
 - lib/canvas_sync/generators/templates/migrations/create_roles.rb
 - lib/canvas_sync/generators/templates/migrations/create_sections.rb
 - lib/canvas_sync/generators/templates/migrations/create_submissions.rb
@@ -355,6 +356,7 @@ files:
 - lib/canvas_sync/generators/templates/models/enrollment.rb
 - lib/canvas_sync/generators/templates/models/group.rb
 - lib/canvas_sync/generators/templates/models/group_membership.rb
+- lib/canvas_sync/generators/templates/models/pseudonym.rb
 - lib/canvas_sync/generators/templates/models/role.rb
 - lib/canvas_sync/generators/templates/models/section.rb
 - lib/canvas_sync/generators/templates/models/submission.rb
@@ -391,7 +393,6 @@ files:
 - lib/canvas_sync/jobs/sync_simple_table_job.rb
 - lib/canvas_sync/jobs/sync_submissions_job.rb
 - lib/canvas_sync/jobs/sync_terms_job.rb
-- lib/canvas_sync/jobs/sync_users_job.rb
 - lib/canvas_sync/processors/assignment_groups_processor.rb
 - lib/canvas_sync/processors/assignments_processor.rb
 - lib/canvas_sync/processors/context_module_items_processor.rb
@@ -420,7 +421,6 @@ files:
 - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
 - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
 - spec/canvas_sync/jobs/sync_terms_job_spec.rb
-- spec/canvas_sync/jobs/sync_users_job_spec.rb
 - spec/canvas_sync/models/accounts_spec.rb
 - spec/canvas_sync/models/admins_spec.rb
 - spec/canvas_sync/models/assignment_group_spec.rb
@@ -596,7 +596,6 @@ test_files:
 - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
 - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
 - spec/canvas_sync/jobs/sync_terms_job_spec.rb
-- spec/canvas_sync/jobs/sync_users_job_spec.rb
 - spec/canvas_sync/models/accounts_spec.rb
 - spec/canvas_sync/models/admins_spec.rb
 - spec/canvas_sync/models/assignment_group_spec.rb

data/lib/canvas_sync/jobs/sync_users_job.rb

@@ -1,26 +0,0 @@
-module CanvasSync
-  module Jobs
-    class SyncUsersJob < ReportStarter
-      # Starts a provisioning report for just users.
-      #
-      # Provisioning reports do not scope users by term, so when we are
-      # running provisioning by term we sync users first so we don't duplicate
-      # the work of syncing all users for each term.
-      #
-      # @param job_chain [Hash]
-      # @param options [Hash]
-      def perform(job_chain, options)
-        super(
-          job_chain,
-          "proservices_provisioning_csv",
-          merge_report_params(job_chain, options, {
-            users: true,
-            include_deleted: true,
-          }, term_scope: false),
-          CanvasSync::Processors::ProvisioningReportProcessor.to_s,
-          { models: ["users"] },
-        )
-      end
-    end
-  end
-end

data/spec/canvas_sync/jobs/sync_users_job_spec.rb

@@ -1,15 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe CanvasSync::Jobs::SyncUsersJob do
-  describe '#perform' do
-    it 'enqueues a ReportStarter for a provisioning report across all terms with just users' do
-      expect_any_instance_of(Bearcat::Client).to receive(:start_report)
-        .with('self', 'proservices_provisioning_csv', { parameters: { users: true, include_deleted: true } })
-        .and_return({ 'id' => 1 })
-
-      expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
-
-      CanvasSync::Jobs::SyncUsersJob.perform_now({ jobs: [], global_options: {}}, {})
-    end
-  end
-end