canvas_sync 0.14.0 → 0.16.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6dc939cb0559d835965e4b61416fa79f19fdb0a4
4
- data.tar.gz: 9e075f746f42ae230ed6219c8e16b00b77526748
3
+ metadata.gz: 8a38d324ba9fe5a06ad6cd7f9bc4fa3686ad8389
4
+ data.tar.gz: 9f67e4ad9dd7dc8be237871dd2224ee19a6e6e48
5
5
  SHA512:
6
- metadata.gz: c4acb86d39b2aaf09ee9de7ae33d689bc28570babd3086098634fa85e0cca9f8144b22193e3b0d1e21c659e31c807e00899b670d0e52f8549e214b899aae17e2
7
- data.tar.gz: a18ac0d166e5e61b24f0a00a0284db3c73ea9bf5ea642f4ac11ccbab0a7b436222d68e4930ed2c3f75d517fe0ef9ebb1bb4752019ea29c988f9ab27e92388f70
6
+ metadata.gz: 500266ca331f28db32e08c8e5d4f780f9e303299f07c3497f22a43236d50901b1f482dd08cd593338f11760aeb91693a1b43880809b024fb7ff6fe0bffe6fcfd
7
+ data.tar.gz: 18add737517d8ee376c4905b953dfdd3519cefebd7787530ed46f8c33fd78f58c21794c8853a8bc3d392a85aecadca536a29ca4479b4fda569a256fcc9a43d6f
@@ -22,6 +22,7 @@ Dir[File.dirname(__FILE__) + "/canvas_sync/concerns/**/*.rb"].each { |file| req
22
22
  module CanvasSync
23
23
  SUPPORTED_MODELS = %w[
24
24
  users
25
+ pseudonyms
25
26
  courses
26
27
  groups
27
28
  group_memberships
@@ -131,7 +132,7 @@ module CanvasSync
131
132
  duped_job_chain[:global_options][:fork_path] ||= []
132
133
  duped_job_chain[:global_options][:fork_keys] ||= []
133
134
  duped_job_chain[:global_options][:fork_path] << job_log.job_id
134
- duped_job_chain[:global_options][:fork_keys] << ['canvas_term_id']
135
+ duped_job_chain[:global_options][:fork_keys] << keys
135
136
  duped_job_chain[:global_options][:on_failure] ||= 'CanvasSync::Jobs::ForkGather.handle_branch_error'
136
137
  sub_items = yield duped_job_chain
137
138
  sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
@@ -145,6 +146,10 @@ module CanvasSync
145
146
  terms.each do |t|
146
147
  return scope.send(t) if scope.respond_to?(t)
147
148
  end
149
+ model = scope.try(:model) || scope
150
+ if model.try(:column_names)&.include?(:workflow_state)
151
+ return scope.where.not(workflow_state: %w[deleted])
152
+ end
148
153
  Rails.logger.warn("Could not filter Syncable Scope for model '#{scope.try(:model)&.name || scope.name}'")
149
154
  scope
150
155
  end
@@ -195,7 +200,6 @@ module CanvasSync
195
200
  model_job_map = {
196
201
  terms: CanvasSync::Jobs::SyncTermsJob,
197
202
  accounts: CanvasSync::Jobs::SyncAccountsJob,
198
- users: CanvasSync::Jobs::SyncUsersJob,
199
203
  roles: CanvasSync::Jobs::SyncRolesJob,
200
204
  admins: CanvasSync::Jobs::SyncAdminsJob,
201
205
 
@@ -223,35 +227,41 @@ module CanvasSync
223
227
 
224
228
  # Accounts, users, roles, and admins are synced before provisioning because they cannot be scoped to term
225
229
  try_add_model_job.call('accounts')
226
- try_add_model_job.call('users') if term_scope.present?
230
+
231
+ # These Models use the provisioning report, but are not term-scoped,
232
+ # so we sync them before to ensure work is not duplicated
233
+ if term_scope.present?
234
+ models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
235
+ jobs.concat(
236
+ generate_provisioning_jobs(first_provisioning_models, options)
237
+ )
238
+ end
239
+
227
240
  try_add_model_job.call('roles')
228
241
  try_add_model_job.call('admins')
229
-
230
242
  pre_provisioning_jobs = jobs
231
- jobs = []
232
243
 
233
244
  ###############################
234
245
  # Post provisioning report jobs
235
246
  ###############################
236
247
 
248
+ jobs = []
237
249
  try_add_model_job.call('assignments')
238
250
  try_add_model_job.call('submissions')
239
251
  try_add_model_job.call('assignment_groups')
240
252
  try_add_model_job.call('context_modules')
241
253
  try_add_model_job.call('context_module_items')
242
-
243
254
  post_provisioning_jobs = jobs
244
255
 
245
- jobs = pre_provisioning_jobs
246
- if models.present?
247
- provisioning_job = {
248
- job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s,
249
- options: { term_scope: term_scope, models: models },
250
- }
251
- provisioning_job[:options].merge!(options[:provisioning]) if options[:provisioning].present?
252
- jobs += Array.wrap(provisioning_job)
253
- end
254
- jobs += post_provisioning_jobs
256
+ ###############################
257
+ # Main provisioning job and queueing
258
+ ###############################
259
+
260
+ jobs = [
261
+ *pre_provisioning_jobs,
262
+ *generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users']),
263
+ *post_provisioning_jobs,
264
+ ]
255
265
 
256
266
  global_options = { legacy_support: legacy_support }
257
267
  global_options[:account_id] = account_id if account_id.present?
@@ -260,6 +270,48 @@ module CanvasSync
260
270
  JobChain.new(jobs: jobs, global_options: global_options)
261
271
  end
262
272
 
273
+ def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
274
+ dup_models = [ *model_list ]
275
+ unique_option_models = {}
276
+
277
+ filtered_models = only_split ? (only_split & model_list) : model_list
278
+ filtered_models.each do |m|
279
+ mopts = options_hash[m.to_sym] || options_hash[default_key]
280
+ unique_option_models[mopts] ||= []
281
+ unique_option_models[mopts] << m
282
+ dup_models.delete(m)
283
+ end
284
+
285
+ if dup_models.present?
286
+ mopts = options_hash[default_key]
287
+ unique_option_models[mopts] ||= []
288
+ unique_option_models[mopts].concat(dup_models)
289
+ end
290
+
291
+ unique_option_models
292
+ end
293
+
294
+ def generate_provisioning_jobs(model_list, options_hash, job_options: {}, only_split: nil, default_key: :provisioning)
295
+ # Group the model options as best we can.
296
+ # This is mainly for backwards compatibility, since 'users' was previously it's own job
297
+ unique_option_models = group_by_job_options(
298
+ model_list,
299
+ options_hash,
300
+ only_split: only_split,
301
+ default_key: default_key,
302
+ )
303
+
304
+ unique_option_models.map do |mopts, models|
305
+ opts = { models: models }
306
+ opts.merge!(job_options)
307
+ opts.merge!(mopts) if mopts.present?
308
+ {
309
+ job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s,
310
+ options: opts,
311
+ }
312
+ end
313
+ end
314
+
263
315
  # Calls the canvas_sync_client in your app. If you have specified an account
264
316
  # ID when starting the job it will pass the account ID to your canvas_sync_client method.
265
317
  #
@@ -146,7 +146,7 @@ module CanvasSync::Concerns
146
146
  # Apply a response Hash from the API to this model's attributes and save if changed?
147
147
  # @param [Hash] api_params API-format Hash
148
148
  # @return [self] self
149
- def update_from_api_params(api_params, **kwargs)
149
+ def update_from_api_params(*args)
150
150
  assign_from_api_params(*args)
151
151
  save if changed?
152
152
  end
@@ -15,13 +15,14 @@ module CanvasSync::Concerns
15
15
  private
16
16
 
17
17
  def legacy_column_apply(cls)
18
- cid_column = "canvas_#{subclass.name.downcase}_id"
19
- column_names = subclass.columns.map(&:name)
18
+ return if cls.abstract_class
19
+ cid_column = "canvas_#{cls.name.downcase}_id"
20
+ column_names = cls.columns.map(&:name)
20
21
  return if column_names.include?('canvas_id') && column_names.include?(cid_column)
21
22
  if column_names.include?('canvas_id')
22
- subclass.alias_attribute(cid_column.to_sym, :canvas_id)
23
+ cls.alias_attribute(cid_column.to_sym, :canvas_id)
23
24
  elsif column_names.include?(cid_column)
24
- subclass.alias_attribute(:canvas_id, cid_column.to_sym)
25
+ cls.alias_attribute(:canvas_id, cid_column.to_sym)
25
26
  end
26
27
  rescue ActiveRecord::StatementInvalid
27
28
  end
@@ -0,0 +1,18 @@
1
+ # <%= autogenerated_migration_warning %>
2
+
3
+ class CreatePseudonyms < ActiveRecord::Migration[5.1]
4
+ def change
5
+ create_table :pseudonyms do |t|
6
+ t.bigint :canvas_id, null: false
7
+ t.bigint :canvas_user_id
8
+ t.string :sis_id
9
+ t.string :unique_id
10
+ t.string :workflow_state
11
+
12
+ t.timestamps
13
+ end
14
+
15
+ add_index :pseudonyms, :canvas_id, unique: true
16
+ add_index :pseudonyms, :canvas_user_id
17
+ end
18
+ end
@@ -8,6 +8,7 @@ class CreateSubmissions < ActiveRecord::Migration[5.1]
8
8
  t.bigint :canvas_assignment_id
9
9
  t.bigint :canvas_user_id
10
10
  t.datetime :submitted_at
11
+ t.datetime :due_at
11
12
  t.datetime :graded_at
12
13
  t.float :score
13
14
  t.float :points_possible
@@ -14,6 +14,9 @@ class Account < ApplicationRecord
14
14
  primary_key: :canvas_id, foreign_key: :canvas_parent_account_id
15
15
  has_many :groups, primary_key: :canvas_id, foreign_key: :canvas_account_id
16
16
 
17
+ scope :active, -> { where.not(workflow_state: 'deleted') }
18
+ # scope :should_canvas_sync, -> { active } # Optional - uses .active if not given
19
+
17
20
  api_syncable({
18
21
  name: :name,
19
22
  workflow_state: :workflow_state,
@@ -0,0 +1,8 @@
1
+ # <%= autogenerated_model_warning %>
2
+
3
+ class Pseudonym < ApplicationRecord
4
+ include CanvasSync::Record
5
+
6
+ validates :canvas_id, uniqueness: true, presence: true
7
+ belongs_to :user, primary_key: :canvas_id, foreign_key: :canvas_user_id
8
+ end
@@ -14,6 +14,7 @@ class Submission < ApplicationRecord
14
14
  canvas_user_id: :user_id,
15
15
  submitted_at: :submitted_at,
16
16
  graded_at: :graded_at,
17
+ cached_due_date: :due_at,
17
18
  score: :score,
18
19
  excused: :excused,
19
20
  workflow_state: :workflow_state,
@@ -5,6 +5,7 @@ class User < ApplicationRecord
5
5
  include CanvasSync::Concerns::ApiSyncable
6
6
 
7
7
  validates :canvas_id, uniqueness: true, presence: true
8
+ has_many :pseudonyms, primary_key: :canvas_id, foreign_key: :canvas_user_id
8
9
  has_many :enrollments, primary_key: :canvas_id, foreign_key: :canvas_user_id
9
10
  has_many :admins, primary_key: :canvas_id, foreign_key: :canvas_user_id
10
11
  has_many :admin_roles, through: :admins, source: :role
@@ -31,7 +31,7 @@ module CanvasSync
31
31
  end
32
32
  end
33
33
 
34
- private
34
+ protected
35
35
 
36
36
  def start_report(report_params, job_chain, options)
37
37
  CanvasSync::Jobs::ReportStarter.perform_later(
@@ -29,6 +29,25 @@ users:
29
29
  database_column_name: login_id
30
30
  type: string
31
31
 
32
+ pseudonyms:
33
+ conflict_target: pseudonym_id
34
+ report_columns:
35
+ pseudonym_id:
36
+ database_column_name: canvas_id
37
+ type: integer
38
+ canvas_user_id:
39
+ database_column_name: canvas_user_id
40
+ type: integer
41
+ sis_user_id:
42
+ database_column_name: sis_id
43
+ type: string
44
+ unique_id:
45
+ database_column_name: unique_id
46
+ type: string
47
+ status:
48
+ database_column_name: workflow_state
49
+ type: string
50
+
32
51
  accounts:
33
52
  conflict_target: canvas_account_id
34
53
  report_columns:
@@ -230,6 +249,9 @@ submissions:
230
249
  submission_date:
231
250
  database_column_name: submitted_at
232
251
  type: datetime
252
+ due_at:
253
+ database_column_name: due_at
254
+ type: datetime
233
255
  graded_date:
234
256
  database_column_name: graded_at
235
257
  type: datetime
@@ -78,6 +78,15 @@ module CanvasSync
78
78
  )
79
79
  end
80
80
 
81
+ def bulk_process_pseudonyms(report_file_path)
82
+ CanvasSync::Importers::BulkImporter.import(
83
+ report_file_path,
84
+ mapping[:pseudonyms][:report_columns],
85
+ Pseudonym,
86
+ mapping[:pseudonyms][:conflict_target].to_sym,
87
+ )
88
+ end
89
+
81
90
  def bulk_process_accounts(report_file_path)
82
91
  CanvasSync::Importers::BulkImporter.import(
83
92
  report_file_path,
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.14.0".freeze
2
+ VERSION = "0.16.3".freeze
3
3
  end
@@ -34,7 +34,7 @@ RSpec.describe CanvasSync do
34
34
  expect(chain.chain_data).to eq({
35
35
  jobs: [
36
36
  { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: { a: 1 } },
37
- { job: CanvasSync::Jobs::SyncUsersJob.to_s, options: { b: 2 } },
37
+ { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'], b: 2 } },
38
38
  { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'], c: 3 } }
39
39
  ],
40
40
  global_options: { legacy_support: false, d: 4 }
@@ -47,7 +47,7 @@ RSpec.describe CanvasSync do
47
47
  expect(chain.chain_data).to eq({
48
48
  jobs: [
49
49
  { job: CanvasSync::Jobs::SyncTermsJob.to_s, options: {} },
50
- { job: CanvasSync::Jobs::SyncUsersJob.to_s, options: {} },
50
+ { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { models: ['users'] } },
51
51
  { job: CanvasSync::Jobs::SyncProvisioningReportJob.to_s, options: { term_scope: 'active', models: ['courses'] } }
52
52
  ],
53
53
  global_options: { legacy_support: false }
@@ -247,4 +247,7 @@ RSpec.describe CanvasSync do
247
247
  end
248
248
  end
249
249
 
250
+ describe ".sync_scope" do
251
+
252
+ end
250
253
  end
@@ -14,6 +14,7 @@ class CreateSubmissions < ActiveRecord::Migration[5.1]
14
14
  t.bigint :canvas_assignment_id
15
15
  t.bigint :canvas_user_id
16
16
  t.datetime :submitted_at
17
+ t.datetime :due_at
17
18
  t.datetime :graded_at
18
19
  t.float :score
19
20
  t.float :points_possible
@@ -225,6 +225,7 @@ ActiveRecord::Schema.define(version: 2020_04_16_214248) do
225
225
  t.bigint "canvas_assignment_id"
226
226
  t.bigint "canvas_user_id"
227
227
  t.datetime "submitted_at"
228
+ t.datetime "due_at"
228
229
  t.datetime "graded_at"
229
230
  t.float "score"
230
231
  t.float "points_possible"
@@ -1,3 +1,3 @@
1
- "canvas user id","sis user id","user name","canvas course id","sis course id","course name","assignment id","assignment name","submission date","graded date","score","points possible","submission id","workflow state","excused"
2
- 1,1,userName,1,1,courseName,1,assignmentName,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,1,graded,true
3
- 1,1,userName,1,1,courseName,2,assignment2Name,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,2,submitted,false
1
+ "canvas user id","sis user id","user name","canvas course id","sis course id","course name","assignment id","assignment name","submission date","graded date","score","points possible","submission id","workflow state","excused","due at"
2
+ 1,1,userName,1,1,courseName,1,assignmentName,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,1,graded,true,2017-09-27 18:28:15 UTC
3
+ 1,1,userName,1,1,courseName,2,assignment2Name,2017-09-21 18:28:15 UTC,2017-09-21 19:36:23 UTC,5,10,2,submitted,false,2017-09-28 18:28:15 UTC
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: canvas_sync
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.14.0
4
+ version: 0.16.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nate Collings
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-04-21 00:00:00.000000000 Z
11
+ date: 2020-09-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -340,6 +340,7 @@ files:
340
340
  - lib/canvas_sync/generators/templates/migrations/create_enrollments.rb
341
341
  - lib/canvas_sync/generators/templates/migrations/create_group_memberships.rb
342
342
  - lib/canvas_sync/generators/templates/migrations/create_groups.rb
343
+ - lib/canvas_sync/generators/templates/migrations/create_pseudonyms.rb
343
344
  - lib/canvas_sync/generators/templates/migrations/create_roles.rb
344
345
  - lib/canvas_sync/generators/templates/migrations/create_sections.rb
345
346
  - lib/canvas_sync/generators/templates/migrations/create_submissions.rb
@@ -355,6 +356,7 @@ files:
355
356
  - lib/canvas_sync/generators/templates/models/enrollment.rb
356
357
  - lib/canvas_sync/generators/templates/models/group.rb
357
358
  - lib/canvas_sync/generators/templates/models/group_membership.rb
359
+ - lib/canvas_sync/generators/templates/models/pseudonym.rb
358
360
  - lib/canvas_sync/generators/templates/models/role.rb
359
361
  - lib/canvas_sync/generators/templates/models/section.rb
360
362
  - lib/canvas_sync/generators/templates/models/submission.rb
@@ -391,7 +393,6 @@ files:
391
393
  - lib/canvas_sync/jobs/sync_simple_table_job.rb
392
394
  - lib/canvas_sync/jobs/sync_submissions_job.rb
393
395
  - lib/canvas_sync/jobs/sync_terms_job.rb
394
- - lib/canvas_sync/jobs/sync_users_job.rb
395
396
  - lib/canvas_sync/processors/assignment_groups_processor.rb
396
397
  - lib/canvas_sync/processors/assignments_processor.rb
397
398
  - lib/canvas_sync/processors/context_module_items_processor.rb
@@ -420,7 +421,6 @@ files:
420
421
  - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
421
422
  - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
422
423
  - spec/canvas_sync/jobs/sync_terms_job_spec.rb
423
- - spec/canvas_sync/jobs/sync_users_job_spec.rb
424
424
  - spec/canvas_sync/models/accounts_spec.rb
425
425
  - spec/canvas_sync/models/admins_spec.rb
426
426
  - spec/canvas_sync/models/assignment_group_spec.rb
@@ -596,7 +596,6 @@ test_files:
596
596
  - spec/canvas_sync/jobs/sync_simple_table_job_spec.rb
597
597
  - spec/canvas_sync/jobs/sync_submissions_job_spec.rb
598
598
  - spec/canvas_sync/jobs/sync_terms_job_spec.rb
599
- - spec/canvas_sync/jobs/sync_users_job_spec.rb
600
599
  - spec/canvas_sync/models/accounts_spec.rb
601
600
  - spec/canvas_sync/models/admins_spec.rb
602
601
  - spec/canvas_sync/models/assignment_group_spec.rb
@@ -1,26 +0,0 @@
1
- module CanvasSync
2
- module Jobs
3
- class SyncUsersJob < ReportStarter
4
- # Starts a provisioning report for just users.
5
- #
6
- # Provisioning reports do not scope users by term, so when we are
7
- # running provisioning by term we sync users first so we don't duplicate
8
- # the work of syncing all users for each term.
9
- #
10
- # @param job_chain [Hash]
11
- # @param options [Hash]
12
- def perform(job_chain, options)
13
- super(
14
- job_chain,
15
- "proservices_provisioning_csv",
16
- merge_report_params(job_chain, options, {
17
- users: true,
18
- include_deleted: true,
19
- }, term_scope: false),
20
- CanvasSync::Processors::ProvisioningReportProcessor.to_s,
21
- { models: ["users"] },
22
- )
23
- end
24
- end
25
- end
26
- end
@@ -1,15 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe CanvasSync::Jobs::SyncUsersJob do
4
- describe '#perform' do
5
- it 'enqueues a ReportStarter for a provisioning report across all terms with just users' do
6
- expect_any_instance_of(Bearcat::Client).to receive(:start_report)
7
- .with('self', 'proservices_provisioning_csv', { parameters: { users: true, include_deleted: true } })
8
- .and_return({ 'id' => 1 })
9
-
10
- expect(CanvasSync::Jobs::ReportChecker).to receive(:set).and_call_original
11
-
12
- CanvasSync::Jobs::SyncUsersJob.perform_now({ jobs: [], global_options: {}}, {})
13
- end
14
- end
15
- end