canvas_sync 0.17.17.beta1 → 0.17.23.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +45 -1
  3. data/lib/canvas_sync/concerns/sync_mapping.rb +112 -0
  4. data/lib/canvas_sync/generators/install_generator.rb +1 -0
  5. data/lib/canvas_sync/generators/templates/migrations/create_grading_period_groups.rb +22 -0
  6. data/lib/canvas_sync/generators/templates/migrations/create_grading_periods.rb +22 -0
  7. data/lib/canvas_sync/generators/templates/migrations/create_user_observers.rb +17 -0
  8. data/lib/canvas_sync/generators/templates/models/grading_period.rb +8 -0
  9. data/lib/canvas_sync/generators/templates/models/grading_period_group.rb +9 -0
  10. data/lib/canvas_sync/generators/templates/models/user_observer.rb +11 -0
  11. data/lib/canvas_sync/importers/bulk_importer.rb +27 -16
  12. data/lib/canvas_sync/job_batches/batch.rb +9 -0
  13. data/lib/canvas_sync/job_batches/chain_builder.rb +9 -1
  14. data/lib/canvas_sync/job_batches/hier_batch_ids.lua +25 -0
  15. data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/css/styles.less +178 -0
  16. data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/js/batch_tree.js +106 -0
  17. data/lib/canvas_sync/job_batches/sidekiq/web/batches_assets/js/util.js +2 -0
  18. data/lib/canvas_sync/job_batches/sidekiq/web/views/_batch_tree.erb +6 -0
  19. data/lib/canvas_sync/job_batches/sidekiq/web/views/_common.erb +13 -0
  20. data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb +15 -88
  21. data/lib/canvas_sync/job_batches/sidekiq/web.rb +93 -0
  22. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +1 -1
  23. data/lib/canvas_sync/jobs/report_checker.rb +37 -4
  24. data/lib/canvas_sync/jobs/report_starter.rb +2 -2
  25. data/lib/canvas_sync/processors/assignment_groups_processor.rb +1 -7
  26. data/lib/canvas_sync/processors/assignments_processor.rb +1 -7
  27. data/lib/canvas_sync/processors/context_module_items_processor.rb +1 -7
  28. data/lib/canvas_sync/processors/context_modules_processor.rb +1 -7
  29. data/lib/canvas_sync/processors/model_mappings.yml +68 -0
  30. data/lib/canvas_sync/processors/normal_processor.rb +3 -3
  31. data/lib/canvas_sync/processors/provisioning_report_processor.rb +21 -63
  32. data/lib/canvas_sync/processors/report_processor.rb +14 -9
  33. data/lib/canvas_sync/processors/submissions_processor.rb +1 -7
  34. data/lib/canvas_sync/record.rb +4 -0
  35. data/lib/canvas_sync/version.rb +1 -1
  36. data/lib/canvas_sync.rb +4 -1
  37. data/spec/canvas_sync/processors/provisioning_report_processor_spec.rb +40 -0
  38. data/spec/dummy/app/models/grading_period.rb +14 -0
  39. data/spec/dummy/app/models/grading_period_group.rb +15 -0
  40. data/spec/dummy/app/models/user_observer.rb +17 -0
  41. data/spec/dummy/db/migrate/20210907233329_create_user_observers.rb +23 -0
  42. data/spec/dummy/db/migrate/20210907233330_create_grading_periods.rb +28 -0
  43. data/spec/dummy/db/migrate/20210907233331_create_grading_period_groups.rb +28 -0
  44. data/spec/dummy/db/schema.rb +42 -1
  45. data/spec/dummy/log/development.log +1167 -0
  46. data/spec/dummy/log/test.log +2775 -0
  47. data/spec/support/fixtures/reports/grading_period_groups.csv +2 -0
  48. data/spec/support/fixtures/reports/grading_periods.csv +3 -0
  49. data/spec/support/fixtures/reports/user_observers.csv +3 -0
  50. metadata +38 -17
@@ -12,7 +12,7 @@ module CanvasSync
12
12
  def perform(report_name, report_params, processor, options, allow_redownloads: false)
13
13
  account_id = options[:account_id] || batch_context[:account_id] || "self"
14
14
  options[:sync_start_time] = DateTime.now.utc.iso8601
15
-
15
+ options[:report_params] = report_params
16
16
  report_id = start_report(account_id, report_name, report_params)
17
17
  # TODO: Restore report caching support (does nayone actually use it?)
18
18
  # report_id = if allow_redownloads
@@ -28,7 +28,7 @@ module CanvasSync
28
28
  report_name,
29
29
  report_id,
30
30
  processor,
31
- options,
31
+ options
32
32
  )
33
33
  end
34
34
  end
@@ -12,13 +12,7 @@ module CanvasSync
12
12
  end
13
13
 
14
14
  def initialize(report_file_path, options)
15
- CanvasSync::Importers::BulkImporter.import(
16
- report_file_path,
17
- mapping[:assignment_groups][:report_columns],
18
- AssignmentGroup,
19
- mapping[:assignment_groups][:conflict_target].to_sym,
20
- import_args: options
21
- )
15
+ do_bulk_import(report_file_path, AssignmentGroup, options: options)
22
16
  end
23
17
  end
24
18
  end
@@ -12,13 +12,7 @@ module CanvasSync
12
12
  end
13
13
 
14
14
  def initialize(report_file_path, options)
15
- CanvasSync::Importers::BulkImporter.import(
16
- report_file_path,
17
- mapping[:assignments][:report_columns],
18
- Assignment,
19
- mapping[:assignments][:conflict_target].to_sym,
20
- import_args: options
21
- )
15
+ do_bulk_import(report_file_path, Assignment, options: options)
22
16
  end
23
17
  end
24
18
  end
@@ -12,13 +12,7 @@ module CanvasSync
12
12
  end
13
13
 
14
14
  def initialize(report_file_path, options)
15
- CanvasSync::Importers::BulkImporter.import(
16
- report_file_path,
17
- mapping[:context_module_items][:report_columns],
18
- ContextModuleItem,
19
- mapping[:context_module_items][:conflict_target].to_sym,
20
- import_args: options
21
- )
15
+ do_bulk_import(report_file_path, ContextModuleItem, options: options)
22
16
  end
23
17
  end
24
18
  end
@@ -12,13 +12,7 @@ module CanvasSync
12
12
  end
13
13
 
14
14
  def initialize(report_file_path, options)
15
- CanvasSync::Importers::BulkImporter.import(
16
- report_file_path,
17
- mapping[:context_modules][:report_columns],
18
- ContextModule,
19
- mapping[:context_modules][:conflict_target].to_sym,
20
- import_args: options
21
- )
15
+ do_bulk_import(report_file_path, ContextModule, options: options)
22
16
  end
23
17
  end
24
18
  end
@@ -407,3 +407,71 @@ group_memberships:
407
407
  status:
408
408
  database_column_name: workflow_state
409
409
  type: string
410
+
411
+ user_observers:
412
+ conflict_target:
413
+ - canvas_observer_id
414
+ - canvas_student_id
415
+ report_columns:
416
+ canvas_observer_id:
417
+ database_column_name: observing_user_id
418
+ type: integer
419
+ canvas_student_id:
420
+ database_column_name: observed_user_id
421
+ type: integer
422
+ status:
423
+ database_column_name: workflow_state
424
+ type: string
425
+
426
+ grading_periods:
427
+ conflict_target: grading_period_id
428
+ report_columns:
429
+ grading_period_id:
430
+ database_column_name: canvas_id
431
+ type: integer
432
+ title:
433
+ database_column_name: title
434
+ type: string
435
+ weight:
436
+ database_column_name: weight
437
+ type: float
438
+ start_date:
439
+ database_column_name: start_date
440
+ type: datetime
441
+ end_date:
442
+ database_column_name: end_date
443
+ type: datetime
444
+ close_date:
445
+ database_column_name: close_date
446
+ type: datetime
447
+ grading_period_group_id:
448
+ database_column_name: canvas_grading_period_group_id
449
+ type: integer
450
+ status:
451
+ database_column_name: workflow_state
452
+ type: string
453
+
454
+ grading_period_groups:
455
+ conflict_target: grading_period_group_id
456
+ report_columns:
457
+ grading_period_group_id:
458
+ database_column_name: canvas_id
459
+ type: integer
460
+ canvas_course_id:
461
+ database_column_name: canvas_course_id
462
+ type: integer
463
+ canvas_account_id:
464
+ database_column_name: canvas_account_id
465
+ type: integer
466
+ title:
467
+ database_column_name: title
468
+ type: string
469
+ weighted:
470
+ database_column_name: weighted
471
+ type: boolean
472
+ display_totals_for_all_grading_periods:
473
+ database_column_name: display_totals_for_all_grading_periods
474
+ type: boolean
475
+ status:
476
+ database_column_name: workflow_state
477
+ type: string
@@ -13,12 +13,12 @@ module CanvasSync
13
13
  end
14
14
 
15
15
  def initialize(report_file_path, options)
16
- conflict_target = mapping[options[:mapping].to_sym][:conflict_target]
16
+ m = mapping[options[:mapping].to_sym]
17
17
  CanvasSync::Importers::BulkImporter.import(
18
18
  report_file_path,
19
- mapping[options[:mapping].to_sym][:report_columns],
19
+ m[:report_columns],
20
20
  options[:klass].constantize,
21
- conflict_target ? conflict_target.to_sym : conflict_target,
21
+ m[:conflict_target],
22
22
  import_args: options
23
23
  )
24
24
  end
@@ -69,94 +69,52 @@ module CanvasSync
69
69
  end
70
70
 
71
71
  def bulk_process_users(report_file_path)
72
- CanvasSync::Importers::BulkImporter.import(
73
- report_file_path,
74
- mapping[:users][:report_columns],
75
- User,
76
- mapping[:users][:conflict_target].to_sym,
77
- import_args: @options
78
- )
72
+ do_bulk_import(report_file_path, User, options: @options)
73
+ end
74
+
75
+ def bulk_process_user_observers(report_file_path)
76
+ do_bulk_import(report_file_path, UserObserver, options: @options)
79
77
  end
80
78
 
81
79
  def bulk_process_pseudonyms(report_file_path)
82
- CanvasSync::Importers::BulkImporter.import(
83
- report_file_path,
84
- mapping[:pseudonyms][:report_columns],
85
- Pseudonym,
86
- mapping[:pseudonyms][:conflict_target].to_sym,
87
- import_args: @options
88
- )
80
+ do_bulk_import(report_file_path, Pseudonym, options: @options)
89
81
  end
90
82
 
91
83
  def bulk_process_accounts(report_file_path)
92
- CanvasSync::Importers::BulkImporter.import(
93
- report_file_path,
94
- mapping[:accounts][:report_columns],
95
- Account,
96
- mapping[:accounts][:conflict_target].to_sym,
97
- import_args: @options
98
- )
84
+ do_bulk_import(report_file_path, Account, options: @options)
99
85
  end
100
86
 
101
87
  def bulk_process_courses(report_file_path)
102
- CanvasSync::Importers::BulkImporter.import(
103
- report_file_path,
104
- mapping[:courses][:report_columns],
105
- Course,
106
- mapping[:courses][:conflict_target].to_sym,
107
- import_args: @options
108
- )
88
+ do_bulk_import(report_file_path, Course, options: @options)
109
89
  end
110
90
 
111
91
  def bulk_process_enrollments(report_file_path)
112
- CanvasSync::Importers::BulkImporter.import(
113
- report_file_path,
114
- mapping[:enrollments][:report_columns],
115
- Enrollment,
116
- mapping[:enrollments][:conflict_target].to_sym,
117
- import_args: @options
118
- )
92
+ do_bulk_import(report_file_path, Enrollment, options: @options)
119
93
  end
120
94
 
121
95
  def bulk_process_sections(report_file_path)
122
- CanvasSync::Importers::BulkImporter.import(
123
- report_file_path,
124
- mapping[:sections][:report_columns],
125
- Section,
126
- mapping[:sections][:conflict_target].to_sym,
127
- import_args: @options
128
- )
96
+ do_bulk_import(report_file_path, Section, options: @options)
129
97
  end
130
98
 
131
99
  def bulk_process_xlist(report_file_path)
132
- CanvasSync::Importers::BulkImporter.import(
133
- report_file_path,
134
- mapping[:xlist][:report_columns],
135
- Section,
136
- mapping[:xlist][:conflict_target].to_sym,
137
- import_args: @options
138
- )
100
+ do_bulk_import(report_file_path, Section, options: @options, mapping_key: :xlist)
139
101
  end
140
102
 
141
103
  def bulk_process_groups(report_file_path)
142
- CanvasSync::Importers::BulkImporter.import(
143
- report_file_path,
144
- mapping[:groups][:report_columns],
145
- Group,
146
- mapping[:groups][:conflict_target].to_sym,
147
- import_args: @options
148
- )
104
+ do_bulk_import(report_file_path, Group, options: @options)
105
+ end
106
+
107
+ def bulk_process_grading_periods(report_file_path)
108
+ do_bulk_import(report_file_path, GradingPeriod, options: @options)
109
+ end
110
+
111
+ def bulk_process_grading_period_groups(report_file_path)
112
+ do_bulk_import(report_file_path, GradingPeriodGroup, options: @options)
149
113
  end
150
114
 
151
115
  # Note that group membership is singular because we override the model name param in sync_provisioning_report_job
152
116
  def bulk_process_group_membership(report_file_path)
153
- CanvasSync::Importers::BulkImporter.import(
154
- report_file_path,
155
- mapping[:group_memberships][:report_columns],
156
- GroupMembership,
157
- mapping[:group_memberships][:conflict_target].to_sym,
158
- import_args: @options
159
- )
117
+ do_bulk_import(report_file_path, GroupMembership, options: @options)
160
118
  end
161
119
  end
162
120
  end
@@ -5,17 +5,22 @@ module CanvasSync
5
5
  # Base report processing class
6
6
  class ReportProcessor
7
7
  def mapping
8
- @mapping ||= begin
9
- mapping = YAML.load_file(File.join(__dir__, "model_mappings.yml")).deep_symbolize_keys!
10
- override_filepath = Rails.root.join("config/canvas_sync_provisioning_mapping.yml")
8
+ SyncMapping::Mapping.default_mappings
9
+ end
11
10
 
12
- if File.file?(override_filepath)
13
- override = YAML.load_file(override_filepath).deep_symbolize_keys!
14
- mapping = mapping.merge(override)
15
- end
11
+ def mapping_for(model, key = nil)
12
+ model.try(:get_sync_mapping, key) || mapping[key || SyncMapping::Mapping.normalize_model_name(model)]
13
+ end
16
14
 
17
- mapping
18
- end
15
+ def do_bulk_import(report_file_path, model, options: {}, mapping_key: nil)
16
+ m = mapping_for(model, mapping_key)
17
+ CanvasSync::Importers::BulkImporter.import(
18
+ report_file_path,
19
+ m[:report_columns],
20
+ model,
21
+ m[:conflict_target],
22
+ import_args: options
23
+ )
19
24
  end
20
25
  end
21
26
  end
@@ -12,13 +12,7 @@ module CanvasSync
12
12
  end
13
13
 
14
14
  def initialize(report_file_path, options)
15
- CanvasSync::Importers::BulkImporter.import(
16
- report_file_path,
17
- mapping[:submissions][:report_columns],
18
- Submission,
19
- mapping[:submissions][:conflict_target].to_sym,
20
- import_args: options
21
- )
15
+ do_bulk_import(report_file_path, Submission, options: options)
22
16
  end
23
17
  end
24
18
  end
@@ -1,6 +1,10 @@
1
+
2
+ require_relative 'concerns/sync_mapping'
3
+
1
4
  module CanvasSync
2
5
  module Record
3
6
  extend ActiveSupport::Concern
7
+ include SyncMapping
4
8
 
5
9
  included do
6
10
  define_model_callbacks :sync_import
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.17.17.beta1".freeze
2
+ VERSION = "0.17.23.beta1".freeze
3
3
  end
data/lib/canvas_sync.rb CHANGED
@@ -41,6 +41,9 @@ module CanvasSync
41
41
  context_modules
42
42
  context_module_items
43
43
  xlist
44
+ user_observers
45
+ grading_periods
46
+ grading_period_groups
44
47
  ].freeze
45
48
 
46
49
  SUPPORTED_TERM_SCOPE_MODELS = %w[
@@ -176,7 +179,7 @@ module CanvasSync
176
179
  # These Models use the provisioning report, but are not term-scoped,
177
180
  # so we sync them outside of the term scoping to ensure work is not duplicated
178
181
  if term_scope.present?
179
- models -= (first_provisioning_models = models & ['users', 'pseudonyms'])
182
+ models -= (first_provisioning_models = models & ['users', 'pseudonyms', 'user_observers'])
180
183
  current_chain.insert(generate_provisioning_jobs(first_provisioning_models, options))
181
184
  end
182
185
 
@@ -62,6 +62,46 @@ RSpec.describe CanvasSync::Processors::ProvisioningReportProcessor do
62
62
  expect(group.canvas_user_id).to eq 10
63
63
  end
64
64
 
65
+ it 'processes user_observers' do
66
+ expect {
67
+ subject.process('spec/support/fixtures/reports/user_observers.csv', { models: ['user_observers'] }, 1)
68
+ }.to change { UserObserver.count }.by(2)
69
+ obj = UserObserver.find_by(observed_user_id: 3)
70
+ expect(obj.workflow_state).to eq 'active'
71
+ expect(obj.observing_user_id).to eq 21
72
+ end
73
+
74
+ it 'processes grading_periods' do
75
+ expect {
76
+ subject.process('spec/support/fixtures/reports/grading_periods.csv', { models: ['grading_periods'] }, 1)
77
+ }.to change { GradingPeriod.count }.by(2)
78
+ obj = GradingPeriod.find_by(canvas_id: 1)
79
+ puts obj.inspect
80
+ expect(obj.title).to eq 'Period 1'
81
+ expect(obj.weight).to eq 0.2
82
+ expect(obj.workflow_state).to eq 'active'
83
+ end
84
+
85
+ it 'processes grading_period_groups' do
86
+ expect {
87
+ subject.process('spec/support/fixtures/reports/grading_period_groups.csv', { models: ['grading_period_groups'] }, 1)
88
+ }.to change { GradingPeriodGroup.count }.by(1)
89
+ obj = GradingPeriodGroup.find_by(canvas_id: 1)
90
+ puts obj.inspect
91
+ expect(obj.title).to eq 'Test Group'
92
+ expect(obj.workflow_state).to eq 'active'
93
+ end
94
+
95
+ it 'model with composite key behaves as expected' do
96
+ expect {
97
+ subject.process('spec/support/fixtures/reports/user_observers.csv', { models: ['user_observers'] }, 1)
98
+ }.to change { UserObserver.count }.by(2)
99
+
100
+ expect {
101
+ subject.process('spec/support/fixtures/reports/user_observers.csv', { models: ['user_observers'] }, 1)
102
+ }.to change { UserObserver.count }.by(0)
103
+ end
104
+
65
105
  context 'options[:models] is multiple models' do
66
106
  it 'extracts the ZIP and processes each model' do
67
107
  user_count = User.count
@@ -0,0 +1,14 @@
1
+ # #
2
+ # AUTO GENERATED MODEL
3
+ # This model was auto generated by the CanvasSync Gem.
4
+ # You can customize it as needed, but make sure you test
5
+ # any changes you make to the auto generated methods.
6
+ #
7
+
8
+
9
+ class GradingPeriod < ApplicationRecord
10
+ include CanvasSync::Record
11
+
12
+ validates :canvas_id, uniqueness: true, presence: true
13
+ belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :canvas_grading_period_group_id, optional: true
14
+ end
@@ -0,0 +1,15 @@
1
+ # #
2
+ # AUTO GENERATED MODEL
3
+ # This model was auto generated by the CanvasSync Gem.
4
+ # You can customize it as needed, but make sure you test
5
+ # any changes you make to the auto generated methods.
6
+ #
7
+
8
+
9
+ class GradingPeriodGroup < ApplicationRecord
10
+ include CanvasSync::Record
11
+
12
+ validates :canvas_id, uniqueness: true, presence: true
13
+ belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
14
+ belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
15
+ end