canvas_sync 0.7.3 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA256:
3
- metadata.gz: b7439a3d10f86fcf936b85479a44a4869eac49c32be3d035ab652442ba3ec880
4
- data.tar.gz: e1bd2bcc305b7c737bcd5150357319c2feaacf04bceff757888a086a9101f293
2
+ SHA1:
3
+ metadata.gz: bcfe0a03413b48c2c1ca696c0d60c7aab54b9a8f
4
+ data.tar.gz: 1c6a2daa99f260e56739463bac62a6ab1d96ed67
5
5
  SHA512:
6
- metadata.gz: 7fb2641bcaa7c472cfd0078c061be94a360da56c46671917fd5096a18fa86b375cbf3dd26f21fcab2618a88d24110fa94548ab9b29536c8f0411a1784c9a91ca
7
- data.tar.gz: cc3bd13b13000947a20dd3cf0b1e5bd3c58733b847e17f19d733ce218f2f973801e9b9a165e4c5667c6f490266ce4727a2c77f691505a7840a39f951de39600c
6
+ metadata.gz: 9b0b91b958c03a577ed3f5b31f2d70436f8577e13450282dd27818cb001eaf45ad19ad9ca3b70feba7c573fa4dec15ab4837c9a354675445f16ddd90162ca21e
7
+ data.tar.gz: 4387a495528f967c274fc5c08f51f753639aca0ffa5b6f3550cf59e59cf144f0c1c87a4585352870b99c5ef3034d1872b12ee34c4800b094960948677d497c97
data/README.md CHANGED
@@ -229,6 +229,85 @@ Sidekiq.configure_server do |config|
229
229
  end
230
230
  ```
231
231
 
232
+ ## Syncronize different reports
233
+ CanvasSync provides the functionality to import data from other reports into an specific table.
234
+
235
+ This can be achived by using the followin method
236
+
237
+ ```ruby
238
+ CanvasSync.provisioning_sync(<array of models to sync>, term_scope: <optional term scope>)
239
+ CanvasSync
240
+ .simple_report_sync(
241
+ [
242
+ {
243
+ report_name: <report name>,
244
+ model: <model to sync>,
245
+ params: <hash with the require parameters the report needs to sync>
246
+ },
247
+ {
248
+ report_name: <report name>,
249
+ model: <model to sync>,
250
+ params: <hash with the require parameters the report needs to sync>
251
+ },
252
+ ...
253
+ ],
254
+ term_scope: <optional term scope>
255
+ )
256
+ ```
257
+
258
+ Example:
259
+
260
+ ```ruby
261
+ CanvasSync
262
+ .simple_report_sync(
263
+ [
264
+ {
265
+ report_name: 'proservices_provisioning_csv',
266
+ model: 'users',
267
+ params: {
268
+ "parameters[include_deleted]" => true,
269
+ "parameters[users]" => true
270
+ }
271
+ },
272
+ {
273
+ report_name: 'proservices_provisioning_csv',
274
+ model: 'accounts',
275
+ params: {
276
+ "parameters[include_deleted]" => true,
277
+ "parameters[accounts]" => true
278
+ }
279
+ }
280
+ ]
281
+ )
282
+ ```
283
+
284
+ Example with the term_scope active:
285
+
286
+ ```ruby
287
+ CanvasSync
288
+ .simple_report_sync(
289
+ [
290
+ {
291
+ report_name: 'proservices_provisioning_csv',
292
+ model: 'sections',
293
+ params: {
294
+ "parameters[include_deleted]" => true,
295
+ "parameters[sections]" => true
296
+ }
297
+ },
298
+ {
299
+ report_name: 'proservices_provisioning_csv',
300
+ model: 'courses',
301
+ params: {
302
+ "parameters[include_deleted]" => true,
303
+ "parameters[courses]" => true
304
+ }
305
+ }
306
+ ],
307
+ term_scope: 'active'
308
+ )
309
+ ```
310
+
232
311
  ## Configuration
233
312
 
234
313
  You can configure CanvasSync settings by doing the following:
data/lib/canvas_sync.rb CHANGED
@@ -7,6 +7,7 @@ require "canvas_sync/jobs/report_starter"
7
7
  require "canvas_sync/jobs/report_checker"
8
8
  require "canvas_sync/jobs/report_processor_job"
9
9
  require "canvas_sync/jobs/sync_provisioning_report_job"
10
+ require "canvas_sync/jobs/sync_simple_table_job.rb"
10
11
  require "canvas_sync/jobs/sync_assignments_job"
11
12
  require "canvas_sync/jobs/sync_submissions_job"
12
13
  require "canvas_sync/jobs/sync_assignment_groups_job"
@@ -52,6 +53,10 @@ module CanvasSync
52
53
  course_section
53
54
  ].freeze
54
55
 
56
+ SUPPORTED_NON_PROV_REPORTS = %w[
57
+ graded_submissions
58
+ ].freeze
59
+
55
60
  class << self
56
61
  # Runs a standard provisioning sync job with no extra report types.
57
62
  # Terms will be synced first using the API. If you are syncing users/roles/admins
@@ -72,6 +77,19 @@ module CanvasSync
72
77
  invoke_next(default_provisioning_report_chain(models, term_scope, legacy_support, account_id))
73
78
  end
74
79
 
80
+ # Runs a report different from provisioning sync job with no extra report types.
81
+ #
82
+ # @param reports_mapping [Array<Hash>] An Array of hash that list the model and params with their report you
83
+ # want to import:
84
+ # [{model: 'submissions', report_name: 'my_report_name_csv', params: { "parameters[include_deleted]" => true } }, ...]
85
+ # @param term_scope [Symbol, nil] An optional symbol representing a scope that exists on the Term model.
86
+ # The provisioning report will be run for each of the terms contained in that scope.
87
+ # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
88
+ # canvas_sync_client methods require an account ID.
89
+ def simple_report_sync(reports_mapping, term_scope: nil, account_id: nil)
90
+ invoke_next(simple_report_chain(reports_mapping, term_scope, account_id))
91
+ end
92
+
75
93
  # Runs a chain of ordered jobs
76
94
  #
77
95
  # See the README for usage and examples
@@ -99,6 +117,33 @@ module CanvasSync
99
117
  next_job_class.perform_later(duped_job_chain, next_job[:options])
100
118
  end
101
119
 
120
+ # Syn any report to an specific set of models
121
+ #
122
+ # @param reports_mapping [Array<Hash>] List of reports with their specific model and params
123
+ # @param term_scope [String]
124
+ # @param account_id [Integer, nil] This optional parameter can be used if your Term creation and
125
+ # canvas_sync_client methods require an account ID.
126
+ def simple_report_chain(reports_mapping, term_scope=nil, account_id=nil)
127
+ jobs = reports_mapping.map do |report|
128
+ {
129
+ job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
130
+ options: {
131
+ report_name: report[:report_name],
132
+ model: report[:model],
133
+ mapping: report[:model],
134
+ klass: report[:model].singularize.capitalize.to_s,
135
+ term_scope: term_scope,
136
+ params: report[:params]
137
+ }
138
+ }
139
+ end
140
+
141
+ global_options = {}
142
+ global_options[:account_id] = account_id if account_id.present?
143
+
144
+ { jobs: jobs, global_options: global_options }
145
+ end
146
+
102
147
  # Syncs terms, users/roles/admins if necessary, then the rest of the specified models.
103
148
  #
104
149
  # @param models [Array<String>]
@@ -22,13 +22,16 @@ module CanvasSync
22
22
  database_column_names = mapping.values.map { |value| value[:database_column_name] }
23
23
  rows = []
24
24
  row_ids = {}
25
+ database_conflict_column_name = conflict_target ? mapping[conflict_target][:database_column_name] : nil
25
26
 
26
27
  CSV.foreach(report_file_path, headers: true, header_converters: :symbol) do |row|
27
28
  row = yield(row) if block_given?
28
29
  next if row.nil?
29
30
 
30
- next if row_ids[row[conflict_target]]
31
- row_ids[row[conflict_target]] = true
31
+ if conflict_target
32
+ next if row_ids[row[conflict_target]]
33
+ row_ids[row[conflict_target]] = true
34
+ end
32
35
 
33
36
  rows << csv_column_names.map do |column|
34
37
  if mapping[column][:type].to_sym == :datetime
@@ -41,24 +44,26 @@ module CanvasSync
41
44
  end
42
45
 
43
46
  if rows.length >= batch_size
44
- perform_import(klass, database_column_names, rows, mapping[conflict_target][:database_column_name], import_args)
47
+ perform_import(klass, database_column_names, rows, database_conflict_column_name, import_args)
45
48
  rows = []
46
49
  row_ids = {}
47
50
  end
48
51
  end
49
52
 
50
- perform_import(klass, database_column_names, rows, mapping[conflict_target][:database_column_name], import_args)
53
+ perform_import(klass, database_column_names, rows, database_conflict_column_name, import_args)
51
54
  end
52
55
 
53
56
  def self.perform_import(klass, columns, rows, conflict_target, import_args={})
54
57
  return if rows.length.zero?
55
58
  columns = columns.dup
56
59
 
57
- options = { validate: false, on_duplicate_key_update: {
58
- conflict_target: conflict_target,
60
+ update_conditions = {
59
61
  condition: condition_sql(klass, columns),
60
62
  columns: columns
61
- } }.merge(import_args)
63
+ }
64
+ update_conditions[:conflict_target] = conflict_target if conflict_target
65
+
66
+ options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
62
67
 
63
68
  options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
64
69
  klass.import(columns, rows, options)
@@ -0,0 +1,41 @@
1
+ module CanvasSync
2
+ module Jobs
3
+ class SyncSimpleTableJob < ReportStarter
4
+ #
5
+ # Starts a report processor for the specified report
6
+ # (the specified report must be enabled)
7
+ #
8
+ # @param job_chain [Hash]
9
+ # @param options [Hash]
10
+ def perform(job_chain, options)
11
+ if options[:term_scope]
12
+ Term.send(options[:term_scope]).find_each do |term|
13
+ # Deep copy the job_chain so each report gets the correct term id passed into
14
+ # its options with no side effects
15
+ duped_job_chain = Marshal.load(Marshal.dump(job_chain))
16
+ duped_job_chain[:global_options][:canvas_term_id] = term.canvas_term_id
17
+ start_report(report_params(options, term.canvas_term_id), duped_job_chain, options)
18
+ end
19
+ else
20
+ start_report(report_params(options), job_chain, options)
21
+ end
22
+ end
23
+
24
+ def start_report(params, job_chain, options)
25
+ CanvasSync::Jobs::ReportStarter.perform_later(
26
+ job_chain,
27
+ options[:report_name],
28
+ params,
29
+ CanvasSync::Processors::NormalProcessor.to_s,
30
+ options,
31
+ )
32
+ end
33
+
34
+ def report_params(options, canvas_term_id=nil)
35
+ params = options[:params] || {}
36
+ params["parameters[enrollment_term_id]"] = canvas_term_id if canvas_term_id
37
+ params
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,26 @@
1
+ require_relative "./report_processor"
2
+
3
+ module CanvasSync
4
+ module Processors
5
+ # Processes any report using the normal bulk importer.
6
+ #
7
+ # @param report_file_path [String]
8
+ # @param options [Hash]
9
+ # @param report_id [Integer]
10
+ class NormalProcessor < ReportProcessor
11
+ def self.process(report_file_path, options, report_id)
12
+ new(report_file_path, options)
13
+ end
14
+
15
+ def initialize(report_file_path, options)
16
+ conflict_target = mapping[options[:mapping].to_sym][:conflict_target]
17
+ CanvasSync::Importers::BulkImporter.import(
18
+ report_file_path,
19
+ mapping[options[:mapping].to_sym][:report_columns],
20
+ options[:klass].constantize,
21
+ conflict_target ? conflict_target.to_sym : conflict_target
22
+ )
23
+ end
24
+ end
25
+ end
26
+ end
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.7.3".freeze
2
+ VERSION = "0.8.0".freeze
3
3
  end
@@ -153,4 +153,81 @@ RSpec.describe CanvasSync do
153
153
  end
154
154
  end
155
155
  end
156
+
157
+ describe ".simple_report_sync" do
158
+
159
+ it 'invokes the first job in the queue and passes on the rest of the job chain' do
160
+ expected_job_chain = CanvasSync.simple_report_chain(
161
+ [
162
+ {
163
+ report_name: 'proservices_provisioning_csv',
164
+ model: 'users',
165
+ params: {
166
+ "parameters[include_deleted]" => true,
167
+ "parameters[users]" => true
168
+ }
169
+ }
170
+ ]
171
+ )
172
+ first_job = expected_job_chain[:jobs].shift
173
+
174
+ expect(CanvasSync::Jobs::SyncSimpleTableJob).to receive(:perform_later)
175
+ .with(
176
+ expected_job_chain,
177
+ first_job[:options]
178
+ )
179
+
180
+ CanvasSync.simple_report_sync(
181
+ [
182
+ {
183
+ report_name: 'proservices_provisioning_csv',
184
+ model: 'users',
185
+ params: {
186
+ "parameters[include_deleted]" => true,
187
+ "parameters[users]" => true
188
+ }
189
+ }
190
+ ]
191
+ )
192
+ end
193
+
194
+ it 'receives the job chain for the specified table' do
195
+ chain = CanvasSync.simple_report_chain(
196
+ [
197
+ {
198
+ report_name: 'proservices_provisioning_csv',
199
+ model: 'users',
200
+ params: {
201
+ "parameters[include_deleted]" => true,
202
+ "parameters[users]" => true
203
+ }
204
+ }
205
+ ]
206
+ )
207
+
208
+ expected_job_chain = {
209
+ jobs: [
210
+ {
211
+ job: CanvasSync::Jobs::SyncSimpleTableJob.to_s,
212
+ options: {
213
+ report_name: 'proservices_provisioning_csv',
214
+ model: 'users',
215
+ mapping: 'users',
216
+ klass: 'User',
217
+ term_scope: nil,
218
+ params: {
219
+ "parameters[include_deleted]" => true,
220
+ "parameters[users]" => true
221
+ }
222
+ }
223
+ }
224
+ ],
225
+ global_options: {}
226
+ }
227
+
228
+ expect(chain).to eq(expected_job_chain)
229
+
230
+ end
231
+ end
232
+
156
233
  end
@@ -0,0 +1,57 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe CanvasSync::Jobs::SyncSimpleTableJob do
4
+ describe '#perform' do
5
+ let(:job_chain) { {jobs: [], global_options: {}} }
6
+
7
+ context 'Simple report' do
8
+ let!(:active_term_1) { FactoryGirl.create(:term) }
9
+ let!(:inactive_term_1) { FactoryGirl.create(:term, workflow_state: 'inactive') }
10
+
11
+ it 'enqueues a ReportStarter for a provisioning report for the specified model for a term' do
12
+ expected_job_chain = Marshal.load(Marshal.dump(job_chain))
13
+ expected_job_chain[:global_options][:canvas_term_id] = active_term_1.canvas_term_id
14
+
15
+ expect(CanvasSync::Jobs::ReportStarter).to receive(:perform_later)
16
+ .with(
17
+ expected_job_chain,
18
+ 'proservices_provisioning_csv',
19
+ {
20
+ "parameters[include_deleted]" => true,
21
+ "parameters[courses]" => true,
22
+ "parameters[enrollment_term_id]" => active_term_1.canvas_term_id
23
+ },
24
+ CanvasSync::Processors::NormalProcessor.to_s,
25
+ {
26
+ report_name: 'proservices_provisioning_csv',
27
+ model: 'courses',
28
+ mapping: 'courses',
29
+ klass: 'Course',
30
+ term_scope: 'active',
31
+ params: {
32
+ "parameters[include_deleted]" => true,
33
+ "parameters[courses]" => true,
34
+ "parameters[enrollment_term_id]" => active_term_1.canvas_term_id
35
+ }
36
+ }
37
+ )
38
+
39
+ CanvasSync::Jobs::SyncSimpleTableJob.perform_now(
40
+ job_chain,
41
+ {
42
+ report_name: 'proservices_provisioning_csv',
43
+ model: 'courses',
44
+ mapping: 'courses',
45
+ klass: 'Course',
46
+ term_scope: 'active',
47
+ params: {
48
+ "parameters[include_deleted]" => true,
49
+ "parameters[courses]" => true
50
+ }
51
+ }
52
+ )
53
+
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,27 @@
1
+ require "spec_helper"
2
+
3
+ RSpec.describe CanvasSync::Processors::NormalProcessor do
4
+ let(:subject) { CanvasSync::Processors::NormalProcessor }
5
+
6
+ describe "#process" do
7
+ it "inserts row" do
8
+ expect {
9
+ subject.process(
10
+ "spec/support/fixtures/reports/courses.csv",
11
+ {
12
+ report_name: 'proservices_provisioning_csv',
13
+ model: 'courses',
14
+ mapping: 'courses',
15
+ klass: 'Course',
16
+ term_scope: 'active',
17
+ params: {
18
+ "parameters[include_deleted]" => true,
19
+ "parameters[courses]" => true
20
+ }
21
+ },
22
+ 1
23
+ )
24
+ }.to change { Course.count }.by(2)
25
+ end
26
+ end
27
+ end