canvas_sync 0.15.1 → 0.16.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (44) hide show
  1. checksums.yaml +5 -5
  2. data/db/migrate/20170915210836_create_canvas_sync_job_log.rb +12 -31
  3. data/db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb +4 -13
  4. data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb +3 -11
  5. data/lib/canvas_sync.rb +11 -27
  6. data/lib/canvas_sync/concerns/api_syncable.rb +27 -0
  7. data/lib/canvas_sync/concerns/legacy_columns.rb +5 -4
  8. data/lib/canvas_sync/generators/templates/migrations/create_submissions.rb +1 -0
  9. data/lib/canvas_sync/generators/templates/models/account.rb +3 -0
  10. data/lib/canvas_sync/generators/templates/models/submission.rb +1 -0
  11. data/lib/canvas_sync/importers/bulk_importer.rb +7 -4
  12. data/lib/canvas_sync/job.rb +8 -2
  13. data/lib/canvas_sync/job_chain.rb +46 -1
  14. data/lib/canvas_sync/jobs/fork_gather.rb +27 -12
  15. data/lib/canvas_sync/jobs/report_starter.rb +1 -1
  16. data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +5 -5
  17. data/lib/canvas_sync/jobs/sync_simple_table_job.rb +4 -4
  18. data/lib/canvas_sync/misc_helper.rb +15 -0
  19. data/lib/canvas_sync/processors/assignment_groups_processor.rb +3 -2
  20. data/lib/canvas_sync/processors/assignments_processor.rb +3 -2
  21. data/lib/canvas_sync/processors/context_module_items_processor.rb +3 -2
  22. data/lib/canvas_sync/processors/context_modules_processor.rb +3 -2
  23. data/lib/canvas_sync/processors/model_mappings.yml +3 -0
  24. data/lib/canvas_sync/processors/normal_processor.rb +2 -1
  25. data/lib/canvas_sync/processors/provisioning_report_processor.rb +10 -2
  26. data/lib/canvas_sync/processors/submissions_processor.rb +3 -2
  27. data/lib/canvas_sync/version.rb +1 -1
  28. data/spec/canvas_sync/jobs/fork_gather_spec.rb +9 -9
  29. data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +2 -2
  30. data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +1 -1
  31. data/spec/dummy/app/models/account.rb +3 -0
  32. data/spec/dummy/app/models/pseudonym.rb +14 -0
  33. data/spec/dummy/app/models/submission.rb +1 -0
  34. data/spec/dummy/app/models/user.rb +1 -0
  35. data/spec/dummy/db/migrate/20190702203627_create_submissions.rb +1 -0
  36. data/spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb +24 -0
  37. data/spec/dummy/db/schema.rb +17 -4
  38. data/spec/dummy/db/test.sqlite3 +0 -0
  39. data/spec/dummy/log/development.log +1248 -0
  40. data/spec/dummy/log/test.log +43258 -0
  41. data/spec/support/fixtures/reports/provisioning_csv_unzipped/courses.csv +3 -0
  42. data/spec/support/fixtures/reports/provisioning_csv_unzipped/users.csv +4 -0
  43. data/spec/support/fixtures/reports/submissions.csv +3 -3
  44. metadata +22 -8
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 498fd2c3e3801c477077eced06fda11055ef8924
4
- data.tar.gz: 2a8b2979721246d70bd29fbb7a7a596480fb3b8e
2
+ SHA256:
3
+ metadata.gz: 9be3b81c22d5b45a02d88fa29b5d57afa152ee7dec6ce0ecf3d81c041507c619
4
+ data.tar.gz: 0c42577064cbf018f8fd8d5f665e1adfbc542e38e704a675e735491543b3ab40
5
5
  SHA512:
6
- metadata.gz: e645ffde592d741f67abddf9067816f4e2c440c7fc44d7af344def28dcc0c277f0573855bd0dceb9e4b883be830516b210d5ef43be79679c5a4bab42f3acb41e
7
- data.tar.gz: c415bffe6b53ca6fc92202119300ff71cf39d8cbfeec7d0387104bcfe1777740c1085d0e2e5ef2694ab7fc9dc3a99346ceda1470c148d844178804622c640d1d
6
+ metadata.gz: 9ea295b8cc43aa6b27bf6067cf684d024430a9fd85608bdc0a243113920b1b65b72c257e6a99644afb5d20455ecf2265949ce941a9ae1d9bea160ab1bee803ba
7
+ data.tar.gz: e63b7f557e792cb850a5799753f9effe436ad391bc6e4442768683b8c4d7c8eea1d1e9bbd6419127f2f4401f6cce6129019fb73192496f7efc1d731d7ef48847
@@ -1,35 +1,16 @@
1
- if Rails.version.to_f >= 5.0
2
- class CreateCanvasSyncJobLog < ActiveRecord::Migration[Rails.version.to_f]
3
- def change
4
- create_table :canvas_sync_job_logs do |t|
5
- t.datetime :started_at
6
- t.datetime :completed_at
7
- t.string :exception
8
- t.text :backtrace
9
- t.string :job_class
10
- t.string :status
11
- t.text :metadata
12
- t.text :job_arguments
1
+ class CreateCanvasSyncJobLog < CanvasSync::MiscHelper::MigrationClass
2
+ def change
3
+ create_table :canvas_sync_job_logs do |t|
4
+ t.datetime :started_at
5
+ t.datetime :completed_at
6
+ t.string :exception
7
+ t.text :backtrace
8
+ t.string :job_class
9
+ t.string :status
10
+ t.text :metadata
11
+ t.text :job_arguments
13
12
 
14
- t.timestamps
15
- end
16
- end
17
- end
18
- else
19
- class CreateCanvasSyncJobLog < ActiveRecord::Migration
20
- def change
21
- create_table :canvas_sync_job_logs do |t|
22
- t.datetime :started_at
23
- t.datetime :completed_at
24
- t.string :exception
25
- t.text :backtrace
26
- t.string :job_class
27
- t.string :status
28
- t.text :metadata
29
- t.text :job_arguments
30
-
31
- t.timestamps
32
- end
13
+ t.timestamps
33
14
  end
34
15
  end
35
16
  end
@@ -1,15 +1,6 @@
1
- if Rails.version.to_f >= 5.0
2
- class AddJobIdToCanvasSyncJobLogs < ActiveRecord::Migration[Rails.version.to_f]
3
- def change
4
- add_column :canvas_sync_job_logs, :job_id, :string
5
- add_index :canvas_sync_job_logs, :job_id
6
- end
7
- end
8
- else
9
- class AddJobIdToCanvasSyncJobLogs < ActiveRecord::Migration
10
- def change
11
- add_column :canvas_sync_job_logs, :job_id, :string
12
- add_index :canvas_sync_job_logs, :job_id
13
- end
1
+ class AddJobIdToCanvasSyncJobLogs < CanvasSync::MiscHelper::MigrationClass
2
+ def change
3
+ add_column :canvas_sync_job_logs, :job_id, :string
4
+ add_index :canvas_sync_job_logs, :job_id
14
5
  end
15
6
  end
@@ -1,13 +1,5 @@
1
- if Rails.version.to_f >= 5.0
2
- class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration[Rails.version.to_f]
3
- def change
4
- add_column :canvas_sync_job_logs, :fork_count, :integer
5
- end
6
- end
7
- else
8
- class AddForkCountToCanvasSyncJobLogs < ActiveRecord::Migration
9
- def change
10
- add_column :canvas_sync_job_logs, :fork_count, :integer
11
- end
1
+ class AddForkCountToCanvasSyncJobLogs < CanvasSync::MiscHelper::MigrationClass
2
+ def change
3
+ add_column :canvas_sync_job_logs, :fork_count, :integer
12
4
  end
13
5
  end
@@ -2,6 +2,7 @@ require "bearcat"
2
2
 
3
3
  require "canvas_sync/version"
4
4
  require "canvas_sync/engine"
5
+ require "canvas_sync/misc_helper"
5
6
  require "canvas_sync/class_callback_executor"
6
7
  require "canvas_sync/job"
7
8
  require "canvas_sync/job_chain"
@@ -99,6 +100,7 @@ module CanvasSync
99
100
  invoke_next(job_chain)
100
101
  end
101
102
 
103
+ # @deprecated
102
104
  def duplicate_chain(job_chain)
103
105
  Marshal.load(Marshal.dump(job_chain))
104
106
  end
@@ -109,35 +111,13 @@ module CanvasSync
109
111
  #
110
112
  # @param job_chain [Hash] A chain of jobs to execute
111
113
  def invoke_next(job_chain, extra_options: {})
112
- job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
113
-
114
- return if job_chain[:jobs].empty?
115
-
116
- # Make sure all job classes are serialized as strings
117
- job_chain[:jobs].each { |job| job[:job] = job[:job].to_s }
118
-
119
- duped_job_chain = Marshal.load(Marshal.dump(job_chain))
120
- jobs = duped_job_chain[:jobs]
121
- next_job = jobs.shift
122
- next_job_class = next_job[:job].constantize
123
- next_options = next_job[:options] || {}
124
- next_options.merge!(extra_options)
125
- next_job_class.perform_later(duped_job_chain, next_options)
114
+ job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
115
+ job_chain.perform_next(extra_options)
126
116
  end
127
117
 
128
- def fork(job_log, job_chain, keys: [])
129
- job_chain = job_chain.chain_data if job_chain.is_a?(JobChain)
130
-
131
- duped_job_chain = Marshal.load(Marshal.dump(job_chain))
132
- duped_job_chain[:global_options][:fork_path] ||= []
133
- duped_job_chain[:global_options][:fork_keys] ||= []
134
- duped_job_chain[:global_options][:fork_path] << job_log.job_id
135
- duped_job_chain[:global_options][:fork_keys] << ['canvas_term_id']
136
- duped_job_chain[:global_options][:on_failure] ||= 'CanvasSync::Jobs::ForkGather.handle_branch_error'
137
- sub_items = yield duped_job_chain
138
- sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
139
- job_log.fork_count = sub_count
140
- sub_items
118
+ def fork(job_log, job_chain, keys: [], &blk)
119
+ job_chain = JobChain.new(job_chain) unless job_chain.is_a?(JobChain)
120
+ job_chain.fork(job_log, keys: keys, &blk)
141
121
  end
142
122
 
143
123
  # Given a Model or Relation, scope it down to items that should be synced
@@ -146,6 +126,10 @@ module CanvasSync
146
126
  terms.each do |t|
147
127
  return scope.send(t) if scope.respond_to?(t)
148
128
  end
129
+ model = scope.try(:model) || scope
130
+ if model.try(:column_names)&.include?(:workflow_state)
131
+ return scope.where.not(workflow_state: %w[deleted])
132
+ end
149
133
  Rails.logger.warn("Could not filter Syncable Scope for model '#{scope.try(:model)&.name || scope.name}'")
150
134
  scope
151
135
  end
@@ -29,6 +29,33 @@ module CanvasSync::Concerns
29
29
  end
30
30
  end
31
31
 
32
+ def bulk_sync_from_api_result(api_array, conflict_target: :canvas_id, import_args: {}, all_pages: true, batch_size: 1000)
33
+ columns = api_sync_options.keys
34
+
35
+ update_conditions = {
36
+ condition: Importers::BulkImporter.condition_sql(self, columns),
37
+ columns: columns,
38
+ }
39
+ update_conditions[:conflict_target] = conflict_target if conflict_target.present?
40
+ options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
41
+
42
+ if all_pages
43
+ batcher = BatchProcessor.new(of: batch_size) do |batch|
44
+ import(columns, batch, options)
45
+ end
46
+ api_array.all_pages_each do |api_item|
47
+ item = new.assign_from_api_params(api_items)
48
+ batcher << item
49
+ end
50
+ batcher.flush
51
+ else
52
+ items = api_array.map do |api_item|
53
+ new.assign_from_api_params(api_items)
54
+ end
55
+ import(columns, batch, options)
56
+ end
57
+ end
58
+
32
59
  def api_sync_options=(opts)
33
60
  @api_sync_options = opts
34
61
  end
@@ -15,13 +15,14 @@ module CanvasSync::Concerns
15
15
  private
16
16
 
17
17
  def legacy_column_apply(cls)
18
- cid_column = "canvas_#{subclass.name.downcase}_id"
19
- column_names = subclass.columns.map(&:name)
18
+ return if cls.abstract_class
19
+ cid_column = "canvas_#{cls.name.downcase}_id"
20
+ column_names = cls.columns.map(&:name)
20
21
  return if column_names.include?('canvas_id') && column_names.include?(cid_column)
21
22
  if column_names.include?('canvas_id')
22
- subclass.alias_attribute(cid_column.to_sym, :canvas_id)
23
+ cls.alias_attribute(cid_column.to_sym, :canvas_id)
23
24
  elsif column_names.include?(cid_column)
24
- subclass.alias_attribute(:canvas_id, cid_column.to_sym)
25
+ cls.alias_attribute(:canvas_id, cid_column.to_sym)
25
26
  end
26
27
  rescue ActiveRecord::StatementInvalid
27
28
  end
@@ -8,6 +8,7 @@ class CreateSubmissions < ActiveRecord::Migration[5.1]
8
8
  t.bigint :canvas_assignment_id
9
9
  t.bigint :canvas_user_id
10
10
  t.datetime :submitted_at
11
+ t.datetime :due_at
11
12
  t.datetime :graded_at
12
13
  t.float :score
13
14
  t.float :points_possible
@@ -14,6 +14,9 @@ class Account < ApplicationRecord
14
14
  primary_key: :canvas_id, foreign_key: :canvas_parent_account_id
15
15
  has_many :groups, primary_key: :canvas_id, foreign_key: :canvas_account_id
16
16
 
17
+ scope :active, -> { where.not(workflow_state: 'deleted') }
18
+ # scope :should_canvas_sync, -> { active } # Optional - uses .active if not given
19
+
17
20
  api_syncable({
18
21
  name: :name,
19
22
  workflow_state: :workflow_state,
@@ -14,6 +14,7 @@ class Submission < ApplicationRecord
14
14
  canvas_user_id: :user_id,
15
15
  submitted_at: :submitted_at,
16
16
  graded_at: :graded_at,
17
+ cached_due_date: :due_at,
17
18
  score: :score,
18
19
  excused: :excused,
19
20
  workflow_state: :workflow_state,
@@ -64,13 +64,12 @@ module CanvasSync
64
64
  columns = columns.dup
65
65
 
66
66
  update_conditions = {
67
- condition: condition_sql(klass, columns),
67
+ condition: condition_sql(klass, columns, import_args[:sync_start_time]),
68
68
  columns: columns
69
69
  }
70
70
  update_conditions[:conflict_target] = conflict_target if conflict_target
71
71
 
72
72
  options = { validate: false, on_duplicate_key_update: update_conditions }.merge(import_args)
73
-
74
73
  options.delete(:on_duplicate_key_update) if options.key?(:on_duplicate_key_ignore)
75
74
  klass.import(columns, rows, options)
76
75
  end
@@ -85,10 +84,14 @@ module CanvasSync
85
84
  # started_at = Time.now
86
85
  # run_the_users_sync!
87
86
  # changed = User.where("updated_at >= ?", started_at)
88
- def self.condition_sql(klass, columns)
87
+ def self.condition_sql(klass, columns, report_start)
89
88
  columns_str = columns.map { |c| "#{klass.quoted_table_name}.#{c}" }.join(", ")
90
89
  excluded_str = columns.map { |c| "EXCLUDED.#{c}" }.join(", ")
91
- "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
90
+ condition_sql = "(#{columns_str}) IS DISTINCT FROM (#{excluded_str})"
91
+ if klass.column_names.include?("updated_at") && report_start
92
+ condition_sql += " AND #{klass.quoted_table_name}.updated_at < '#{report_start}'"
93
+ end
94
+ condition_sql
92
95
  end
93
96
 
94
97
  def self.batch_size
@@ -3,6 +3,8 @@ require "active_job"
3
3
  module CanvasSync
4
4
  # Inherit from this class to build a Job that will log to the canvas_sync_job_logs table
5
5
  class Job < ActiveJob::Base
6
+ attr_reader :job_chain, :job_log
7
+
6
8
  before_enqueue do |job|
7
9
  create_job_log(job)
8
10
  end
@@ -13,7 +15,11 @@ module CanvasSync
13
15
  @job_log.started_at = Time.now
14
16
  @job_log.save
15
17
 
16
- @job_chain = job.arguments[0] if job.arguments[0].is_a?(Hash) && job.arguments[0].include?(:jobs)
18
+ if job.arguments[0].is_a?(Hash) && job.arguments[0].include?(:jobs)
19
+ # @job_chain = JobChain.new(job.arguments[0])
20
+ @job_chain = job.arguments[0]
21
+ job.arguments[0] = @job_chain
22
+ end
17
23
 
18
24
  begin
19
25
  block.call
@@ -22,7 +28,7 @@ module CanvasSync
22
28
  @job_log.exception = "#{e.class}: #{e.message}"
23
29
  @job_log.backtrace = e.backtrace.join('\n')
24
30
  @job_log.status = JobLog::ERROR_STATUS
25
- if @job_chain&.[](:global_options)&.[](:on_failure)&.present?
31
+ if @job_chain&.dig(:global_options, :on_failure)&.present?
26
32
  begin
27
33
  class_name, method = @job_chain[:global_options][:on_failure].split('.')
28
34
  klass = class_name.constantize
@@ -45,7 +45,52 @@ module CanvasSync
45
45
  end
46
46
 
47
47
  def process!(extra_options: {})
48
- CanvasSync::invoke_next(self, extra_options: extra_options)
48
+ perform_next(extra_options)
49
+ end
50
+
51
+ def duplicate
52
+ self.class.new(Marshal.load(Marshal.dump(chain_data)))
53
+ end
54
+
55
+ def normalize!
56
+ @chain_data[:global_options] ||= {}
57
+ end
58
+
59
+ def serialize
60
+ normalize!
61
+ chain_data
62
+ end
63
+
64
+ def perform_next(extra_options = {})
65
+ return if jobs.empty?
66
+
67
+ # Make sure all job classes are serialized as strings
68
+ jobs.each { |job| job[:job] = job[:job].to_s }
69
+
70
+ duped_job_chain = duplicate
71
+
72
+ jobs = duped_job_chain[:jobs]
73
+ next_job = jobs.shift
74
+ next_job_class = next_job[:job].constantize
75
+ next_options = next_job[:options] || {}
76
+ next_options.merge!(extra_options)
77
+ next_job_class.perform_later(duped_job_chain.serialize, next_options)
78
+ end
79
+
80
+ def fork(job_log, keys: [])
81
+ duped_job_chain = duplicate
82
+ duped_job_chain[:fork_state] ||= {}
83
+ duped_job_chain[:fork_state][:forking_path] ||= []
84
+ duped_job_chain[:fork_state][:pre_fork_globals] ||= []
85
+
86
+ duped_job_chain[:fork_state][:forking_path] << job_log.job_id
87
+ duped_job_chain[:fork_state][:pre_fork_globals] << global_options
88
+ # duped_job_chain[:global_options][:on_failure] ||= ['CanvasSync::Jobs::ForkGather.handle_branch_error']
89
+
90
+ sub_items = yield duped_job_chain
91
+ sub_count = sub_items.respond_to?(:count) ? sub_items.count : sub_items
92
+ job_log.update!(fork_count: sub_count)
93
+ sub_items
49
94
  end
50
95
 
51
96
  private
@@ -4,25 +4,40 @@ module CanvasSync
4
4
  def perform(job_chain, options)
5
5
  forked_job = self.class.forked_at_job(job_chain)
6
6
 
7
- if forked_job.present?
8
- forked_job.with_lock do
9
- forked_job.fork_count -= 1
10
- forked_job.save!
11
- end
7
+ while true
8
+ if forked_job.present?
9
+ forked_job.with_lock do
10
+ forked_job.fork_count -= 1
11
+ forked_job.save!
12
+ end
13
+
14
+ if forked_job.fork_count <= 0
15
+ pfgs = job_chain[:fork_state][:pre_fork_globals].pop
16
+ job_chain[:global_options] = pfgs
12
17
 
13
- if forked_job.fork_count <= 0
14
- (job_chain[:global_options][:fork_keys] || []).pop&.each do |k|
15
- job_chain[:global_options].delete(k.to_sym)
18
+ if options[:gather_all]
19
+ # If we want to gather all, repeat for the next level fork
20
+ forked_job = self.class.forked_at_job(job_chain)
21
+ else
22
+ forked_job = nil
23
+ end
24
+ else
25
+ # If a fork was found and it isn't complete, break the loop before continuing the chain
26
+ break
16
27
  end
17
- CanvasSync.invoke_next(job_chain)
28
+
29
+ # Repeat this logic for [if gather_all] the next fork up, or [if not gather_all] nil
30
+ next
18
31
  end
19
- else
32
+
33
+ # If there is no current fork (either not in a fork, or all forks were closed), continue the chain
20
34
  CanvasSync.invoke_next(job_chain)
35
+ break
21
36
  end
22
37
  end
23
38
 
24
39
  def self.handle_branch_error(e, job_chain:, skip_invoke: false, **kwargs)
25
- return nil unless job_chain&.[](:global_options)&.[](:fork_path).present?
40
+ return nil unless job_chain&.dig(:fork_state, :forking_path).present?
26
41
 
27
42
  duped_chain = CanvasSync.duplicate_chain(job_chain)
28
43
  job_list = duped_chain[:jobs]
@@ -46,7 +61,7 @@ module CanvasSync
46
61
  protected
47
62
 
48
63
  def self.forked_at_job(job_chain)
49
- fork_item = (job_chain[:global_options][:fork_path] || []).pop
64
+ fork_item = (job_chain.dig(:fork_state, :forking_path) || []).pop
50
65
 
51
66
  if fork_item.present?
52
67
  CanvasSync::JobLog.find_by(job_id: fork_item)
@@ -12,7 +12,7 @@ module CanvasSync
12
12
  # @return [nil]
13
13
  def perform(job_chain, report_name, report_params, processor, options, allow_redownloads: false)
14
14
  account_id = options[:account_id] || job_chain[:global_options][:account_id] || "self"
15
-
15
+ options[:sync_start_time] = DateTime.now.utc.iso8601
16
16
  report_id = if allow_redownloads
17
17
  get_cached_report(job_chain, account_id, report_name, report_params)
18
18
  else
@@ -8,15 +8,15 @@ module CanvasSync
8
8
  # models to sync.
9
9
  def perform(job_chain, options)
10
10
  if options[:term_scope]
11
- sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |job_chain|
11
+ sub_reports = CanvasSync.fork(@job_log, job_chain, keys: [:canvas_term_id]) do |fork_template|
12
12
  Term.send(options[:term_scope]).find_each.map do |term|
13
+ fork = fork_template.duplicate
13
14
  # Deep copy the job_chain so each report gets the correct term id passed into
14
15
  # its options with no side effects
15
16
  term_id = get_term_id(term)
16
- duped_job_chain = Marshal.load(Marshal.dump(job_chain))
17
- duped_job_chain[:global_options][:canvas_term_id] = term_id
17
+ fork[:global_options][:canvas_term_id] = term_id
18
18
  {
19
- job_chain: duped_job_chain,
19
+ job_chain: fork.serialize,
20
20
  params: report_params(options, term_id),
21
21
  options: options,
22
22
  }
@@ -31,7 +31,7 @@ module CanvasSync
31
31
  end
32
32
  end
33
33
 
34
- private
34
+ protected
35
35
 
36
36
  def start_report(report_params, job_chain, options)
37
37
  CanvasSync::Jobs::ReportStarter.perform_later(