canvas_sync 0.16.4 → 0.17.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +49 -137
  3. data/app/models/canvas_sync/sync_batch.rb +5 -0
  4. data/db/migrate/20170915210836_create_canvas_sync_job_log.rb +12 -31
  5. data/db/migrate/20180725155729_add_job_id_to_canvas_sync_job_logs.rb +4 -13
  6. data/db/migrate/20190916154829_add_fork_count_to_canvas_sync_job_logs.rb +3 -11
  7. data/db/migrate/20201018210836_create_canvas_sync_sync_batches.rb +11 -0
  8. data/lib/canvas_sync.rb +35 -118
  9. data/lib/canvas_sync/job.rb +5 -5
  10. data/lib/canvas_sync/job_batches/batch.rb +399 -0
  11. data/lib/canvas_sync/job_batches/batch_aware_job.rb +62 -0
  12. data/lib/canvas_sync/job_batches/callback.rb +153 -0
  13. data/lib/canvas_sync/job_batches/chain_builder.rb +203 -0
  14. data/lib/canvas_sync/job_batches/context_hash.rb +147 -0
  15. data/lib/canvas_sync/job_batches/jobs/base_job.rb +7 -0
  16. data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +18 -0
  17. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +73 -0
  18. data/lib/canvas_sync/job_batches/sidekiq.rb +91 -0
  19. data/lib/canvas_sync/job_batches/status.rb +63 -0
  20. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +34 -0
  21. data/lib/canvas_sync/jobs/report_checker.rb +3 -6
  22. data/lib/canvas_sync/jobs/report_processor_job.rb +2 -5
  23. data/lib/canvas_sync/jobs/report_starter.rb +27 -19
  24. data/lib/canvas_sync/jobs/sync_accounts_job.rb +3 -5
  25. data/lib/canvas_sync/jobs/sync_admins_job.rb +2 -4
  26. data/lib/canvas_sync/jobs/sync_assignment_groups_job.rb +2 -4
  27. data/lib/canvas_sync/jobs/sync_assignments_job.rb +2 -4
  28. data/lib/canvas_sync/jobs/sync_context_module_items_job.rb +2 -4
  29. data/lib/canvas_sync/jobs/sync_context_modules_job.rb +2 -4
  30. data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb +4 -31
  31. data/lib/canvas_sync/jobs/sync_roles_job.rb +2 -5
  32. data/lib/canvas_sync/jobs/sync_simple_table_job.rb +11 -32
  33. data/lib/canvas_sync/jobs/sync_submissions_job.rb +2 -4
  34. data/lib/canvas_sync/jobs/sync_terms_job.rb +22 -7
  35. data/lib/canvas_sync/misc_helper.rb +15 -0
  36. data/lib/canvas_sync/version.rb +1 -1
  37. data/spec/canvas_sync/canvas_sync_spec.rb +126 -153
  38. data/spec/canvas_sync/jobs/job_spec.rb +9 -17
  39. data/spec/canvas_sync/jobs/report_checker_spec.rb +1 -3
  40. data/spec/canvas_sync/jobs/report_processor_job_spec.rb +0 -3
  41. data/spec/canvas_sync/jobs/report_starter_spec.rb +19 -28
  42. data/spec/canvas_sync/jobs/sync_admins_job_spec.rb +1 -4
  43. data/spec/canvas_sync/jobs/sync_assignment_groups_job_spec.rb +2 -1
  44. data/spec/canvas_sync/jobs/sync_assignments_job_spec.rb +3 -2
  45. data/spec/canvas_sync/jobs/sync_context_module_items_job_spec.rb +3 -2
  46. data/spec/canvas_sync/jobs/sync_context_modules_job_spec.rb +3 -2
  47. data/spec/canvas_sync/jobs/sync_provisioning_report_job_spec.rb +3 -35
  48. data/spec/canvas_sync/jobs/sync_roles_job_spec.rb +1 -4
  49. data/spec/canvas_sync/jobs/sync_simple_table_job_spec.rb +5 -12
  50. data/spec/canvas_sync/jobs/sync_submissions_job_spec.rb +2 -1
  51. data/spec/canvas_sync/jobs/sync_terms_job_spec.rb +1 -4
  52. data/spec/dummy/app/models/account.rb +3 -0
  53. data/spec/dummy/app/models/pseudonym.rb +14 -0
  54. data/spec/dummy/app/models/submission.rb +1 -0
  55. data/spec/dummy/app/models/user.rb +1 -0
  56. data/spec/dummy/config/environments/test.rb +2 -0
  57. data/spec/dummy/db/migrate/20201016181346_create_pseudonyms.rb +24 -0
  58. data/spec/dummy/db/schema.rb +24 -4
  59. data/spec/job_batching/batch_aware_job_spec.rb +100 -0
  60. data/spec/job_batching/batch_spec.rb +363 -0
  61. data/spec/job_batching/callback_spec.rb +38 -0
  62. data/spec/job_batching/flow_spec.rb +91 -0
  63. data/spec/job_batching/integration/integration.rb +57 -0
  64. data/spec/job_batching/integration/nested.rb +88 -0
  65. data/spec/job_batching/integration/simple.rb +47 -0
  66. data/spec/job_batching/integration/workflow.rb +134 -0
  67. data/spec/job_batching/integration_helper.rb +48 -0
  68. data/spec/job_batching/sidekiq_spec.rb +124 -0
  69. data/spec/job_batching/status_spec.rb +92 -0
  70. data/spec/job_batching/support/base_job.rb +14 -0
  71. data/spec/job_batching/support/sample_callback.rb +2 -0
  72. data/spec/spec_helper.rb +10 -0
  73. metadata +90 -8
  74. data/lib/canvas_sync/job_chain.rb +0 -57
  75. data/lib/canvas_sync/jobs/fork_gather.rb +0 -59
  76. data/spec/canvas_sync/jobs/fork_gather_spec.rb +0 -73
@@ -0,0 +1,153 @@
1
+ module CanvasSync
2
+ module JobBatches
3
+ class Batch
4
+ module Callback
5
+
6
+ VALID_CALLBACKS = %w[success complete dead].freeze
7
+
8
+ module CallbackWorkerCommon
9
+ def perform(definition, event, opts, bid, parent_bid)
10
+ return unless VALID_CALLBACKS.include?(event)
11
+
12
+ method = nil
13
+ target = :instance
14
+ clazz = definition
15
+ if clazz.is_a?(String)
16
+ if clazz.include?('#')
17
+ clazz, method = clazz.split("#")
18
+ elsif clazz.include?('.')
19
+ clazz, method = clazz.split(".")
20
+ target = :class
21
+ end
22
+ end
23
+
24
+ method ||= "on_#{event}"
25
+ status = Batch::Status.new(bid)
26
+
27
+ if clazz && object = Object.const_get(clazz)
28
+ target = target == :instance ? object.new : object
29
+ if target.respond_to?(method)
30
+ target.send(method, status, opts)
31
+ else
32
+ Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")
33
+ end
34
+ else
35
+ Batch.logger.warn("Invalid callback method #{definition} - Class #{clazz} not found")
36
+ end
37
+ end
38
+ end
39
+
40
+ class ActiveJobCallbackWorker < ActiveJob::Base
41
+ include CallbackWorkerCommon
42
+
43
+ def self.enqueue_all(args, queue)
44
+ args.each do |arg_set|
45
+ set(queue: queue).perform_later(*arg_set)
46
+ end
47
+ end
48
+ end
49
+
50
+ if defined?(::Sidekiq)
51
+ class SidekiqCallbackWorker
52
+ include ::Sidekiq::Worker
53
+ include CallbackWorkerCommon
54
+
55
+ def self.enqueue_all(args, queue)
56
+ return if args.empty?
57
+
58
+ ::Sidekiq::Client.push_bulk(
59
+ 'class' => self,
60
+ 'args' => args,
61
+ 'queue' => queue
62
+ )
63
+ end
64
+ end
65
+ Worker = SidekiqCallbackWorker
66
+ else
67
+ Worker = ActiveJobCallbackWorker
68
+ end
69
+
70
+ class Finalize
71
+ def dispatch status, opts
72
+ bid = opts["bid"]
73
+ callback_bid = status.bid
74
+ event = opts["event"].to_sym
75
+ callback_batch = bid != callback_bid
76
+
77
+ Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
78
+
79
+ batch_status = Status.new bid
80
+ send(event, bid, batch_status, batch_status.parent_bid)
81
+
82
+ # Different events are run in different callback batches
83
+ Batch.cleanup_redis callback_bid if callback_batch
84
+ Batch.cleanup_redis bid if event == :success
85
+ end
86
+
87
+ def success(bid, status, parent_bid)
88
+ return unless parent_bid
89
+
90
+ _, _, success, _, complete, pending, children, failure = Batch.redis do |r|
91
+ r.multi do
92
+ r.sadd("BID-#{parent_bid}-success", bid)
93
+ r.expire("BID-#{parent_bid}-success", Batch::BID_EXPIRE_TTL)
94
+ r.scard("BID-#{parent_bid}-success")
95
+ r.sadd("BID-#{parent_bid}-complete", bid)
96
+ r.scard("BID-#{parent_bid}-complete")
97
+ r.hincrby("BID-#{parent_bid}", "pending", 0)
98
+ r.hincrby("BID-#{parent_bid}", "children", 0)
99
+ r.scard("BID-#{parent_bid}-failed")
100
+ end
101
+ end
102
+ # if job finished successfully and parent batch completed call parent complete callback
103
+ # Success callback is called after complete callback
104
+ if complete == children && pending == failure
105
+ Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
106
+ Batch.enqueue_callbacks(:complete, parent_bid)
107
+ end
108
+ end
109
+
110
+ def complete(bid, status, parent_bid)
111
+ pending, children, success = Batch.redis do |r|
112
+ r.multi do
113
+ r.hincrby("BID-#{bid}", "pending", 0)
114
+ r.hincrby("BID-#{bid}", "children", 0)
115
+ r.scard("BID-#{bid}-success")
116
+ end
117
+ end
118
+
119
+ # if we batch was successful run success callback
120
+ if pending.to_i.zero? && children == success
121
+ Batch.enqueue_callbacks(:success, bid)
122
+
123
+ elsif parent_bid
124
+ # if batch was not successfull check and see if its parent is complete
125
+ # if the parent is complete we trigger the complete callback
126
+ # We don't want to run this if the batch was successfull because the success
127
+ # callback may add more jobs to the parent batch
128
+
129
+ Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
130
+ _, complete, pending, children, failure = Batch.redis do |r|
131
+ r.multi do
132
+ r.sadd("BID-#{parent_bid}-complete", bid)
133
+ r.scard("BID-#{parent_bid}-complete")
134
+ r.hincrby("BID-#{parent_bid}", "pending", 0)
135
+ r.hincrby("BID-#{parent_bid}", "children", 0)
136
+ r.scard("BID-#{parent_bid}-failed")
137
+ end
138
+ end
139
+ if complete == children && pending == failure
140
+ Batch.enqueue_callbacks(:complete, parent_bid)
141
+ end
142
+ end
143
+ end
144
+
145
+ def cleanup_redis bid, callback_bid=nil
146
+ Batch.cleanup_redis bid
147
+ Batch.cleanup_redis callback_bid if callback_bid
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,203 @@
1
+ module CanvasSync
2
+ module JobBatches
3
+ class ChainBuilder
4
+ VALID_PLACEMENT_PARAMETERS = %i[before after with].freeze
5
+
6
+ attr_reader :base_job
7
+
8
+ def initialize(base_type = SerialBatchJob)
9
+ if base_type.is_a?(Hash)
10
+ @base_job = base_type
11
+ else
12
+ @base_job = {
13
+ job: base_type,
14
+ parameters: [],
15
+ }
16
+ end
17
+ end
18
+
19
+ def process!
20
+ normalize!
21
+ self.class.enqueue_job(base_job)
22
+ end
23
+
24
+ def [](key)
25
+ if key.is_a?(Class)
26
+ get_sub_chain(key)
27
+ else
28
+ @base_job[key]
29
+ end
30
+ end
31
+
32
+ def params
33
+ ParamsMapper.new(self[:parameters])
34
+ end
35
+
36
+ def <<(new_job)
37
+ insert_at(-1, new_job)
38
+ end
39
+
40
+ def insert_at(position, new_jobs)
41
+ chain = self.class.get_chain_parameter(base_job)
42
+ new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
43
+ chain.insert(-1, *new_jobs)
44
+ end
45
+
46
+ def insert(new_jobs, **kwargs)
47
+ invalid_params = kwargs.keys - VALID_PLACEMENT_PARAMETERS
48
+ raise "Invalid placement parameters: #{invalid_params.map(&:to_s).join(', ')}" if invalid_params.present?
49
+ raise "At most one placement parameter may be provided" if kwargs.values.compact.length > 1
50
+
51
+ new_jobs = [new_jobs] unless new_jobs.is_a?(Array)
52
+
53
+ if !kwargs.present?
54
+ insert_at(-1, new_jobs)
55
+ else
56
+ placement = kwargs.keys[0]
57
+ relative_to = kwargs.values[0]
58
+
59
+ matching_jobs = find_matching_jobs(relative_to)
60
+ raise "Could not find a \"#{relative_to}\" job in the chain" if matching_jobs.count == 0
61
+ raise "Found multiple \"#{relative_to}\" jobs in the chain" if matching_jobs.count > 1
62
+
63
+ parent_job, sub_index = matching_jobs[0]
64
+ chain = self.class.get_chain_parameter(parent_job)
65
+ needed_parent_type = placement == :with ? ConcurrentBatchJob : SerialBatchJob
66
+
67
+ if parent_job[:job] != needed_parent_type
68
+ old_job = chain[sub_index]
69
+ parent_job = chain[sub_index] = {
70
+ job: needed_parent_type,
71
+ parameters: [],
72
+ }
73
+ sub_index = 0
74
+ chain = self.class.get_chain_parameter(parent_job)
75
+ chain << old_job
76
+ end
77
+
78
+ if placement == :with
79
+ chain.insert(-1, *new_jobs)
80
+ else
81
+ sub_index += 1 if placement == :after
82
+ chain.insert(sub_index, *new_jobs)
83
+ end
84
+ end
85
+ end
86
+
87
+ def get_sub_chain(sub_type)
88
+ matching_jobs = find_matching_jobs(sub_type)
89
+ raise "Found multiple \"#{sub_type}\" jobs in the chain" if matching_jobs.count > 1
90
+ return nil if matching_jobs.count == 0
91
+
92
+ new(matching_jobs[0])
93
+ end
94
+
95
+ def normalize!(job_def = self.base_job)
96
+ if job_def.is_a?(ChainBuilder)
97
+ job_def.normalize!
98
+ else
99
+ job_def[:job] = job_def[:job].to_s
100
+ if (chain = self.class.get_chain_parameter(job_def, raise_error: false)).present?
101
+ chain.map! { |sub_job| normalize!(sub_job) }
102
+ end
103
+ job_def
104
+ end
105
+ end
106
+
107
+ private
108
+
109
+ def find_matching_jobs(search_job, parent_job = self.base_job)
110
+ return to_enum(:find_matching_jobs, search_job, parent_job) unless block_given?
111
+
112
+ sub_jobs = self.class.get_chain_parameter(parent_job)
113
+ sub_jobs.each_with_index do |sub_job, i|
114
+ if sub_job[:job].to_s == search_job.to_s
115
+ yield [parent_job, i]
116
+ elsif self.class._job_type_definitions[sub_job[:job]]
117
+ find_matching_jobs(search_job) { |item| yield item }
118
+ end
119
+ end
120
+ end
121
+
122
+ class << self
123
+ def _job_type_definitions
124
+ @job_type_definitions ||= {}
125
+ end
126
+
127
+ def register_chain_job(job_class, chain_parameter, **options)
128
+ _job_type_definitions[job_class.to_s] = {
129
+ **options,
130
+ chain_parameter: chain_parameter,
131
+ }
132
+ end
133
+
134
+ def get_chain_parameter(job_def, raise_error: true)
135
+ unless _job_type_definitions[job_def[:job].to_s].present?
136
+ raise "Job Type #{base_job[:job].to_s} does not accept a sub-chain" if raise_error
137
+ return nil
138
+ end
139
+
140
+ key = _job_type_definitions[job_def[:job].to_s][:chain_parameter]
141
+ mapper = ParamsMapper.new(job_def[:parameters])
142
+ mapper[key] ||= []
143
+ end
144
+
145
+ def enqueue_job(job_def)
146
+ job_class = job_def[:job].constantize
147
+ job_options = job_def[:parameters] || []
148
+ if job_class.respond_to? :perform_async
149
+ job_class.perform_async(*job_options)
150
+ else
151
+ job_class.perform_later(*job_options)
152
+ end
153
+ end
154
+ end
155
+ end
156
+
157
+ ChainBuilder.register_chain_job(ConcurrentBatchJob, 0)
158
+ ChainBuilder.register_chain_job(SerialBatchJob, 0)
159
+
160
+ class ParamsMapper
161
+ def initialize(backend)
162
+ @backend = backend
163
+ end
164
+
165
+ def [](key)
166
+ get_parameter(key)
167
+ end
168
+
169
+ def []=(key, value)
170
+ set_parameter(key, value)
171
+ end
172
+
173
+ def to_a
174
+ @backend
175
+ end
176
+
177
+ private
178
+
179
+ def get_parameter(key)
180
+ if key.is_a?(Numeric)
181
+ @backend[key]
182
+ else
183
+ kwargs = @backend.last
184
+ return nil unless kwargs.is_a?(Hash)
185
+ kwargs[key]
186
+ end
187
+ end
188
+
189
+ def set_parameter(key, value)
190
+ if key.is_a?(Numeric)
191
+ @backend[key] = value
192
+ else
193
+ kwargs = @backend.last
194
+ unless kwargs.is_a?(Hash)
195
+ kwargs = {}
196
+ @backend.push(kwargs)
197
+ end
198
+ kwargs[key] = value
199
+ end
200
+ end
201
+ end
202
+ end
203
+ end
@@ -0,0 +1,147 @@
1
+ module CanvasSync
2
+ module JobBatches
3
+ class ContextHash
4
+ delegate_missing_to :flatten
5
+
6
+ def initialize(bid, hash = nil)
7
+ @bid_stack = [bid]
8
+ @hash_map = {}
9
+ @dirty = false
10
+ @flattened = nil
11
+ @hash_map[bid] = hash.with_indifferent_access if hash
12
+ end
13
+
14
+ # Local is "the nearest batch with a context value"
15
+ # This allows for, for example, SerialBatchJob to have a modifiable context stored on it's main Batch
16
+ # that can be accessed transparently from one of it's internal, context-less Batches
17
+ def local_bid
18
+ bid = @bid_stack[-1]
19
+ while bid.present?
20
+ bhash = reolve_hash(bid)
21
+ return bid if bhash
22
+ bid = get_parent_bid(bid)
23
+ end
24
+ nil
25
+ end
26
+
27
+ def local
28
+ @hash_map[local_bid]
29
+ end
30
+
31
+ def set_local(new_hash)
32
+ @dirty = true
33
+ local.clear.merge!(new_hash)
34
+ end
35
+
36
+ def clear
37
+ local.clear
38
+ @flattened = nil
39
+ @dirty = true
40
+ self
41
+ end
42
+
43
+ def []=(key, value)
44
+ @flattened = nil
45
+ @dirty = true
46
+ local[key] = value
47
+ end
48
+
49
+ def [](key)
50
+ bid = @bid_stack[-1]
51
+ while bid.present?
52
+ bhash = reolve_hash(bid)
53
+ return bhash[key] if bhash&.key?(key)
54
+ bid = get_parent_bid(bid)
55
+ end
56
+ nil
57
+ end
58
+
59
+ def reload!
60
+ @dirty = false
61
+ @hash_map = {}
62
+ self
63
+ end
64
+
65
+ def save!(force: false)
66
+ return unless dirty? || force
67
+ Batch.redis do |r|
68
+ r.hset("BID-#{local_bid}", 'context', JSON.unparse(local))
69
+ end
70
+ end
71
+
72
+ def dirty?
73
+ @dirty
74
+ end
75
+
76
+ def is_a?(arg)
77
+ return true if Hash <= arg
78
+ super
79
+ end
80
+
81
+ def flatten
82
+ return @flattened if @flattened
83
+
84
+ load_all
85
+ flattened = {}
86
+ @bid_stack.compact.each do |bid|
87
+ flattened.merge!(@hash_map[bid]) if @hash_map[bid]
88
+ end
89
+ flattened.freeze
90
+
91
+ @flattened = flattened.with_indifferent_access
92
+ end
93
+
94
+ private
95
+
96
+ def get_parent_hash(bid)
97
+ reolve_hash(get_parent_bid(bid)).freeze
98
+ end
99
+
100
+ def get_parent_bid(bid)
101
+ index = @bid_stack.index(bid)
102
+ raise "Invalid BID #{bid}" if index.nil? # Sanity Check - this shouldn't happen
103
+
104
+ index -= 1
105
+ if index >= 0
106
+ @bid_stack[index]
107
+ else
108
+ pbid = Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
109
+ @bid_stack.unshift(pbid)
110
+ pbid
111
+ end
112
+ end
113
+
114
+ def reolve_hash(bid)
115
+ return nil unless bid.present?
116
+ return @hash_map[bid] if @hash_map.key?(bid)
117
+
118
+ context_json, editable = Batch.redis do |r|
119
+ r.multi do
120
+ r.hget("BID-#{bid}", "context")
121
+ r.hget("BID-#{bid}", "allow_context_changes")
122
+ end
123
+ end
124
+
125
+ if context_json.present?
126
+ context_hash = JSON.parse(context_json)
127
+ context_hash = context_hash.with_indifferent_access
128
+ context_hash.each do |k, v|
129
+ v.freeze
130
+ end
131
+ context_hash.freeze unless editable
132
+
133
+ @hash_map[bid] = context_hash
134
+ else
135
+ @hash_map[bid] = nil
136
+ end
137
+ end
138
+
139
+ def load_all
140
+ while @bid_stack[0].present?
141
+ get_parent_hash(@bid_stack[0])
142
+ end
143
+ @hash_map
144
+ end
145
+ end
146
+ end
147
+ end