canvas_sync 0.17.0.beta14 → 0.17.3.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +58 -0
  3. data/lib/canvas_sync.rb +24 -5
  4. data/lib/canvas_sync/importers/bulk_importer.rb +7 -4
  5. data/lib/canvas_sync/job_batches/batch.rb +75 -95
  6. data/lib/canvas_sync/job_batches/callback.rb +19 -29
  7. data/lib/canvas_sync/job_batches/context_hash.rb +8 -5
  8. data/lib/canvas_sync/job_batches/hincr_max.lua +5 -0
  9. data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +99 -0
  10. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +6 -65
  11. data/lib/canvas_sync/job_batches/pool.rb +209 -0
  12. data/lib/canvas_sync/job_batches/redis_model.rb +67 -0
  13. data/lib/canvas_sync/job_batches/redis_script.rb +163 -0
  14. data/lib/canvas_sync/job_batches/sidekiq.rb +22 -1
  15. data/lib/canvas_sync/job_batches/status.rb +0 -5
  16. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +4 -2
  17. data/lib/canvas_sync/jobs/report_starter.rb +1 -0
  18. data/lib/canvas_sync/processors/assignment_groups_processor.rb +3 -2
  19. data/lib/canvas_sync/processors/assignments_processor.rb +3 -2
  20. data/lib/canvas_sync/processors/context_module_items_processor.rb +3 -2
  21. data/lib/canvas_sync/processors/context_modules_processor.rb +3 -2
  22. data/lib/canvas_sync/processors/normal_processor.rb +2 -1
  23. data/lib/canvas_sync/processors/provisioning_report_processor.rb +10 -2
  24. data/lib/canvas_sync/processors/submissions_processor.rb +3 -2
  25. data/lib/canvas_sync/version.rb +1 -1
  26. data/spec/dummy/log/test.log +67741 -0
  27. data/spec/job_batching/batch_aware_job_spec.rb +1 -0
  28. data/spec/job_batching/batch_spec.rb +72 -15
  29. data/spec/job_batching/callback_spec.rb +1 -1
  30. data/spec/job_batching/flow_spec.rb +0 -1
  31. data/spec/job_batching/integration/fail_then_succeed.rb +42 -0
  32. data/spec/job_batching/integration_helper.rb +6 -4
  33. data/spec/job_batching/sidekiq_spec.rb +1 -0
  34. data/spec/job_batching/status_spec.rb +1 -17
  35. metadata +9 -2
@@ -0,0 +1,67 @@
1
+ module CanvasSync
2
+ module JobBatches
3
+ module RedisModel
4
+ extend ActiveSupport::Concern
5
+
6
+ class_methods do
7
+ def redis_attr(key, type = :string, read_only: true)
8
+ class_eval <<-RUBY, __FILE__, __LINE__ + 1
9
+ def #{key}=(value)
10
+ raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
11
+ @#{key} = value
12
+ if :#{type} == :json
13
+ value = JSON.unparse(value)
14
+ end
15
+ persist_bid_attr('#{key}', value)
16
+ end
17
+
18
+ def #{key}
19
+ return @#{key} if defined?(@#{key})
20
+ if (@initialized || @existing)
21
+ value = read_bid_attr('#{key}')
22
+ if :#{type} == :bool
23
+ value = value == 'true'
24
+ elsif :#{type} == :int
25
+ value = value.to_i
26
+ elsif :#{type} == :float
27
+ value = value.to_f
28
+ elsif :#{type} == :json
29
+ value = JSON.parse(value)
30
+ end
31
+ @#{key} = value
32
+ end
33
+ end
34
+ RUBY
35
+ end
36
+ end
37
+
38
+ def persist_bid_attr(attribute, value)
39
+ if @initialized || @existing
40
+ redis do |r|
41
+ r.multi do
42
+ r.hset(redis_key, attribute, value)
43
+ r.expire(redis_key, Batch::BID_EXPIRE_TTL)
44
+ end
45
+ end
46
+ else
47
+ @pending_attrs ||= {}
48
+ @pending_attrs[attribute] = value
49
+ end
50
+ end
51
+
52
+ def read_bid_attr(attribute)
53
+ redis do |r|
54
+ r.hget(redis_key, attribute)
55
+ end
56
+ end
57
+
58
+ def flush_pending_attrs
59
+ redis do |r|
60
+ r.mapped_hmset(redis_key, @pending_attrs)
61
+ end
62
+ @initialized = true
63
+ @pending_attrs = {}
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,163 @@
1
+ require 'pathname'
2
+ require 'digest/sha1'
3
+ require 'erb'
4
+
5
+ # Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
6
+
7
+ module CanvasSync
8
+ module JobBatches
9
+ # {RedisScript} represents a lua script in the filesystem. It loads the script
10
+ # from disk and handles talking to redis to execute it. Error handling
11
+ # is handled by {LuaError}.
12
+ class RedisScript
13
+
14
+ # Loads the script file from disk and calculates its +SHA1+ sum.
15
+ #
16
+ # @param file [Pathname] the full path to the indicated file
17
+ def initialize(file)
18
+ @file = Pathname.new(file)
19
+ end
20
+
21
+ # Passes the script and supplied arguments to redis for evaulation.
22
+ # It first attempts to use a script redis has already cached by using
23
+ # the +EVALSHA+ command, but falls back to providing the full script
24
+ # text via +EVAL+ if redis has not seen this script before. Future
25
+ # invocations will then use +EVALSHA+ without erroring.
26
+ #
27
+ # @param redis [Redis] the redis connection to run against
28
+ # @param args [*Objects] the arguments to the script
29
+ # @return [Object] the value passed back by redis after script execution
30
+ # @raise [LuaError] if the script failed to compile of encountered a
31
+ # runtime error
32
+ def call(redis, *args)
33
+ t = Time.now
34
+ begin
35
+ redis.evalsha(digest, *args)
36
+ rescue => e
37
+ e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
38
+ end
39
+ rescue => e
40
+ if LuaError.intercepts?(e)
41
+ raise LuaError.new(e, @file, content)
42
+ else
43
+ raise
44
+ end
45
+ end
46
+
47
+ def content
48
+ @content ||= load_lua(@file)
49
+ end
50
+
51
+ def digest
52
+ @digest ||= Digest::SHA1.hexdigest content
53
+ end
54
+
55
+ private
56
+
57
+ def script_path
58
+ Rails.root + 'app/redis_lua'
59
+ end
60
+
61
+ def relative_path
62
+ @path ||= @file.relative_path_from(script_path)
63
+ end
64
+
65
+ def load_lua(file)
66
+ TemplateContext.new(script_path).template(script_path + file)
67
+ end
68
+
69
+ class TemplateContext
70
+ def initialize(script_path)
71
+ @script_path = script_path
72
+ end
73
+
74
+ def template(pathname)
75
+ @partial_templates ||= {}
76
+ ERB.new(File.read(pathname)).result binding
77
+ end
78
+
79
+ # helper method to include a lua partial within another lua script
80
+ #
81
+ # @param relative_path [String] the relative path to the script from
82
+ # `script_path`
83
+ def include_partial(relative_path)
84
+ unless @partial_templates.has_key? relative_path
85
+ @partial_templates[relative_path] = nil
86
+ template( Pathname.new("#{@script_path}/#{relative_path}") )
87
+ end
88
+ end
89
+ end
90
+
91
+ # Reformats errors raised by redis representing failures while executing
92
+ # a lua script. The default errors have confusing messages and backtraces,
93
+ # and a type of +RuntimeError+. This class improves the message and
94
+ # modifies the backtrace to include the lua script itself in a reasonable
95
+ # way.
96
+ class LuaError < StandardError
97
+ PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
98
+ WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
99
+ CONTEXT_LINE_NUMBER = 2
100
+
101
+ attr_reader :error, :file, :content
102
+
103
+ # Is this error one that should be reformatted?
104
+ #
105
+ # @param error [StandardError] the original error raised by redis
106
+ # @return [Boolean] is this an error that should be reformatted?
107
+ def self.intercepts? error
108
+ error.message =~ PATTERN
109
+ end
110
+
111
+ # Initialize a new {LuaError} from an existing redis error, adjusting
112
+ # the message and backtrace in the process.
113
+ #
114
+ # @param error [StandardError] the original error raised by redis
115
+ # @param file [Pathname] full path to the lua file the error ocurred in
116
+ # @param content [String] lua file content the error ocurred in
117
+ def initialize error, file, content
118
+ @error = error
119
+ @file = file
120
+ @content = content
121
+
122
+ @error.message =~ PATTERN
123
+ _stage, line_number, message = $1, $2, $3
124
+ error_context = generate_error_context(content, line_number.to_i)
125
+
126
+ super "#{message}\n\n#{error_context}\n\n"
127
+ set_backtrace generate_backtrace file, line_number
128
+ end
129
+
130
+ private
131
+
132
+ def generate_error_context(content, line_number)
133
+ lines = content.lines.to_a
134
+ beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
135
+ ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
136
+ line_number_width = ending_line_number.to_s.length
137
+
138
+ (beginning_line_number..ending_line_number).map do |number|
139
+ indicator = number == line_number ? '=>' : ' '
140
+ formatted_number = "%#{line_number_width}d" % number
141
+ " #{indicator} #{formatted_number}: #{lines[number - 1]}"
142
+ end.join.chomp
143
+ end
144
+
145
+ def generate_backtrace(file, line_number)
146
+ pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
147
+ index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
148
+ pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
149
+ pre_wolverine.unshift("#{file}:#{line_number}")
150
+ pre_wolverine
151
+ end
152
+
153
+ def backtrace_before_entering_wolverine(backtrace)
154
+ backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
155
+ end
156
+
157
+ def line_from_wolverine(line)
158
+ line.split(':').first.include?(WOLVERINE_LIB_PATH)
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end
@@ -15,11 +15,31 @@ module CanvasSync
15
15
  Thread.current[:batch]
16
16
  end
17
17
 
18
+ def batch_context
19
+ batch&.context || {}
20
+ end
21
+
18
22
  def valid_within_batch?
19
23
  batch.valid?
20
24
  end
21
25
  end
22
26
 
27
+ class SidekiqCallbackWorker
28
+ include ::Sidekiq::Worker
29
+ include WorkerExtension
30
+ include Batch::Callback::CallbackWorkerCommon
31
+
32
+ def self.enqueue_all(args, queue)
33
+ return if args.empty?
34
+
35
+ ::Sidekiq::Client.push_bulk(
36
+ 'class' => self,
37
+ 'args' => args,
38
+ 'queue' => queue
39
+ )
40
+ end
41
+ end
42
+
23
43
  class ClientMiddleware
24
44
  def call(_worker, msg, _queue, _redis_pool = nil)
25
45
  if (batch = Thread.current[:batch]) && should_handle_batch?(msg)
@@ -29,7 +49,7 @@ module CanvasSync
29
49
  end
30
50
 
31
51
  def should_handle_batch?(msg)
32
- return false if msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && msg['wrapped'].constantize.is_a?(BatchAwareJob)
52
+ return false if msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && msg['wrapped'].constantize < BatchAwareJob
33
53
  true
34
54
  end
35
55
  end
@@ -92,6 +112,7 @@ module CanvasSync
92
112
  # This alias helps apartment-sidekiq set itself up correctly
93
113
  ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
94
114
  ::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
115
+ Batch::Callback.worker_class = SidekiqCallbackWorker
95
116
  end
96
117
  end
97
118
  end
@@ -24,10 +24,6 @@ module CanvasSync
24
24
  Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
25
25
  end
26
26
 
27
- def total
28
- Batch.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
29
- end
30
-
31
27
  def parent_bid
32
28
  Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
33
29
  end
@@ -63,7 +59,6 @@ module CanvasSync
63
59
  def data
64
60
  {
65
61
  bid: bid,
66
- total: total,
67
62
  failures: failures,
68
63
  pending: pending,
69
64
  created_at: created_at,
@@ -6,7 +6,9 @@ module CanvasSync
6
6
  def perform(chain_definition, globals = {})
7
7
  @globals = globals
8
8
 
9
- if !globals[:updated_after].present? || globals[:updated_after] == true
9
+ if globals[:updated_after] == false
10
+ globals[:updated_after] = nil
11
+ elsif !globals[:updated_after].present? || globals[:updated_after] == true
10
12
  last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
11
13
  globals[:full_sync_every] ||= "sunday/2"
12
14
  globals[:updated_after] = last_batch&.started_at&.iso8601
@@ -39,7 +41,7 @@ module CanvasSync
39
41
  return true unless last_full_sync.present?
40
42
  return false unless opt.is_a?(String)
41
43
 
42
- case r.strip
44
+ case opt.strip
43
45
  when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
44
46
  m = Regexp.last_match
45
47
  day = m[1]
@@ -11,6 +11,7 @@ module CanvasSync
11
11
  # @return [nil]
12
12
  def perform(report_name, report_params, processor, options, allow_redownloads: false)
13
13
  account_id = options[:account_id] || batch_context[:account_id] || "self"
14
+ options[:sync_start_time] = DateTime.now.utc.iso8601
14
15
 
15
16
  report_id = start_report(account_id, report_name, report_params)
16
17
  # TODO: Restore report caching support (does nayone actually use it?)
@@ -8,15 +8,16 @@ module CanvasSync
8
8
  # @param options [Hash]
9
9
  class AssignmentGroupsProcessor < ReportProcessor
10
10
  def self.process(report_file_path, _options, report_id)
11
- new(report_file_path)
11
+ new(report_file_path, _options)
12
12
  end
13
13
 
14
- def initialize(report_file_path)
14
+ def initialize(report_file_path, options)
15
15
  CanvasSync::Importers::BulkImporter.import(
16
16
  report_file_path,
17
17
  mapping[:assignment_groups][:report_columns],
18
18
  AssignmentGroup,
19
19
  mapping[:assignment_groups][:conflict_target].to_sym,
20
+ import_args: options
20
21
  )
21
22
  end
22
23
  end
@@ -8,15 +8,16 @@ module CanvasSync
8
8
  # @param options [Hash]
9
9
  class AssignmentsProcessor < ReportProcessor
10
10
  def self.process(report_file_path, _options, report_id)
11
- new(report_file_path)
11
+ new(report_file_path, _options)
12
12
  end
13
13
 
14
- def initialize(report_file_path)
14
+ def initialize(report_file_path, options)
15
15
  CanvasSync::Importers::BulkImporter.import(
16
16
  report_file_path,
17
17
  mapping[:assignments][:report_columns],
18
18
  Assignment,
19
19
  mapping[:assignments][:conflict_target].to_sym,
20
+ import_args: options
20
21
  )
21
22
  end
22
23
  end
@@ -8,15 +8,16 @@ module CanvasSync
8
8
  # @param options [Hash]
9
9
  class ContextModuleItemsProcessor < ReportProcessor
10
10
  def self.process(report_file_path, _options, report_id)
11
- new(report_file_path)
11
+ new(report_file_path, _options)
12
12
  end
13
13
 
14
- def initialize(report_file_path)
14
+ def initialize(report_file_path, options)
15
15
  CanvasSync::Importers::BulkImporter.import(
16
16
  report_file_path,
17
17
  mapping[:context_module_items][:report_columns],
18
18
  ContextModuleItem,
19
19
  mapping[:context_module_items][:conflict_target].to_sym,
20
+ import_args: options
20
21
  )
21
22
  end
22
23
  end
@@ -8,15 +8,16 @@ module CanvasSync
8
8
  # @param options [Hash]
9
9
  class ContextModulesProcessor < ReportProcessor
10
10
  def self.process(report_file_path, _options, report_id)
11
- new(report_file_path)
11
+ new(report_file_path, _options)
12
12
  end
13
13
 
14
- def initialize(report_file_path)
14
+ def initialize(report_file_path, options)
15
15
  CanvasSync::Importers::BulkImporter.import(
16
16
  report_file_path,
17
17
  mapping[:context_modules][:report_columns],
18
18
  ContextModule,
19
19
  mapping[:context_modules][:conflict_target].to_sym,
20
+ import_args: options
20
21
  )
21
22
  end
22
23
  end
@@ -18,7 +18,8 @@ module CanvasSync
18
18
  report_file_path,
19
19
  mapping[options[:mapping].to_sym][:report_columns],
20
20
  options[:klass].constantize,
21
- conflict_target ? conflict_target.to_sym : conflict_target
21
+ conflict_target ? conflict_target.to_sym : conflict_target,
22
+ import_args: options
22
23
  )
23
24
  end
24
25
  end
@@ -21,7 +21,6 @@ module CanvasSync
21
21
 
22
22
  def initialize(report_file_path, options) # rubocop:disable Metrics/AbcSize
23
23
  @options = options
24
-
25
24
  if options[:models].length == 1
26
25
  run_import(options[:models][0], report_file_path)
27
26
  else
@@ -75,6 +74,7 @@ module CanvasSync
75
74
  mapping[:users][:report_columns],
76
75
  User,
77
76
  mapping[:users][:conflict_target].to_sym,
77
+ import_args: @options
78
78
  )
79
79
  end
80
80
 
@@ -84,6 +84,7 @@ module CanvasSync
84
84
  mapping[:pseudonyms][:report_columns],
85
85
  Pseudonym,
86
86
  mapping[:pseudonyms][:conflict_target].to_sym,
87
+ import_args: @options
87
88
  )
88
89
  end
89
90
 
@@ -92,7 +93,8 @@ module CanvasSync
92
93
  report_file_path,
93
94
  mapping[:accounts][:report_columns],
94
95
  Account,
95
- mapping[:accounts][:conflict_target].to_sym
96
+ mapping[:accounts][:conflict_target].to_sym,
97
+ import_args: @options
96
98
  )
97
99
  end
98
100
 
@@ -102,6 +104,7 @@ module CanvasSync
102
104
  mapping[:courses][:report_columns],
103
105
  Course,
104
106
  mapping[:courses][:conflict_target].to_sym,
107
+ import_args: @options
105
108
  )
106
109
  end
107
110
 
@@ -111,6 +114,7 @@ module CanvasSync
111
114
  mapping[:enrollments][:report_columns],
112
115
  Enrollment,
113
116
  mapping[:enrollments][:conflict_target].to_sym,
117
+ import_args: @options
114
118
  )
115
119
  end
116
120
 
@@ -120,6 +124,7 @@ module CanvasSync
120
124
  mapping[:sections][:report_columns],
121
125
  Section,
122
126
  mapping[:sections][:conflict_target].to_sym,
127
+ import_args: @options
123
128
  )
124
129
  end
125
130
 
@@ -129,6 +134,7 @@ module CanvasSync
129
134
  mapping[:xlist][:report_columns],
130
135
  Section,
131
136
  mapping[:xlist][:conflict_target].to_sym,
137
+ import_args: @options
132
138
  )
133
139
  end
134
140
 
@@ -138,6 +144,7 @@ module CanvasSync
138
144
  mapping[:groups][:report_columns],
139
145
  Group,
140
146
  mapping[:groups][:conflict_target].to_sym,
147
+ import_args: @options
141
148
  )
142
149
  end
143
150
 
@@ -148,6 +155,7 @@ module CanvasSync
148
155
  mapping[:group_memberships][:report_columns],
149
156
  GroupMembership,
150
157
  mapping[:group_memberships][:conflict_target].to_sym,
158
+ import_args: @options
151
159
  )
152
160
  end
153
161
  end