canvas_sync 0.17.0.beta15 → 0.17.3.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +58 -0
  3. data/lib/canvas_sync/importers/bulk_importer.rb +7 -4
  4. data/lib/canvas_sync/job_batches/batch.rb +81 -107
  5. data/lib/canvas_sync/job_batches/callback.rb +20 -29
  6. data/lib/canvas_sync/job_batches/context_hash.rb +8 -5
  7. data/lib/canvas_sync/job_batches/hincr_max.lua +5 -0
  8. data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +99 -0
  9. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +6 -65
  10. data/lib/canvas_sync/job_batches/pool.rb +193 -0
  11. data/lib/canvas_sync/job_batches/redis_model.rb +67 -0
  12. data/lib/canvas_sync/job_batches/redis_script.rb +163 -0
  13. data/lib/canvas_sync/job_batches/sidekiq.rb +22 -1
  14. data/lib/canvas_sync/job_batches/status.rb +0 -5
  15. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +4 -2
  16. data/lib/canvas_sync/jobs/report_starter.rb +1 -0
  17. data/lib/canvas_sync/processors/assignment_groups_processor.rb +3 -2
  18. data/lib/canvas_sync/processors/assignments_processor.rb +3 -2
  19. data/lib/canvas_sync/processors/context_module_items_processor.rb +3 -2
  20. data/lib/canvas_sync/processors/context_modules_processor.rb +3 -2
  21. data/lib/canvas_sync/processors/normal_processor.rb +2 -1
  22. data/lib/canvas_sync/processors/provisioning_report_processor.rb +10 -2
  23. data/lib/canvas_sync/processors/submissions_processor.rb +3 -2
  24. data/lib/canvas_sync/version.rb +1 -1
  25. data/spec/dummy/log/test.log +78907 -0
  26. data/spec/job_batching/batch_aware_job_spec.rb +1 -0
  27. data/spec/job_batching/batch_spec.rb +72 -15
  28. data/spec/job_batching/callback_spec.rb +1 -1
  29. data/spec/job_batching/flow_spec.rb +5 -11
  30. data/spec/job_batching/integration/fail_then_succeed.rb +42 -0
  31. data/spec/job_batching/integration_helper.rb +6 -4
  32. data/spec/job_batching/sidekiq_spec.rb +1 -0
  33. data/spec/job_batching/status_spec.rb +1 -17
  34. metadata +9 -2
@@ -0,0 +1,163 @@
1
+ require 'pathname'
2
+ require 'digest/sha1'
3
+ require 'erb'
4
+
5
+ # Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
6
+
7
+ module CanvasSync
8
+ module JobBatches
9
+ # {RedisScript} represents a lua script in the filesystem. It loads the script
10
+ # from disk and handles talking to redis to execute it. Error handling
11
+ # is handled by {LuaError}.
12
+ class RedisScript
13
+
14
+ # Loads the script file from disk and calculates its +SHA1+ sum.
15
+ #
16
+ # @param file [Pathname] the full path to the indicated file
17
+ def initialize(file)
18
+ @file = Pathname.new(file)
19
+ end
20
+
21
+ # Passes the script and supplied arguments to redis for evaulation.
22
+ # It first attempts to use a script redis has already cached by using
23
+ # the +EVALSHA+ command, but falls back to providing the full script
24
+ # text via +EVAL+ if redis has not seen this script before. Future
25
+ # invocations will then use +EVALSHA+ without erroring.
26
+ #
27
+ # @param redis [Redis] the redis connection to run against
28
+ # @param args [*Objects] the arguments to the script
29
+ # @return [Object] the value passed back by redis after script execution
30
+ # @raise [LuaError] if the script failed to compile of encountered a
31
+ # runtime error
32
+ def call(redis, *args)
33
+ t = Time.now
34
+ begin
35
+ redis.evalsha(digest, *args)
36
+ rescue => e
37
+ e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
38
+ end
39
+ rescue => e
40
+ if LuaError.intercepts?(e)
41
+ raise LuaError.new(e, @file, content)
42
+ else
43
+ raise
44
+ end
45
+ end
46
+
47
+ def content
48
+ @content ||= load_lua(@file)
49
+ end
50
+
51
+ def digest
52
+ @digest ||= Digest::SHA1.hexdigest content
53
+ end
54
+
55
+ private
56
+
57
+ def script_path
58
+ Rails.root + 'app/redis_lua'
59
+ end
60
+
61
+ def relative_path
62
+ @path ||= @file.relative_path_from(script_path)
63
+ end
64
+
65
+ def load_lua(file)
66
+ TemplateContext.new(script_path).template(script_path + file)
67
+ end
68
+
69
+ class TemplateContext
70
+ def initialize(script_path)
71
+ @script_path = script_path
72
+ end
73
+
74
+ def template(pathname)
75
+ @partial_templates ||= {}
76
+ ERB.new(File.read(pathname)).result binding
77
+ end
78
+
79
+ # helper method to include a lua partial within another lua script
80
+ #
81
+ # @param relative_path [String] the relative path to the script from
82
+ # `script_path`
83
+ def include_partial(relative_path)
84
+ unless @partial_templates.has_key? relative_path
85
+ @partial_templates[relative_path] = nil
86
+ template( Pathname.new("#{@script_path}/#{relative_path}") )
87
+ end
88
+ end
89
+ end
90
+
91
+ # Reformats errors raised by redis representing failures while executing
92
+ # a lua script. The default errors have confusing messages and backtraces,
93
+ # and a type of +RuntimeError+. This class improves the message and
94
+ # modifies the backtrace to include the lua script itself in a reasonable
95
+ # way.
96
+ class LuaError < StandardError
97
+ PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
98
+ WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
99
+ CONTEXT_LINE_NUMBER = 2
100
+
101
+ attr_reader :error, :file, :content
102
+
103
+ # Is this error one that should be reformatted?
104
+ #
105
+ # @param error [StandardError] the original error raised by redis
106
+ # @return [Boolean] is this an error that should be reformatted?
107
+ def self.intercepts? error
108
+ error.message =~ PATTERN
109
+ end
110
+
111
+ # Initialize a new {LuaError} from an existing redis error, adjusting
112
+ # the message and backtrace in the process.
113
+ #
114
+ # @param error [StandardError] the original error raised by redis
115
+ # @param file [Pathname] full path to the lua file the error ocurred in
116
+ # @param content [String] lua file content the error ocurred in
117
+ def initialize error, file, content
118
+ @error = error
119
+ @file = file
120
+ @content = content
121
+
122
+ @error.message =~ PATTERN
123
+ _stage, line_number, message = $1, $2, $3
124
+ error_context = generate_error_context(content, line_number.to_i)
125
+
126
+ super "#{message}\n\n#{error_context}\n\n"
127
+ set_backtrace generate_backtrace file, line_number
128
+ end
129
+
130
+ private
131
+
132
+ def generate_error_context(content, line_number)
133
+ lines = content.lines.to_a
134
+ beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
135
+ ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
136
+ line_number_width = ending_line_number.to_s.length
137
+
138
+ (beginning_line_number..ending_line_number).map do |number|
139
+ indicator = number == line_number ? '=>' : ' '
140
+ formatted_number = "%#{line_number_width}d" % number
141
+ " #{indicator} #{formatted_number}: #{lines[number - 1]}"
142
+ end.join.chomp
143
+ end
144
+
145
+ def generate_backtrace(file, line_number)
146
+ pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
147
+ index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
148
+ pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
149
+ pre_wolverine.unshift("#{file}:#{line_number}")
150
+ pre_wolverine
151
+ end
152
+
153
+ def backtrace_before_entering_wolverine(backtrace)
154
+ backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
155
+ end
156
+
157
+ def line_from_wolverine(line)
158
+ line.split(':').first.include?(WOLVERINE_LIB_PATH)
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end
@@ -15,11 +15,31 @@ module CanvasSync
15
15
  Thread.current[:batch]
16
16
  end
17
17
 
18
+ def batch_context
19
+ batch&.context || {}
20
+ end
21
+
18
22
  def valid_within_batch?
19
23
  batch.valid?
20
24
  end
21
25
  end
22
26
 
27
+ class SidekiqCallbackWorker
28
+ include ::Sidekiq::Worker
29
+ include WorkerExtension
30
+ include Batch::Callback::CallbackWorkerCommon
31
+
32
+ def self.enqueue_all(args, queue)
33
+ return if args.empty?
34
+
35
+ ::Sidekiq::Client.push_bulk(
36
+ 'class' => self,
37
+ 'args' => args,
38
+ 'queue' => queue
39
+ )
40
+ end
41
+ end
42
+
23
43
  class ClientMiddleware
24
44
  def call(_worker, msg, _queue, _redis_pool = nil)
25
45
  if (batch = Thread.current[:batch]) && should_handle_batch?(msg)
@@ -29,7 +49,7 @@ module CanvasSync
29
49
  end
30
50
 
31
51
  def should_handle_batch?(msg)
32
- return false if msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && msg['wrapped'].constantize.is_a?(BatchAwareJob)
52
+ return false if msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && msg['wrapped'].constantize < BatchAwareJob
33
53
  true
34
54
  end
35
55
  end
@@ -92,6 +112,7 @@ module CanvasSync
92
112
  # This alias helps apartment-sidekiq set itself up correctly
93
113
  ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
94
114
  ::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
115
+ Batch::Callback.worker_class = SidekiqCallbackWorker
95
116
  end
96
117
  end
97
118
  end
@@ -24,10 +24,6 @@ module CanvasSync
24
24
  Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
25
25
  end
26
26
 
27
- def total
28
- Batch.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
29
- end
30
-
31
27
  def parent_bid
32
28
  Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
33
29
  end
@@ -63,7 +59,6 @@ module CanvasSync
63
59
  def data
64
60
  {
65
61
  bid: bid,
66
- total: total,
67
62
  failures: failures,
68
63
  pending: pending,
69
64
  created_at: created_at,
@@ -6,7 +6,9 @@ module CanvasSync
6
6
  def perform(chain_definition, globals = {})
7
7
  @globals = globals
8
8
 
9
- if !globals[:updated_after].present? || globals[:updated_after] == true
9
+ if globals[:updated_after] == false
10
+ globals[:updated_after] = nil
11
+ elsif !globals[:updated_after].present? || globals[:updated_after] == true
10
12
  last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
11
13
  globals[:full_sync_every] ||= "sunday/2"
12
14
  globals[:updated_after] = last_batch&.started_at&.iso8601
@@ -39,7 +41,7 @@ module CanvasSync
39
41
  return true unless last_full_sync.present?
40
42
  return false unless opt.is_a?(String)
41
43
 
42
- case r.strip
44
+ case opt.strip
43
45
  when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
44
46
  m = Regexp.last_match
45
47
  day = m[1]
@@ -11,6 +11,7 @@ module CanvasSync
11
11
  # @return [nil]
12
12
  def perform(report_name, report_params, processor, options, allow_redownloads: false)
13
13
  account_id = options[:account_id] || batch_context[:account_id] || "self"
14
+ options[:sync_start_time] = DateTime.now.utc.iso8601
14
15
 
15
16
  report_id = start_report(account_id, report_name, report_params)
16
17
  # TODO: Restore report caching support (does nayone actually use it?)
@@ -8,15 +8,16 @@ module CanvasSync
8
8
  # @param options [Hash]
9
9
  class AssignmentGroupsProcessor < ReportProcessor
10
10
  def self.process(report_file_path, _options, report_id)
11
- new(report_file_path)
11
+ new(report_file_path, _options)
12
12
  end
13
13
 
14
- def initialize(report_file_path)
14
+ def initialize(report_file_path, options)
15
15
  CanvasSync::Importers::BulkImporter.import(
16
16
  report_file_path,
17
17
  mapping[:assignment_groups][:report_columns],
18
18
  AssignmentGroup,
19
19
  mapping[:assignment_groups][:conflict_target].to_sym,
20
+ import_args: options
20
21
  )
21
22
  end
22
23
  end
@@ -8,15 +8,16 @@ module CanvasSync
8
8
  # @param options [Hash]
9
9
  class AssignmentsProcessor < ReportProcessor
10
10
  def self.process(report_file_path, _options, report_id)
11
- new(report_file_path)
11
+ new(report_file_path, _options)
12
12
  end
13
13
 
14
- def initialize(report_file_path)
14
+ def initialize(report_file_path, options)
15
15
  CanvasSync::Importers::BulkImporter.import(
16
16
  report_file_path,
17
17
  mapping[:assignments][:report_columns],
18
18
  Assignment,
19
19
  mapping[:assignments][:conflict_target].to_sym,
20
+ import_args: options
20
21
  )
21
22
  end
22
23
  end
@@ -8,15 +8,16 @@ module CanvasSync
8
8
  # @param options [Hash]
9
9
  class ContextModuleItemsProcessor < ReportProcessor
10
10
  def self.process(report_file_path, _options, report_id)
11
- new(report_file_path)
11
+ new(report_file_path, _options)
12
12
  end
13
13
 
14
- def initialize(report_file_path)
14
+ def initialize(report_file_path, options)
15
15
  CanvasSync::Importers::BulkImporter.import(
16
16
  report_file_path,
17
17
  mapping[:context_module_items][:report_columns],
18
18
  ContextModuleItem,
19
19
  mapping[:context_module_items][:conflict_target].to_sym,
20
+ import_args: options
20
21
  )
21
22
  end
22
23
  end
@@ -8,15 +8,16 @@ module CanvasSync
8
8
  # @param options [Hash]
9
9
  class ContextModulesProcessor < ReportProcessor
10
10
  def self.process(report_file_path, _options, report_id)
11
- new(report_file_path)
11
+ new(report_file_path, _options)
12
12
  end
13
13
 
14
- def initialize(report_file_path)
14
+ def initialize(report_file_path, options)
15
15
  CanvasSync::Importers::BulkImporter.import(
16
16
  report_file_path,
17
17
  mapping[:context_modules][:report_columns],
18
18
  ContextModule,
19
19
  mapping[:context_modules][:conflict_target].to_sym,
20
+ import_args: options
20
21
  )
21
22
  end
22
23
  end
@@ -18,7 +18,8 @@ module CanvasSync
18
18
  report_file_path,
19
19
  mapping[options[:mapping].to_sym][:report_columns],
20
20
  options[:klass].constantize,
21
- conflict_target ? conflict_target.to_sym : conflict_target
21
+ conflict_target ? conflict_target.to_sym : conflict_target,
22
+ import_args: options
22
23
  )
23
24
  end
24
25
  end
@@ -21,7 +21,6 @@ module CanvasSync
21
21
 
22
22
  def initialize(report_file_path, options) # rubocop:disable Metrics/AbcSize
23
23
  @options = options
24
-
25
24
  if options[:models].length == 1
26
25
  run_import(options[:models][0], report_file_path)
27
26
  else
@@ -75,6 +74,7 @@ module CanvasSync
75
74
  mapping[:users][:report_columns],
76
75
  User,
77
76
  mapping[:users][:conflict_target].to_sym,
77
+ import_args: @options
78
78
  )
79
79
  end
80
80
 
@@ -84,6 +84,7 @@ module CanvasSync
84
84
  mapping[:pseudonyms][:report_columns],
85
85
  Pseudonym,
86
86
  mapping[:pseudonyms][:conflict_target].to_sym,
87
+ import_args: @options
87
88
  )
88
89
  end
89
90
 
@@ -92,7 +93,8 @@ module CanvasSync
92
93
  report_file_path,
93
94
  mapping[:accounts][:report_columns],
94
95
  Account,
95
- mapping[:accounts][:conflict_target].to_sym
96
+ mapping[:accounts][:conflict_target].to_sym,
97
+ import_args: @options
96
98
  )
97
99
  end
98
100
 
@@ -102,6 +104,7 @@ module CanvasSync
102
104
  mapping[:courses][:report_columns],
103
105
  Course,
104
106
  mapping[:courses][:conflict_target].to_sym,
107
+ import_args: @options
105
108
  )
106
109
  end
107
110
 
@@ -111,6 +114,7 @@ module CanvasSync
111
114
  mapping[:enrollments][:report_columns],
112
115
  Enrollment,
113
116
  mapping[:enrollments][:conflict_target].to_sym,
117
+ import_args: @options
114
118
  )
115
119
  end
116
120
 
@@ -120,6 +124,7 @@ module CanvasSync
120
124
  mapping[:sections][:report_columns],
121
125
  Section,
122
126
  mapping[:sections][:conflict_target].to_sym,
127
+ import_args: @options
123
128
  )
124
129
  end
125
130
 
@@ -129,6 +134,7 @@ module CanvasSync
129
134
  mapping[:xlist][:report_columns],
130
135
  Section,
131
136
  mapping[:xlist][:conflict_target].to_sym,
137
+ import_args: @options
132
138
  )
133
139
  end
134
140
 
@@ -138,6 +144,7 @@ module CanvasSync
138
144
  mapping[:groups][:report_columns],
139
145
  Group,
140
146
  mapping[:groups][:conflict_target].to_sym,
147
+ import_args: @options
141
148
  )
142
149
  end
143
150
 
@@ -148,6 +155,7 @@ module CanvasSync
148
155
  mapping[:group_memberships][:report_columns],
149
156
  GroupMembership,
150
157
  mapping[:group_memberships][:conflict_target].to_sym,
158
+ import_args: @options
151
159
  )
152
160
  end
153
161
  end
@@ -8,15 +8,16 @@ module CanvasSync
8
8
  # @param options [Hash]
9
9
  class SubmissionsProcessor < ReportProcessor
10
10
  def self.process(report_file_path, _options, report_id)
11
- new(report_file_path)
11
+ new(report_file_path, _options)
12
12
  end
13
13
 
14
- def initialize(report_file_path)
14
+ def initialize(report_file_path, options)
15
15
  CanvasSync::Importers::BulkImporter.import(
16
16
  report_file_path,
17
17
  mapping[:submissions][:report_columns],
18
18
  Submission,
19
19
  mapping[:submissions][:conflict_target].to_sym,
20
+ import_args: options
20
21
  )
21
22
  end
22
23
  end
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.17.0.beta15".freeze
2
+ VERSION = "0.17.3.beta2".freeze
3
3
  end