canvas_sync 0.17.0.beta15 → 0.17.3.beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +58 -0
- data/lib/canvas_sync/importers/bulk_importer.rb +7 -4
- data/lib/canvas_sync/job_batches/batch.rb +81 -107
- data/lib/canvas_sync/job_batches/callback.rb +20 -29
- data/lib/canvas_sync/job_batches/context_hash.rb +8 -5
- data/lib/canvas_sync/job_batches/hincr_max.lua +5 -0
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +99 -0
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +6 -65
- data/lib/canvas_sync/job_batches/pool.rb +193 -0
- data/lib/canvas_sync/job_batches/redis_model.rb +67 -0
- data/lib/canvas_sync/job_batches/redis_script.rb +163 -0
- data/lib/canvas_sync/job_batches/sidekiq.rb +22 -1
- data/lib/canvas_sync/job_batches/status.rb +0 -5
- data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +4 -2
- data/lib/canvas_sync/jobs/report_starter.rb +1 -0
- data/lib/canvas_sync/processors/assignment_groups_processor.rb +3 -2
- data/lib/canvas_sync/processors/assignments_processor.rb +3 -2
- data/lib/canvas_sync/processors/context_module_items_processor.rb +3 -2
- data/lib/canvas_sync/processors/context_modules_processor.rb +3 -2
- data/lib/canvas_sync/processors/normal_processor.rb +2 -1
- data/lib/canvas_sync/processors/provisioning_report_processor.rb +10 -2
- data/lib/canvas_sync/processors/submissions_processor.rb +3 -2
- data/lib/canvas_sync/version.rb +1 -1
- data/spec/dummy/log/test.log +78907 -0
- data/spec/job_batching/batch_aware_job_spec.rb +1 -0
- data/spec/job_batching/batch_spec.rb +72 -15
- data/spec/job_batching/callback_spec.rb +1 -1
- data/spec/job_batching/flow_spec.rb +5 -11
- data/spec/job_batching/integration/fail_then_succeed.rb +42 -0
- data/spec/job_batching/integration_helper.rb +6 -4
- data/spec/job_batching/sidekiq_spec.rb +1 -0
- data/spec/job_batching/status_spec.rb +1 -17
- metadata +9 -2
@@ -0,0 +1,163 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
require 'digest/sha1'
|
3
|
+
require 'erb'
|
4
|
+
|
5
|
+
# Modified from https://github.com/Shopify/wolverine/blob/master/lib/wolverine/script.rb
|
6
|
+
|
7
|
+
module CanvasSync
|
8
|
+
module JobBatches
|
9
|
+
# {RedisScript} represents a lua script in the filesystem. It loads the script
|
10
|
+
# from disk and handles talking to redis to execute it. Error handling
|
11
|
+
# is handled by {LuaError}.
|
12
|
+
class RedisScript
|
13
|
+
|
14
|
+
# Loads the script file from disk and calculates its +SHA1+ sum.
|
15
|
+
#
|
16
|
+
# @param file [Pathname] the full path to the indicated file
|
17
|
+
def initialize(file)
|
18
|
+
@file = Pathname.new(file)
|
19
|
+
end
|
20
|
+
|
21
|
+
# Passes the script and supplied arguments to redis for evaulation.
|
22
|
+
# It first attempts to use a script redis has already cached by using
|
23
|
+
# the +EVALSHA+ command, but falls back to providing the full script
|
24
|
+
# text via +EVAL+ if redis has not seen this script before. Future
|
25
|
+
# invocations will then use +EVALSHA+ without erroring.
|
26
|
+
#
|
27
|
+
# @param redis [Redis] the redis connection to run against
|
28
|
+
# @param args [*Objects] the arguments to the script
|
29
|
+
# @return [Object] the value passed back by redis after script execution
|
30
|
+
# @raise [LuaError] if the script failed to compile of encountered a
|
31
|
+
# runtime error
|
32
|
+
def call(redis, *args)
|
33
|
+
t = Time.now
|
34
|
+
begin
|
35
|
+
redis.evalsha(digest, *args)
|
36
|
+
rescue => e
|
37
|
+
e.message =~ /NOSCRIPT/ ? redis.eval(content, *args) : raise
|
38
|
+
end
|
39
|
+
rescue => e
|
40
|
+
if LuaError.intercepts?(e)
|
41
|
+
raise LuaError.new(e, @file, content)
|
42
|
+
else
|
43
|
+
raise
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def content
|
48
|
+
@content ||= load_lua(@file)
|
49
|
+
end
|
50
|
+
|
51
|
+
def digest
|
52
|
+
@digest ||= Digest::SHA1.hexdigest content
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def script_path
|
58
|
+
Rails.root + 'app/redis_lua'
|
59
|
+
end
|
60
|
+
|
61
|
+
def relative_path
|
62
|
+
@path ||= @file.relative_path_from(script_path)
|
63
|
+
end
|
64
|
+
|
65
|
+
def load_lua(file)
|
66
|
+
TemplateContext.new(script_path).template(script_path + file)
|
67
|
+
end
|
68
|
+
|
69
|
+
class TemplateContext
|
70
|
+
def initialize(script_path)
|
71
|
+
@script_path = script_path
|
72
|
+
end
|
73
|
+
|
74
|
+
def template(pathname)
|
75
|
+
@partial_templates ||= {}
|
76
|
+
ERB.new(File.read(pathname)).result binding
|
77
|
+
end
|
78
|
+
|
79
|
+
# helper method to include a lua partial within another lua script
|
80
|
+
#
|
81
|
+
# @param relative_path [String] the relative path to the script from
|
82
|
+
# `script_path`
|
83
|
+
def include_partial(relative_path)
|
84
|
+
unless @partial_templates.has_key? relative_path
|
85
|
+
@partial_templates[relative_path] = nil
|
86
|
+
template( Pathname.new("#{@script_path}/#{relative_path}") )
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
# Reformats errors raised by redis representing failures while executing
|
92
|
+
# a lua script. The default errors have confusing messages and backtraces,
|
93
|
+
# and a type of +RuntimeError+. This class improves the message and
|
94
|
+
# modifies the backtrace to include the lua script itself in a reasonable
|
95
|
+
# way.
|
96
|
+
class LuaError < StandardError
|
97
|
+
PATTERN = /ERR Error (compiling|running) script \(.*?\): .*?:(\d+): (.*)/
|
98
|
+
WOLVERINE_LIB_PATH = File.expand_path('../../', __FILE__)
|
99
|
+
CONTEXT_LINE_NUMBER = 2
|
100
|
+
|
101
|
+
attr_reader :error, :file, :content
|
102
|
+
|
103
|
+
# Is this error one that should be reformatted?
|
104
|
+
#
|
105
|
+
# @param error [StandardError] the original error raised by redis
|
106
|
+
# @return [Boolean] is this an error that should be reformatted?
|
107
|
+
def self.intercepts? error
|
108
|
+
error.message =~ PATTERN
|
109
|
+
end
|
110
|
+
|
111
|
+
# Initialize a new {LuaError} from an existing redis error, adjusting
|
112
|
+
# the message and backtrace in the process.
|
113
|
+
#
|
114
|
+
# @param error [StandardError] the original error raised by redis
|
115
|
+
# @param file [Pathname] full path to the lua file the error ocurred in
|
116
|
+
# @param content [String] lua file content the error ocurred in
|
117
|
+
def initialize error, file, content
|
118
|
+
@error = error
|
119
|
+
@file = file
|
120
|
+
@content = content
|
121
|
+
|
122
|
+
@error.message =~ PATTERN
|
123
|
+
_stage, line_number, message = $1, $2, $3
|
124
|
+
error_context = generate_error_context(content, line_number.to_i)
|
125
|
+
|
126
|
+
super "#{message}\n\n#{error_context}\n\n"
|
127
|
+
set_backtrace generate_backtrace file, line_number
|
128
|
+
end
|
129
|
+
|
130
|
+
private
|
131
|
+
|
132
|
+
def generate_error_context(content, line_number)
|
133
|
+
lines = content.lines.to_a
|
134
|
+
beginning_line_number = [1, line_number - CONTEXT_LINE_NUMBER].max
|
135
|
+
ending_line_number = [lines.count, line_number + CONTEXT_LINE_NUMBER].min
|
136
|
+
line_number_width = ending_line_number.to_s.length
|
137
|
+
|
138
|
+
(beginning_line_number..ending_line_number).map do |number|
|
139
|
+
indicator = number == line_number ? '=>' : ' '
|
140
|
+
formatted_number = "%#{line_number_width}d" % number
|
141
|
+
" #{indicator} #{formatted_number}: #{lines[number - 1]}"
|
142
|
+
end.join.chomp
|
143
|
+
end
|
144
|
+
|
145
|
+
def generate_backtrace(file, line_number)
|
146
|
+
pre_wolverine = backtrace_before_entering_wolverine(@error.backtrace)
|
147
|
+
index_of_first_wolverine_line = (@error.backtrace.size - pre_wolverine.size - 1)
|
148
|
+
pre_wolverine.unshift(@error.backtrace[index_of_first_wolverine_line])
|
149
|
+
pre_wolverine.unshift("#{file}:#{line_number}")
|
150
|
+
pre_wolverine
|
151
|
+
end
|
152
|
+
|
153
|
+
def backtrace_before_entering_wolverine(backtrace)
|
154
|
+
backtrace.reverse.take_while { |line| ! line_from_wolverine(line) }.reverse
|
155
|
+
end
|
156
|
+
|
157
|
+
def line_from_wolverine(line)
|
158
|
+
line.split(':').first.include?(WOLVERINE_LIB_PATH)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
@@ -15,11 +15,31 @@ module CanvasSync
|
|
15
15
|
Thread.current[:batch]
|
16
16
|
end
|
17
17
|
|
18
|
+
def batch_context
|
19
|
+
batch&.context || {}
|
20
|
+
end
|
21
|
+
|
18
22
|
def valid_within_batch?
|
19
23
|
batch.valid?
|
20
24
|
end
|
21
25
|
end
|
22
26
|
|
27
|
+
class SidekiqCallbackWorker
|
28
|
+
include ::Sidekiq::Worker
|
29
|
+
include WorkerExtension
|
30
|
+
include Batch::Callback::CallbackWorkerCommon
|
31
|
+
|
32
|
+
def self.enqueue_all(args, queue)
|
33
|
+
return if args.empty?
|
34
|
+
|
35
|
+
::Sidekiq::Client.push_bulk(
|
36
|
+
'class' => self,
|
37
|
+
'args' => args,
|
38
|
+
'queue' => queue
|
39
|
+
)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
23
43
|
class ClientMiddleware
|
24
44
|
def call(_worker, msg, _queue, _redis_pool = nil)
|
25
45
|
if (batch = Thread.current[:batch]) && should_handle_batch?(msg)
|
@@ -29,7 +49,7 @@ module CanvasSync
|
|
29
49
|
end
|
30
50
|
|
31
51
|
def should_handle_batch?(msg)
|
32
|
-
return false if msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && msg['wrapped'].constantize
|
52
|
+
return false if msg['class'] == 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper' && msg['wrapped'].constantize < BatchAwareJob
|
33
53
|
true
|
34
54
|
end
|
35
55
|
end
|
@@ -92,6 +112,7 @@ module CanvasSync
|
|
92
112
|
# This alias helps apartment-sidekiq set itself up correctly
|
93
113
|
::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
|
94
114
|
::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
|
115
|
+
Batch::Callback.worker_class = SidekiqCallbackWorker
|
95
116
|
end
|
96
117
|
end
|
97
118
|
end
|
@@ -24,10 +24,6 @@ module CanvasSync
|
|
24
24
|
Batch.redis { |r| r.hget("BID-#{bid}", 'created_at') }
|
25
25
|
end
|
26
26
|
|
27
|
-
def total
|
28
|
-
Batch.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
|
29
|
-
end
|
30
|
-
|
31
27
|
def parent_bid
|
32
28
|
Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
|
33
29
|
end
|
@@ -63,7 +59,6 @@ module CanvasSync
|
|
63
59
|
def data
|
64
60
|
{
|
65
61
|
bid: bid,
|
66
|
-
total: total,
|
67
62
|
failures: failures,
|
68
63
|
pending: pending,
|
69
64
|
created_at: created_at,
|
@@ -6,7 +6,9 @@ module CanvasSync
|
|
6
6
|
def perform(chain_definition, globals = {})
|
7
7
|
@globals = globals
|
8
8
|
|
9
|
-
if
|
9
|
+
if globals[:updated_after] == false
|
10
|
+
globals[:updated_after] = nil
|
11
|
+
elsif !globals[:updated_after].present? || globals[:updated_after] == true
|
10
12
|
last_batch = SyncBatch.where(status: 'completed', batch_genre: genre).last
|
11
13
|
globals[:full_sync_every] ||= "sunday/2"
|
12
14
|
globals[:updated_after] = last_batch&.started_at&.iso8601
|
@@ -39,7 +41,7 @@ module CanvasSync
|
|
39
41
|
return true unless last_full_sync.present?
|
40
42
|
return false unless opt.is_a?(String)
|
41
43
|
|
42
|
-
case
|
44
|
+
case opt.strip
|
43
45
|
when %r{^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)(?:/(\d+))?$}
|
44
46
|
m = Regexp.last_match
|
45
47
|
day = m[1]
|
@@ -11,6 +11,7 @@ module CanvasSync
|
|
11
11
|
# @return [nil]
|
12
12
|
def perform(report_name, report_params, processor, options, allow_redownloads: false)
|
13
13
|
account_id = options[:account_id] || batch_context[:account_id] || "self"
|
14
|
+
options[:sync_start_time] = DateTime.now.utc.iso8601
|
14
15
|
|
15
16
|
report_id = start_report(account_id, report_name, report_params)
|
16
17
|
# TODO: Restore report caching support (does nayone actually use it?)
|
@@ -8,15 +8,16 @@ module CanvasSync
|
|
8
8
|
# @param options [Hash]
|
9
9
|
class AssignmentGroupsProcessor < ReportProcessor
|
10
10
|
def self.process(report_file_path, _options, report_id)
|
11
|
-
new(report_file_path)
|
11
|
+
new(report_file_path, _options)
|
12
12
|
end
|
13
13
|
|
14
|
-
def initialize(report_file_path)
|
14
|
+
def initialize(report_file_path, options)
|
15
15
|
CanvasSync::Importers::BulkImporter.import(
|
16
16
|
report_file_path,
|
17
17
|
mapping[:assignment_groups][:report_columns],
|
18
18
|
AssignmentGroup,
|
19
19
|
mapping[:assignment_groups][:conflict_target].to_sym,
|
20
|
+
import_args: options
|
20
21
|
)
|
21
22
|
end
|
22
23
|
end
|
@@ -8,15 +8,16 @@ module CanvasSync
|
|
8
8
|
# @param options [Hash]
|
9
9
|
class AssignmentsProcessor < ReportProcessor
|
10
10
|
def self.process(report_file_path, _options, report_id)
|
11
|
-
new(report_file_path)
|
11
|
+
new(report_file_path, _options)
|
12
12
|
end
|
13
13
|
|
14
|
-
def initialize(report_file_path)
|
14
|
+
def initialize(report_file_path, options)
|
15
15
|
CanvasSync::Importers::BulkImporter.import(
|
16
16
|
report_file_path,
|
17
17
|
mapping[:assignments][:report_columns],
|
18
18
|
Assignment,
|
19
19
|
mapping[:assignments][:conflict_target].to_sym,
|
20
|
+
import_args: options
|
20
21
|
)
|
21
22
|
end
|
22
23
|
end
|
@@ -8,15 +8,16 @@ module CanvasSync
|
|
8
8
|
# @param options [Hash]
|
9
9
|
class ContextModuleItemsProcessor < ReportProcessor
|
10
10
|
def self.process(report_file_path, _options, report_id)
|
11
|
-
new(report_file_path)
|
11
|
+
new(report_file_path, _options)
|
12
12
|
end
|
13
13
|
|
14
|
-
def initialize(report_file_path)
|
14
|
+
def initialize(report_file_path, options)
|
15
15
|
CanvasSync::Importers::BulkImporter.import(
|
16
16
|
report_file_path,
|
17
17
|
mapping[:context_module_items][:report_columns],
|
18
18
|
ContextModuleItem,
|
19
19
|
mapping[:context_module_items][:conflict_target].to_sym,
|
20
|
+
import_args: options
|
20
21
|
)
|
21
22
|
end
|
22
23
|
end
|
@@ -8,15 +8,16 @@ module CanvasSync
|
|
8
8
|
# @param options [Hash]
|
9
9
|
class ContextModulesProcessor < ReportProcessor
|
10
10
|
def self.process(report_file_path, _options, report_id)
|
11
|
-
new(report_file_path)
|
11
|
+
new(report_file_path, _options)
|
12
12
|
end
|
13
13
|
|
14
|
-
def initialize(report_file_path)
|
14
|
+
def initialize(report_file_path, options)
|
15
15
|
CanvasSync::Importers::BulkImporter.import(
|
16
16
|
report_file_path,
|
17
17
|
mapping[:context_modules][:report_columns],
|
18
18
|
ContextModule,
|
19
19
|
mapping[:context_modules][:conflict_target].to_sym,
|
20
|
+
import_args: options
|
20
21
|
)
|
21
22
|
end
|
22
23
|
end
|
@@ -18,7 +18,8 @@ module CanvasSync
|
|
18
18
|
report_file_path,
|
19
19
|
mapping[options[:mapping].to_sym][:report_columns],
|
20
20
|
options[:klass].constantize,
|
21
|
-
conflict_target ? conflict_target.to_sym : conflict_target
|
21
|
+
conflict_target ? conflict_target.to_sym : conflict_target,
|
22
|
+
import_args: options
|
22
23
|
)
|
23
24
|
end
|
24
25
|
end
|
@@ -21,7 +21,6 @@ module CanvasSync
|
|
21
21
|
|
22
22
|
def initialize(report_file_path, options) # rubocop:disable Metrics/AbcSize
|
23
23
|
@options = options
|
24
|
-
|
25
24
|
if options[:models].length == 1
|
26
25
|
run_import(options[:models][0], report_file_path)
|
27
26
|
else
|
@@ -75,6 +74,7 @@ module CanvasSync
|
|
75
74
|
mapping[:users][:report_columns],
|
76
75
|
User,
|
77
76
|
mapping[:users][:conflict_target].to_sym,
|
77
|
+
import_args: @options
|
78
78
|
)
|
79
79
|
end
|
80
80
|
|
@@ -84,6 +84,7 @@ module CanvasSync
|
|
84
84
|
mapping[:pseudonyms][:report_columns],
|
85
85
|
Pseudonym,
|
86
86
|
mapping[:pseudonyms][:conflict_target].to_sym,
|
87
|
+
import_args: @options
|
87
88
|
)
|
88
89
|
end
|
89
90
|
|
@@ -92,7 +93,8 @@ module CanvasSync
|
|
92
93
|
report_file_path,
|
93
94
|
mapping[:accounts][:report_columns],
|
94
95
|
Account,
|
95
|
-
mapping[:accounts][:conflict_target].to_sym
|
96
|
+
mapping[:accounts][:conflict_target].to_sym,
|
97
|
+
import_args: @options
|
96
98
|
)
|
97
99
|
end
|
98
100
|
|
@@ -102,6 +104,7 @@ module CanvasSync
|
|
102
104
|
mapping[:courses][:report_columns],
|
103
105
|
Course,
|
104
106
|
mapping[:courses][:conflict_target].to_sym,
|
107
|
+
import_args: @options
|
105
108
|
)
|
106
109
|
end
|
107
110
|
|
@@ -111,6 +114,7 @@ module CanvasSync
|
|
111
114
|
mapping[:enrollments][:report_columns],
|
112
115
|
Enrollment,
|
113
116
|
mapping[:enrollments][:conflict_target].to_sym,
|
117
|
+
import_args: @options
|
114
118
|
)
|
115
119
|
end
|
116
120
|
|
@@ -120,6 +124,7 @@ module CanvasSync
|
|
120
124
|
mapping[:sections][:report_columns],
|
121
125
|
Section,
|
122
126
|
mapping[:sections][:conflict_target].to_sym,
|
127
|
+
import_args: @options
|
123
128
|
)
|
124
129
|
end
|
125
130
|
|
@@ -129,6 +134,7 @@ module CanvasSync
|
|
129
134
|
mapping[:xlist][:report_columns],
|
130
135
|
Section,
|
131
136
|
mapping[:xlist][:conflict_target].to_sym,
|
137
|
+
import_args: @options
|
132
138
|
)
|
133
139
|
end
|
134
140
|
|
@@ -138,6 +144,7 @@ module CanvasSync
|
|
138
144
|
mapping[:groups][:report_columns],
|
139
145
|
Group,
|
140
146
|
mapping[:groups][:conflict_target].to_sym,
|
147
|
+
import_args: @options
|
141
148
|
)
|
142
149
|
end
|
143
150
|
|
@@ -148,6 +155,7 @@ module CanvasSync
|
|
148
155
|
mapping[:group_memberships][:report_columns],
|
149
156
|
GroupMembership,
|
150
157
|
mapping[:group_memberships][:conflict_target].to_sym,
|
158
|
+
import_args: @options
|
151
159
|
)
|
152
160
|
end
|
153
161
|
end
|
@@ -8,15 +8,16 @@ module CanvasSync
|
|
8
8
|
# @param options [Hash]
|
9
9
|
class SubmissionsProcessor < ReportProcessor
|
10
10
|
def self.process(report_file_path, _options, report_id)
|
11
|
-
new(report_file_path)
|
11
|
+
new(report_file_path, _options)
|
12
12
|
end
|
13
13
|
|
14
|
-
def initialize(report_file_path)
|
14
|
+
def initialize(report_file_path, options)
|
15
15
|
CanvasSync::Importers::BulkImporter.import(
|
16
16
|
report_file_path,
|
17
17
|
mapping[:submissions][:report_columns],
|
18
18
|
Submission,
|
19
19
|
mapping[:submissions][:conflict_target].to_sym,
|
20
|
+
import_args: options
|
20
21
|
)
|
21
22
|
end
|
22
23
|
end
|
data/lib/canvas_sync/version.rb
CHANGED