canvas_sync 0.21.1.beta1 → 0.22.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/canvas_sync/concerns/auto_relations.rb +11 -0
- data/lib/canvas_sync/config.rb +3 -5
- data/lib/canvas_sync/generators/templates/models/rubric.rb +2 -1
- data/lib/canvas_sync/job_batches/batch.rb +432 -402
- data/lib/canvas_sync/job_batches/callback.rb +100 -114
- data/lib/canvas_sync/job_batches/chain_builder.rb +194 -196
- data/lib/canvas_sync/job_batches/{active_job.rb → compat/active_job.rb} +2 -2
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/helpers.rb +1 -1
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web.rb +3 -3
- data/lib/canvas_sync/job_batches/{sidekiq.rb → compat/sidekiq.rb} +35 -22
- data/lib/canvas_sync/job_batches/compat.rb +20 -0
- data/lib/canvas_sync/job_batches/context_hash.rb +124 -126
- data/lib/canvas_sync/job_batches/jobs/base_job.rb +2 -4
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +125 -127
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/pool.rb +193 -195
- data/lib/canvas_sync/job_batches/redis_model.rb +50 -52
- data/lib/canvas_sync/job_batches/redis_script.rb +129 -131
- data/lib/canvas_sync/job_batches/status.rb +85 -87
- data/lib/canvas_sync/job_uniqueness/compat/active_job.rb +75 -0
- data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb +135 -0
- data/lib/canvas_sync/job_uniqueness/compat.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/configuration.rb +25 -0
- data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb +47 -0
- data/lib/canvas_sync/job_uniqueness/lock_context.rb +171 -0
- data/lib/canvas_sync/job_uniqueness/locksmith.rb +92 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/base.rb +32 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/log.rb +13 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/null_strategy.rb +9 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/raise.rb +11 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reject.rb +21 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reschedule.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict.rb +41 -0
- data/lib/canvas_sync/job_uniqueness/strategy/base.rb +104 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_and_while_executing.rb +35 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executed.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executing.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_expired.rb +16 -0
- data/lib/canvas_sync/job_uniqueness/strategy/while_executing.rb +26 -0
- data/lib/canvas_sync/job_uniqueness/strategy.rb +27 -0
- data/lib/canvas_sync/job_uniqueness/unique_job_common.rb +79 -0
- data/lib/canvas_sync/misc_helper.rb +1 -1
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +4 -3
- data/spec/dummy/app/models/rubric.rb +2 -1
- data/spec/dummy/config/environments/test.rb +1 -1
- data/spec/job_batching/batch_spec.rb +49 -7
- data/spec/job_batching/{active_job_spec.rb → compat/active_job_spec.rb} +2 -2
- data/spec/job_batching/{sidekiq_spec.rb → compat/sidekiq_spec.rb} +14 -12
- data/spec/job_batching/flow_spec.rb +1 -1
- data/spec/job_batching/integration_helper.rb +1 -1
- data/spec/job_batching/status_spec.rb +2 -2
- data/spec/job_uniqueness/compat/active_job_spec.rb +49 -0
- data/spec/job_uniqueness/compat/sidekiq_spec.rb +68 -0
- data/spec/job_uniqueness/lock_context_spec.rb +95 -0
- data/spec/job_uniqueness/on_conflict/log_spec.rb +11 -0
- data/spec/job_uniqueness/on_conflict/raise_spec.rb +10 -0
- data/spec/job_uniqueness/on_conflict/reschedule_spec.rb +24 -0
- data/spec/job_uniqueness/on_conflict_spec.rb +16 -0
- data/spec/job_uniqueness/spec_helper.rb +14 -0
- data/spec/job_uniqueness/strategy/base_spec.rb +100 -0
- data/spec/job_uniqueness/strategy/until_and_while_executing_spec.rb +48 -0
- data/spec/job_uniqueness/strategy/until_executed_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_executing_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_expired_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/while_executing_spec.rb +33 -0
- data/spec/job_uniqueness/support/lock_strategy.rb +28 -0
- data/spec/job_uniqueness/support/on_conflict.rb +24 -0
- data/spec/job_uniqueness/support/test_worker.rb +19 -0
- data/spec/job_uniqueness/unique_job_common_spec.rb +45 -0
- data/spec/spec_helper.rb +1 -1
- metadata +278 -204
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/css/styles.less +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/batch_tree.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/util.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batch_tree.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batches_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_common.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_jobs_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_pagination.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batch.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batches.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pool.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pools.erb +0 -0
@@ -0,0 +1,104 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
class Base
|
4
|
+
attr_reader :lock_context
|
5
|
+
|
6
|
+
def initialize(lock_context)
|
7
|
+
@lock_context = lock_context
|
8
|
+
@conflict_strategies = {}
|
9
|
+
end
|
10
|
+
|
11
|
+
class_attribute :_locks_on, instance_writer: false
|
12
|
+
|
13
|
+
def self.locks_on(*origins)
|
14
|
+
if origins.present?
|
15
|
+
orgins = Array(origins).map(&:to_sym)
|
16
|
+
self._locks_on = origins
|
17
|
+
else
|
18
|
+
self._locks_on || [:enqueue, :perform]
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def on_enqueue; end
|
23
|
+
def on_perform; end
|
24
|
+
|
25
|
+
def batch_callback(event, batch_status)
|
26
|
+
if event == :success
|
27
|
+
unlock
|
28
|
+
else
|
29
|
+
unlock_cond = lock_context.config[:unlock_on_failure]
|
30
|
+
|
31
|
+
if (event == :complete && unlock_cond == :any) || (event == :death && unlock_cond == :death) || (event == :stagnated && unlock_cond == :stagnant)
|
32
|
+
unlock
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
delegate :locked?, to: :locksmith
|
38
|
+
|
39
|
+
protected
|
40
|
+
|
41
|
+
def key
|
42
|
+
lock_context.base_key
|
43
|
+
end
|
44
|
+
|
45
|
+
def wrap_in_batch(&blk)
|
46
|
+
if Thread.current[:unique_jobs_previous_jid] # Ensure we don't re-wrap in a batch when rescheduling
|
47
|
+
return blk.call
|
48
|
+
end
|
49
|
+
|
50
|
+
batch = CanvasSync::JobBatches::Batch.new
|
51
|
+
batch.context = {
|
52
|
+
uniqueness_lock_key: key,
|
53
|
+
}
|
54
|
+
|
55
|
+
CanvasSync::JobBatches::Batch::Callback::VALID_CALLBACKS.each do |callback|
|
56
|
+
callback = callback.to_sym
|
57
|
+
batch.on(callback, self.class.to_s + ".internal_batch_callback", {
|
58
|
+
event: callback,
|
59
|
+
lock_strategy: self.class.to_s,
|
60
|
+
lock_key: key,
|
61
|
+
lock_context: lock_context.serialize,
|
62
|
+
})
|
63
|
+
end
|
64
|
+
|
65
|
+
batch.jobs do
|
66
|
+
return blk.call
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.internal_batch_callback(batch_status, opts)
|
71
|
+
strategy_class = opts[:lock_strategy].constantize
|
72
|
+
lock_context = LockContext.from_serialized(opts[:lock_context])
|
73
|
+
strategy = strategy_class.new(lock_context)
|
74
|
+
# TODO Should this route through LockContext#handle_lifecycle!?
|
75
|
+
strategy.batch_callback(opts[:event].to_sym, batch_status)
|
76
|
+
end
|
77
|
+
|
78
|
+
def lock!(purpose, wait: nil)
|
79
|
+
locked = nil
|
80
|
+
if purpose == :enqueue
|
81
|
+
if Thread.current[:unique_jobs_previous_jid].present?
|
82
|
+
locked = locksmith.swap_locks(Thread.current[:unique_jobs_previous_jid])
|
83
|
+
else
|
84
|
+
locked = locksmith.lock()
|
85
|
+
end
|
86
|
+
elsif purpose == :perform
|
87
|
+
locked = locksmith.execute { lock_context.job_id }
|
88
|
+
end
|
89
|
+
|
90
|
+
CanvasSync::JobUniqueness.logger.debug { "Requested lock of #{key} for #{purpose} - (#{locked || 'Not Obtained!'})" }
|
91
|
+
|
92
|
+
raise CouldNotLockError.new(lock_context, source: purpose) if !locked
|
93
|
+
end
|
94
|
+
|
95
|
+
def unlock()
|
96
|
+
locksmith.unlock
|
97
|
+
end
|
98
|
+
|
99
|
+
def locksmith
|
100
|
+
@locksmith ||= Locksmith.new(key, lock_context)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
# Implements two locks - one while enqueued and one while performing
|
4
|
+
class UntilAndWhileExecuting < Base
|
5
|
+
locks_on :enqueue, :perform
|
6
|
+
|
7
|
+
def on_enqueue
|
8
|
+
# Obtain lock
|
9
|
+
lock!(:enqueue)
|
10
|
+
|
11
|
+
# Proceed with enqueuing the job, wrapping it in a batch
|
12
|
+
runtime_lock.on_enqueue do
|
13
|
+
yield
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def on_perform
|
18
|
+
# Obtain Runtime lock
|
19
|
+
runtime_lock.on_perform do
|
20
|
+
# Release Queue lock
|
21
|
+
unlock()
|
22
|
+
|
23
|
+
# Run the job
|
24
|
+
yield
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def runtime_lock
|
31
|
+
@runtime_lock ||= Strategy::WhileExecuting.new(lock_context)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
class UntilExecuted < Base
|
4
|
+
locks_on :enqueue, :perform
|
5
|
+
|
6
|
+
def on_enqueue
|
7
|
+
lock!(:enqueue)
|
8
|
+
|
9
|
+
wrap_in_batch do
|
10
|
+
yield
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def on_perform
|
15
|
+
lock!(:perform)
|
16
|
+
yield
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
class UntilExecuting < Base
|
4
|
+
locks_on :enqueue
|
5
|
+
|
6
|
+
def on_enqueue
|
7
|
+
lock!(:enqueue)
|
8
|
+
yield
|
9
|
+
end
|
10
|
+
|
11
|
+
def on_perform
|
12
|
+
unlock
|
13
|
+
yield
|
14
|
+
end
|
15
|
+
|
16
|
+
# TODO Define behavior when an error occurs during perform().
|
17
|
+
# SUJ's behavior is to relock, but this has some edge-cases (like how do we handle if another job already took the lock?)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
class WhileExecuting < Base
|
4
|
+
locks_on :perform
|
5
|
+
|
6
|
+
RUN_SUFFIX = ":RUN"
|
7
|
+
|
8
|
+
def on_enqueue
|
9
|
+
wrap_in_batch do
|
10
|
+
yield
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def on_perform
|
15
|
+
lock!(:perform)
|
16
|
+
yield
|
17
|
+
end
|
18
|
+
|
19
|
+
protected
|
20
|
+
|
21
|
+
def key
|
22
|
+
super + RUN_SUFFIX
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
extend ActiveSupport::Autoload
|
4
|
+
|
5
|
+
autoload :Base
|
6
|
+
|
7
|
+
autoload :UntilExpired
|
8
|
+
autoload :UntilExecuted
|
9
|
+
autoload :UntilExecuting
|
10
|
+
autoload :UntilAndWhileExecuting
|
11
|
+
autoload :WhileExecuting
|
12
|
+
|
13
|
+
class << self
|
14
|
+
def lookup(strategy)
|
15
|
+
matching_strategy(strategy.to_s.camelize) ||
|
16
|
+
CanvasSync::JobUniqueness.config.lock_strategies[strategy] ||
|
17
|
+
raise(ArgumentError, "strategy: #{strategy} is not found. Is it declared in the configuration?")
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def matching_strategy(const)
|
23
|
+
const_get(const, false) if const_defined?(const, false)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
|
2
|
+
module CanvasSync::JobUniqueness
|
3
|
+
module UniqueJobCommon
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
class_attribute :unique_job_options, instance_writer: false
|
8
|
+
end
|
9
|
+
|
10
|
+
class_methods do
|
11
|
+
# ensure_uniqueness(
|
12
|
+
# strategy: :until_executed, # :until_executed, :until_executing, :until_expired, :until_and_while_executing, :while_executing
|
13
|
+
# on_conflict: :raise, # :raise, :replace, :log, :reject, :reschedule, { enqueue: ..., perform: ... }, proc
|
14
|
+
# lock_ttl: 7.days, # seconds
|
15
|
+
# lock_timeout: 0, # seconds
|
16
|
+
|
17
|
+
# scope: :per_queue, # :global, :per_queue, string ("<class>-<queue>"), proc
|
18
|
+
# hash: ->{ { ... } },
|
19
|
+
|
20
|
+
# # In the case of UntilExecuted and WhileExecuting, how should the execution lock be released in an error condition?
|
21
|
+
# # :any - Release the lock when the Job's (implicit) Batch is :complete
|
22
|
+
# # :death - Release the lock when the Job's Batch receives the :death callback
|
23
|
+
# # :stagnant - Release the lock when the Job's Batch receives the :stagnant callback
|
24
|
+
# # :expire - Do not release the lock until it expires
|
25
|
+
# unlock_on_failure: :stagnant,
|
26
|
+
# )
|
27
|
+
def ensure_uniqueness(**kwargs)
|
28
|
+
if self.unique_job_options.present?
|
29
|
+
raise ArgumentError, "ensure_uniqueness can only be called once per job class"
|
30
|
+
end
|
31
|
+
|
32
|
+
begin
|
33
|
+
require "sidekiq_unique_jobs"
|
34
|
+
rescue LoadError
|
35
|
+
raise LoadError, "SidekiqUniqueJobs is required for ensure_uniqueness"
|
36
|
+
end
|
37
|
+
|
38
|
+
OnConflict.validate!(kwargs[:on_conflict], kwargs[:strategy]) if kwargs[:on_conflict].present?
|
39
|
+
|
40
|
+
kwargs[:scope] ||= :per_queue
|
41
|
+
kwargs[:ttl] ||= 30.days.to_i
|
42
|
+
kwargs[:timeout] ||= 0
|
43
|
+
kwargs[:limit] ||= 1
|
44
|
+
|
45
|
+
self.unique_job_options = kwargs
|
46
|
+
|
47
|
+
include UniqueJobMethods
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
module UniqueJobMethods
|
52
|
+
extend ActiveSupport::Concern
|
53
|
+
|
54
|
+
class_methods do
|
55
|
+
def unlock!(jid = nil, args: nil, kwargs: nil, queue: nil)
|
56
|
+
queue = self.try(:default_queue_name) || try(:queue)
|
57
|
+
raise ArgumentError, "Must specify queue:" unless queue.is_a?(String) || queue.is_a?(Symbol)
|
58
|
+
|
59
|
+
temp_context = LockContext.new({ job_clazz: self, jid: jid, queue: queue, args: args, kwargs: kwargs })
|
60
|
+
strategy = temp_context.lock_strategy
|
61
|
+
locksmith = strategy.send(:locksmith)
|
62
|
+
|
63
|
+
if jid
|
64
|
+
locksmith.unlock!()
|
65
|
+
else
|
66
|
+
locksmith.locked_jids.each do |jid|
|
67
|
+
unlock!(jid, args: args, kwargs: kwargs, queue: queue)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# def unlock_all!(queue: :all)
|
73
|
+
# # TODO Public API to manually remove all locks for this class
|
74
|
+
# end
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -34,7 +34,7 @@ module CanvasSync
|
|
34
34
|
# TODO Support globalid
|
35
35
|
model_class = load_constant(job[:model])
|
36
36
|
find_by = job[:find_by]
|
37
|
-
target = find_by.is_a?(Hash) ? model_class.find_by(find_by) : model_class.find_by(id: find_by)
|
37
|
+
target = find_by.is_a?(Hash) ? model_class.find_by(**find_by) : model_class.find_by(id: find_by)
|
38
38
|
target.send(job[:method], *job_args, **job_kwargs)
|
39
39
|
elsif job[:class]
|
40
40
|
target = load_constant(job[:class])
|
data/lib/canvas_sync/version.rb
CHANGED
data/lib/canvas_sync.rb
CHANGED
@@ -16,6 +16,7 @@ require "canvas_sync/batch_processor"
|
|
16
16
|
require "canvas_sync/config"
|
17
17
|
|
18
18
|
require "canvas_sync/job_batches/batch"
|
19
|
+
require "canvas_sync/job_uniqueness/job_uniqueness"
|
19
20
|
|
20
21
|
Dir[File.dirname(__FILE__) + "/canvas_sync/jobs/*.rb"].each { |file| require file }
|
21
22
|
Dir[File.dirname(__FILE__) + "/canvas_sync/processors/*.rb"].each { |file| require file }
|
@@ -249,7 +250,7 @@ module CanvasSync
|
|
249
250
|
global_options[:account_id] = account_id if account_id.present?
|
250
251
|
global_options.merge!(globals) if globals
|
251
252
|
|
252
|
-
JobBatches::ChainBuilder.build(CanvasSync::Jobs::BeginSyncChainJob, [], global_options, &blk)
|
253
|
+
JobBatches::ChainBuilder.build(CanvasSync::Jobs::BeginSyncChainJob, *[], **global_options, &blk)
|
253
254
|
end
|
254
255
|
|
255
256
|
def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
|
@@ -338,7 +339,7 @@ module CanvasSync
|
|
338
339
|
def logger
|
339
340
|
return @logger if defined? @logger
|
340
341
|
@logger = Logger.new(STDOUT)
|
341
|
-
@logger.level = Logger::
|
342
|
+
@logger.level = Logger::WARN
|
342
343
|
@logger
|
343
344
|
end
|
344
345
|
|
@@ -347,7 +348,7 @@ module CanvasSync
|
|
347
348
|
end
|
348
349
|
|
349
350
|
def redis_prefix
|
350
|
-
pfx =
|
351
|
+
pfx = config.redis_key_prefix
|
351
352
|
pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
|
352
353
|
pfx
|
353
354
|
end
|
@@ -17,8 +17,9 @@ class Rubric < ApplicationRecord
|
|
17
17
|
belongs_to :rubric, primary_key: :canvas_id, foreign_key: :canvas_rubric_id , optional: true # based on another rubric
|
18
18
|
belongs_to :context, polymorphic: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type, optional: true
|
19
19
|
|
20
|
-
has_many :rubric_associations, primary_key: :canvas_id, foreign_key: :canvas_association_id, foreign_type: :canvas_association_type
|
21
20
|
has_many :rubric_assessments, through: :rubric_associations, dependent: :destroy
|
21
|
+
has_many :rubric_associations, primary_key: :canvas_id, foreign_key: :canvas_rubric_id
|
22
|
+
has_many :associated_objects, through: :rubric_associations, source: :association_object
|
22
23
|
|
23
24
|
api_syncable({
|
24
25
|
canvas_id: :id,
|
@@ -5,7 +5,7 @@ Rails.application.configure do
|
|
5
5
|
# test suite. You never need to work with it otherwise. Remember that
|
6
6
|
# your test database is "scratch space" for the test suite and is wiped
|
7
7
|
# and recreated between test runs. Don't rely on the data there!
|
8
|
-
config.cache_classes =
|
8
|
+
config.cache_classes = false
|
9
9
|
|
10
10
|
# Do not eager load code on boot. This avoids loading your whole application
|
11
11
|
# just for the purpose of running a single test. If you are using a tool that
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
class
|
3
|
+
class BatchTestWorker < BatchTestJobBase
|
4
4
|
def perform
|
5
5
|
end
|
6
6
|
end
|
@@ -173,6 +173,48 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
173
173
|
end
|
174
174
|
end
|
175
175
|
|
176
|
+
describe '#process_dead_job' do
|
177
|
+
let(:batch) { CanvasSync::JobBatches::Batch.new }
|
178
|
+
let(:bid) { batch.bid }
|
179
|
+
let(:jid) { 'ABCD' }
|
180
|
+
before { CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
|
181
|
+
|
182
|
+
context 'dead' do
|
183
|
+
let(:failed_jid) { 'xxx' }
|
184
|
+
|
185
|
+
it 'tries to call death callback' do
|
186
|
+
allow(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:stagnated, bid)
|
187
|
+
expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:death, bid)
|
188
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
189
|
+
end
|
190
|
+
|
191
|
+
it 'add job to failed list' do
|
192
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, 'failed-job-id')
|
193
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
194
|
+
failed = CanvasSync::JobBatches::Batch.redis { |r| r.smembers("BID-#{bid}-dead") }
|
195
|
+
expect(failed).to contain_exactly('xxx', 'failed-job-id')
|
196
|
+
end
|
197
|
+
|
198
|
+
it 'automattically triggers the :stagnated callback' do
|
199
|
+
expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:stagnated, bid)
|
200
|
+
allow(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:death, bid)
|
201
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
202
|
+
end
|
203
|
+
|
204
|
+
it 'does not trigger :stagnated if pending jobs are still present' do
|
205
|
+
CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 2) }
|
206
|
+
expect(CanvasSync::JobBatches::Batch).to_not receive(:enqueue_callbacks).with(:stagnated, bid)
|
207
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
208
|
+
end
|
209
|
+
|
210
|
+
it 'does not trigger :stagnated if pending batches are still present' do
|
211
|
+
CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 2) }
|
212
|
+
expect(CanvasSync::JobBatches::Batch).to_not receive(:enqueue_callbacks).with(:stagnated, bid)
|
213
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
176
218
|
describe '#process_successful_job' do
|
177
219
|
let(:batch) { CanvasSync::JobBatches::Batch.new }
|
178
220
|
let(:bid) { batch.bid }
|
@@ -182,7 +224,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
182
224
|
context 'complete' do
|
183
225
|
before { batch.on(:complete, Object) }
|
184
226
|
# before { batch.increment_job_queue(bid) }
|
185
|
-
# before { batch.jobs do
|
227
|
+
# before { batch.jobs do BatchTestWorker.perform_async end }
|
186
228
|
# before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id') }
|
187
229
|
|
188
230
|
it 'tries to call complete callback' do
|
@@ -211,7 +253,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
211
253
|
|
212
254
|
it 'triggers callbacks as expected' do
|
213
255
|
ActiveJob::Base.queue_adapter = :sidekiq
|
214
|
-
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
|
256
|
+
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Compat::Sidekiq::SidekiqCallbackWorker
|
215
257
|
|
216
258
|
callback_instance = double('SampleCallback')
|
217
259
|
expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
|
@@ -229,7 +271,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
229
271
|
|
230
272
|
it 'delays triggering callbacks if keep_open is set' do
|
231
273
|
ActiveJob::Base.queue_adapter = :sidekiq
|
232
|
-
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
|
274
|
+
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Compat::Sidekiq::SidekiqCallbackWorker
|
233
275
|
|
234
276
|
callback_instance = double('SampleCallback')
|
235
277
|
expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
|
@@ -303,7 +345,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
303
345
|
let(:batch) { CanvasSync::JobBatches::Batch.new }
|
304
346
|
|
305
347
|
it 'increments pending' do
|
306
|
-
batch.jobs do
|
348
|
+
batch.jobs do BatchTestWorker.perform_async end
|
307
349
|
pending = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'pending') }
|
308
350
|
expect(pending).to eq('1')
|
309
351
|
end
|
@@ -338,7 +380,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
338
380
|
|
339
381
|
context 'With ActiveJob Adapter' do
|
340
382
|
around(:all) do |block|
|
341
|
-
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::ActiveJob::ActiveJobCallbackWorker
|
383
|
+
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Compat::ActiveJob::ActiveJobCallbackWorker
|
342
384
|
block.run
|
343
385
|
end
|
344
386
|
|
@@ -396,7 +438,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
396
438
|
|
397
439
|
context 'With Sidekiq Adapter' do
|
398
440
|
around(:all) do |block|
|
399
|
-
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
|
441
|
+
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Compat::Sidekiq::SidekiqCallbackWorker
|
400
442
|
block.run
|
401
443
|
end
|
402
444
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
RSpec.describe CanvasSync::JobBatches::ActiveJob do
|
4
|
-
describe CanvasSync::JobBatches::ActiveJob::BatchAwareJob do
|
3
|
+
RSpec.describe CanvasSync::JobBatches::Compat::ActiveJob do
|
4
|
+
describe CanvasSync::JobBatches::Compat::ActiveJob::BatchAwareJob do
|
5
5
|
include ActiveJob::TestHelper
|
6
6
|
|
7
7
|
after do
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
4
|
-
describe CanvasSync::JobBatches::Sidekiq::ServerMiddleware do
|
3
|
+
RSpec.describe CanvasSync::JobBatches::Compat::Sidekiq do
|
4
|
+
describe CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware do
|
5
5
|
context 'when without batch' do
|
6
6
|
it 'just yields' do
|
7
7
|
yielded = false
|
@@ -44,7 +44,7 @@ RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
47
|
-
describe CanvasSync::JobBatches::Sidekiq::ClientMiddleware do
|
47
|
+
describe CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware do
|
48
48
|
context 'when without batch' do
|
49
49
|
it 'just yields' do
|
50
50
|
yielded = false
|
@@ -80,16 +80,17 @@ RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
|
80
80
|
end
|
81
81
|
end
|
82
82
|
|
83
|
-
RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
84
|
-
let(:config) { class_double(Sidekiq) }
|
83
|
+
RSpec.describe CanvasSync::JobBatches::Compat::Sidekiq do
|
84
|
+
let(:config) { defined?(Sidekiq::Config) ? double(Sidekiq::Config) : class_double(Sidekiq) }
|
85
85
|
let(:client_middleware) { double(Sidekiq::Middleware::Chain) }
|
86
86
|
|
87
87
|
context 'client' do
|
88
88
|
it 'adds client middleware' do
|
89
|
-
|
89
|
+
allow(Sidekiq).to receive(:configure_client).and_yield(config)
|
90
90
|
expect(config).to receive(:client_middleware).and_yield(client_middleware)
|
91
|
-
expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
|
92
|
-
CanvasSync::JobBatches::Sidekiq.
|
91
|
+
expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware)
|
92
|
+
CanvasSync::JobBatches::Compat::Sidekiq.instance_variable_set(:@already_configured, false)
|
93
|
+
CanvasSync::JobBatches::Compat::Sidekiq.configure
|
93
94
|
end
|
94
95
|
end
|
95
96
|
|
@@ -98,14 +99,15 @@ RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
|
98
99
|
let(:death_handlers) { double(Array) }
|
99
100
|
|
100
101
|
it 'adds client and server middleware' do
|
101
|
-
|
102
|
+
allow(Sidekiq).to receive(:configure_server).and_yield(config)
|
102
103
|
expect(config).to receive(:client_middleware).and_yield(client_middleware)
|
103
104
|
expect(config).to receive(:server_middleware).and_yield(server_middleware)
|
104
105
|
expect(config).to receive(:death_handlers).and_return(death_handlers)
|
105
|
-
expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
|
106
|
-
expect(server_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
|
106
|
+
expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware)
|
107
|
+
expect(server_middleware).to receive(:add).with(CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware)
|
107
108
|
expect(death_handlers).to receive(:<<)
|
108
|
-
CanvasSync::JobBatches::Sidekiq.
|
109
|
+
CanvasSync::JobBatches::Compat::Sidekiq.instance_variable_set(:@already_configured, false)
|
110
|
+
CanvasSync::JobBatches::Compat::Sidekiq.configure
|
109
111
|
end
|
110
112
|
end
|
111
113
|
|
@@ -2,7 +2,7 @@ require 'spec_helper'
|
|
2
2
|
require 'sidekiq/testing'
|
3
3
|
|
4
4
|
Sidekiq::Testing.server_middleware do |chain|
|
5
|
-
chain.add CanvasSync::JobBatches::Sidekiq::ServerMiddleware
|
5
|
+
chain.add CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware
|
6
6
|
end
|
7
7
|
|
8
8
|
CanvasSync.redis { |r| r.flushdb }
|
@@ -19,7 +19,7 @@ RSpec.describe CanvasSync::JobBatches::Batch::Status do
|
|
19
19
|
end
|
20
20
|
|
21
21
|
context 'when more than 0' do
|
22
|
-
before { batch.jobs do
|
22
|
+
before { batch.jobs do BatchTestWorker.perform_async end }
|
23
23
|
it 'returns pending jobs' do
|
24
24
|
expect(subject.pending).to eq(1)
|
25
25
|
end
|
@@ -68,7 +68,7 @@ RSpec.describe CanvasSync::JobBatches::Batch::Status do
|
|
68
68
|
describe '#created_at' do
|
69
69
|
it 'returns time' do
|
70
70
|
batch = CanvasSync::JobBatches::Batch.new
|
71
|
-
batch.jobs do
|
71
|
+
batch.jobs do BatchTestWorker.perform_async end
|
72
72
|
status = described_class.new(batch.bid)
|
73
73
|
expect(status.created_at).not_to be_nil
|
74
74
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'job_uniqueness/spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe CanvasSync::JobUniqueness::Compat::ActiveJob do
|
4
|
+
context 'Job Extension' do
|
5
|
+
it 'includes UniqueJobExtension' do
|
6
|
+
expect(ActiveJob::Base < CanvasSync::JobUniqueness::Compat::ActiveJob::JobExtension).to be true
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'has the ensure_uniqueness method' do
|
10
|
+
expect(ActiveJob::Base.method(:ensure_uniqueness)).to be_present
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
context 'Job' do
|
15
|
+
let(:test_job) do
|
16
|
+
Class.new(ActiveJob::Base) do
|
17
|
+
ensure_uniqueness(
|
18
|
+
strategy: :until_executed,
|
19
|
+
)
|
20
|
+
def perform; end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
before(:each) do
|
25
|
+
stub_const('TestJob', test_job)
|
26
|
+
ActiveJob::Base.queue_adapter = :sidekiq
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'runs as expected' do
|
30
|
+
strategy = CanvasSync::JobUniqueness::Strategy::UntilExecuted.new(nil)
|
31
|
+
allow_any_instance_of(CanvasSync::JobUniqueness::LockContext).to receive(:lock_strategy) do |lock_context|
|
32
|
+
strategy.instance_variable_set(:@lock_context, lock_context)
|
33
|
+
strategy
|
34
|
+
end
|
35
|
+
allow(CanvasSync::JobUniqueness::Strategy::UntilExecuted).to receive(:new).and_return(strategy)
|
36
|
+
|
37
|
+
expect(strategy).to receive(:on_enqueue).and_call_original
|
38
|
+
expect(strategy).to receive(:on_perform).and_call_original
|
39
|
+
expect(strategy).to receive(:batch_callback).with(:complete, anything).and_call_original
|
40
|
+
expect(strategy).to receive(:batch_callback).with(:success, anything).and_call_original
|
41
|
+
|
42
|
+
expect_any_instance_of(test_job).to receive(:perform)
|
43
|
+
|
44
|
+
Sidekiq::Testing.inline! do
|
45
|
+
test_job.perform_later
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|