canvas_sync 0.21.1 → 0.22.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/canvas_sync/concerns/auto_relations.rb +11 -0
- data/lib/canvas_sync/config.rb +3 -5
- data/lib/canvas_sync/generators/templates/models/rubric.rb +2 -1
- data/lib/canvas_sync/job_batches/batch.rb +432 -402
- data/lib/canvas_sync/job_batches/callback.rb +100 -114
- data/lib/canvas_sync/job_batches/chain_builder.rb +194 -196
- data/lib/canvas_sync/job_batches/{active_job.rb → compat/active_job.rb} +2 -2
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/helpers.rb +1 -1
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web.rb +3 -3
- data/lib/canvas_sync/job_batches/{sidekiq.rb → compat/sidekiq.rb} +35 -22
- data/lib/canvas_sync/job_batches/compat.rb +20 -0
- data/lib/canvas_sync/job_batches/context_hash.rb +124 -126
- data/lib/canvas_sync/job_batches/jobs/base_job.rb +2 -4
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +125 -127
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/pool.rb +193 -195
- data/lib/canvas_sync/job_batches/redis_model.rb +50 -52
- data/lib/canvas_sync/job_batches/redis_script.rb +129 -131
- data/lib/canvas_sync/job_batches/status.rb +85 -87
- data/lib/canvas_sync/job_uniqueness/compat/active_job.rb +75 -0
- data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb +135 -0
- data/lib/canvas_sync/job_uniqueness/compat.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/configuration.rb +25 -0
- data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb +47 -0
- data/lib/canvas_sync/job_uniqueness/lock_context.rb +171 -0
- data/lib/canvas_sync/job_uniqueness/locksmith.rb +92 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/base.rb +32 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/log.rb +13 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/null_strategy.rb +9 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/raise.rb +11 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reject.rb +21 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reschedule.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict.rb +41 -0
- data/lib/canvas_sync/job_uniqueness/strategy/base.rb +104 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_and_while_executing.rb +35 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executed.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executing.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_expired.rb +16 -0
- data/lib/canvas_sync/job_uniqueness/strategy/while_executing.rb +26 -0
- data/lib/canvas_sync/job_uniqueness/strategy.rb +27 -0
- data/lib/canvas_sync/job_uniqueness/unique_job_common.rb +79 -0
- data/lib/canvas_sync/misc_helper.rb +1 -1
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +4 -3
- data/spec/dummy/app/models/rubric.rb +2 -1
- data/spec/dummy/config/environments/test.rb +1 -1
- data/spec/job_batching/batch_spec.rb +49 -7
- data/spec/job_batching/{active_job_spec.rb → compat/active_job_spec.rb} +2 -2
- data/spec/job_batching/{sidekiq_spec.rb → compat/sidekiq_spec.rb} +14 -12
- data/spec/job_batching/flow_spec.rb +1 -1
- data/spec/job_batching/integration_helper.rb +1 -1
- data/spec/job_batching/status_spec.rb +2 -2
- data/spec/job_uniqueness/compat/active_job_spec.rb +49 -0
- data/spec/job_uniqueness/compat/sidekiq_spec.rb +68 -0
- data/spec/job_uniqueness/lock_context_spec.rb +95 -0
- data/spec/job_uniqueness/on_conflict/log_spec.rb +11 -0
- data/spec/job_uniqueness/on_conflict/raise_spec.rb +10 -0
- data/spec/job_uniqueness/on_conflict/reschedule_spec.rb +24 -0
- data/spec/job_uniqueness/on_conflict_spec.rb +16 -0
- data/spec/job_uniqueness/spec_helper.rb +14 -0
- data/spec/job_uniqueness/strategy/base_spec.rb +100 -0
- data/spec/job_uniqueness/strategy/until_and_while_executing_spec.rb +48 -0
- data/spec/job_uniqueness/strategy/until_executed_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_executing_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_expired_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/while_executing_spec.rb +33 -0
- data/spec/job_uniqueness/support/lock_strategy.rb +28 -0
- data/spec/job_uniqueness/support/on_conflict.rb +24 -0
- data/spec/job_uniqueness/support/test_worker.rb +19 -0
- data/spec/job_uniqueness/unique_job_common_spec.rb +45 -0
- data/spec/spec_helper.rb +1 -1
- metadata +278 -204
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/css/styles.less +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/batch_tree.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/util.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batch_tree.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batches_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_common.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_jobs_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_pagination.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batch.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batches.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pool.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pools.erb +0 -0
@@ -0,0 +1,104 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
class Base
|
4
|
+
attr_reader :lock_context
|
5
|
+
|
6
|
+
def initialize(lock_context)
|
7
|
+
@lock_context = lock_context
|
8
|
+
@conflict_strategies = {}
|
9
|
+
end
|
10
|
+
|
11
|
+
class_attribute :_locks_on, instance_writer: false
|
12
|
+
|
13
|
+
def self.locks_on(*origins)
|
14
|
+
if origins.present?
|
15
|
+
orgins = Array(origins).map(&:to_sym)
|
16
|
+
self._locks_on = origins
|
17
|
+
else
|
18
|
+
self._locks_on || [:enqueue, :perform]
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def on_enqueue; end
|
23
|
+
def on_perform; end
|
24
|
+
|
25
|
+
def batch_callback(event, batch_status)
|
26
|
+
if event == :success
|
27
|
+
unlock
|
28
|
+
else
|
29
|
+
unlock_cond = lock_context.config[:unlock_on_failure]
|
30
|
+
|
31
|
+
if (event == :complete && unlock_cond == :any) || (event == :death && unlock_cond == :death) || (event == :stagnated && unlock_cond == :stagnant)
|
32
|
+
unlock
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
delegate :locked?, to: :locksmith
|
38
|
+
|
39
|
+
protected
|
40
|
+
|
41
|
+
def key
|
42
|
+
lock_context.base_key
|
43
|
+
end
|
44
|
+
|
45
|
+
def wrap_in_batch(&blk)
|
46
|
+
if Thread.current[:unique_jobs_previous_jid] # Ensure we don't re-wrap in a batch when rescheduling
|
47
|
+
return blk.call
|
48
|
+
end
|
49
|
+
|
50
|
+
batch = CanvasSync::JobBatches::Batch.new
|
51
|
+
batch.context = {
|
52
|
+
uniqueness_lock_key: key,
|
53
|
+
}
|
54
|
+
|
55
|
+
CanvasSync::JobBatches::Batch::Callback::VALID_CALLBACKS.each do |callback|
|
56
|
+
callback = callback.to_sym
|
57
|
+
batch.on(callback, self.class.to_s + ".internal_batch_callback", {
|
58
|
+
event: callback,
|
59
|
+
lock_strategy: self.class.to_s,
|
60
|
+
lock_key: key,
|
61
|
+
lock_context: lock_context.serialize,
|
62
|
+
})
|
63
|
+
end
|
64
|
+
|
65
|
+
batch.jobs do
|
66
|
+
return blk.call
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.internal_batch_callback(batch_status, opts)
|
71
|
+
strategy_class = opts[:lock_strategy].constantize
|
72
|
+
lock_context = LockContext.from_serialized(opts[:lock_context])
|
73
|
+
strategy = strategy_class.new(lock_context)
|
74
|
+
# TODO Should this route through LockContext#handle_lifecycle!?
|
75
|
+
strategy.batch_callback(opts[:event].to_sym, batch_status)
|
76
|
+
end
|
77
|
+
|
78
|
+
def lock!(purpose, wait: nil)
|
79
|
+
locked = nil
|
80
|
+
if purpose == :enqueue
|
81
|
+
if Thread.current[:unique_jobs_previous_jid].present?
|
82
|
+
locked = locksmith.swap_locks(Thread.current[:unique_jobs_previous_jid])
|
83
|
+
else
|
84
|
+
locked = locksmith.lock()
|
85
|
+
end
|
86
|
+
elsif purpose == :perform
|
87
|
+
locked = locksmith.execute { lock_context.job_id }
|
88
|
+
end
|
89
|
+
|
90
|
+
CanvasSync::JobUniqueness.logger.debug { "Requested lock of #{key} for #{purpose} - (#{locked || 'Not Obtained!'})" }
|
91
|
+
|
92
|
+
raise CouldNotLockError.new(lock_context, source: purpose) if !locked
|
93
|
+
end
|
94
|
+
|
95
|
+
def unlock()
|
96
|
+
locksmith.unlock
|
97
|
+
end
|
98
|
+
|
99
|
+
def locksmith
|
100
|
+
@locksmith ||= Locksmith.new(key, lock_context)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
# Implements two locks - one while enqueued and one while performing
|
4
|
+
class UntilAndWhileExecuting < Base
|
5
|
+
locks_on :enqueue, :perform
|
6
|
+
|
7
|
+
def on_enqueue
|
8
|
+
# Obtain lock
|
9
|
+
lock!(:enqueue)
|
10
|
+
|
11
|
+
# Proceed with enqueuing the job, wrapping it in a batch
|
12
|
+
runtime_lock.on_enqueue do
|
13
|
+
yield
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def on_perform
|
18
|
+
# Obtain Runtime lock
|
19
|
+
runtime_lock.on_perform do
|
20
|
+
# Release Queue lock
|
21
|
+
unlock()
|
22
|
+
|
23
|
+
# Run the job
|
24
|
+
yield
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def runtime_lock
|
31
|
+
@runtime_lock ||= Strategy::WhileExecuting.new(lock_context)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
class UntilExecuted < Base
|
4
|
+
locks_on :enqueue, :perform
|
5
|
+
|
6
|
+
def on_enqueue
|
7
|
+
lock!(:enqueue)
|
8
|
+
|
9
|
+
wrap_in_batch do
|
10
|
+
yield
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def on_perform
|
15
|
+
lock!(:perform)
|
16
|
+
yield
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
class UntilExecuting < Base
|
4
|
+
locks_on :enqueue
|
5
|
+
|
6
|
+
def on_enqueue
|
7
|
+
lock!(:enqueue)
|
8
|
+
yield
|
9
|
+
end
|
10
|
+
|
11
|
+
def on_perform
|
12
|
+
unlock
|
13
|
+
yield
|
14
|
+
end
|
15
|
+
|
16
|
+
# TODO Define behavior when an error occurs during perform().
|
17
|
+
# SUJ's behavior is to relock, but this has some edge-cases (like how do we handle if another job already took the lock?)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
class WhileExecuting < Base
|
4
|
+
locks_on :perform
|
5
|
+
|
6
|
+
RUN_SUFFIX = ":RUN"
|
7
|
+
|
8
|
+
def on_enqueue
|
9
|
+
wrap_in_batch do
|
10
|
+
yield
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def on_perform
|
15
|
+
lock!(:perform)
|
16
|
+
yield
|
17
|
+
end
|
18
|
+
|
19
|
+
protected
|
20
|
+
|
21
|
+
def key
|
22
|
+
super + RUN_SUFFIX
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module CanvasSync::JobUniqueness
|
2
|
+
module Strategy
|
3
|
+
extend ActiveSupport::Autoload
|
4
|
+
|
5
|
+
autoload :Base
|
6
|
+
|
7
|
+
autoload :UntilExpired
|
8
|
+
autoload :UntilExecuted
|
9
|
+
autoload :UntilExecuting
|
10
|
+
autoload :UntilAndWhileExecuting
|
11
|
+
autoload :WhileExecuting
|
12
|
+
|
13
|
+
class << self
|
14
|
+
def lookup(strategy)
|
15
|
+
matching_strategy(strategy.to_s.camelize) ||
|
16
|
+
CanvasSync::JobUniqueness.config.lock_strategies[strategy] ||
|
17
|
+
raise(ArgumentError, "strategy: #{strategy} is not found. Is it declared in the configuration?")
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def matching_strategy(const)
|
23
|
+
const_get(const, false) if const_defined?(const, false)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
|
2
|
+
module CanvasSync::JobUniqueness
|
3
|
+
module UniqueJobCommon
|
4
|
+
extend ActiveSupport::Concern
|
5
|
+
|
6
|
+
included do
|
7
|
+
class_attribute :unique_job_options, instance_writer: false
|
8
|
+
end
|
9
|
+
|
10
|
+
class_methods do
|
11
|
+
# ensure_uniqueness(
|
12
|
+
# strategy: :until_executed, # :until_executed, :until_executing, :until_expired, :until_and_while_executing, :while_executing
|
13
|
+
# on_conflict: :raise, # :raise, :replace, :log, :reject, :reschedule, { enqueue: ..., perform: ... }, proc
|
14
|
+
# lock_ttl: 7.days, # seconds
|
15
|
+
# lock_timeout: 0, # seconds
|
16
|
+
|
17
|
+
# scope: :per_queue, # :global, :per_queue, string ("<class>-<queue>"), proc
|
18
|
+
# hash: ->{ { ... } },
|
19
|
+
|
20
|
+
# # In the case of UntilExecuted and WhileExecuting, how should the execution lock be released in an error condition?
|
21
|
+
# # :any - Release the lock when the Job's (implicit) Batch is :complete
|
22
|
+
# # :death - Release the lock when the Job's Batch receives the :death callback
|
23
|
+
# # :stagnant - Release the lock when the Job's Batch receives the :stagnant callback
|
24
|
+
# # :expire - Do not release the lock until it expires
|
25
|
+
# unlock_on_failure: :stagnant,
|
26
|
+
# )
|
27
|
+
def ensure_uniqueness(**kwargs)
|
28
|
+
if self.unique_job_options.present?
|
29
|
+
raise ArgumentError, "ensure_uniqueness can only be called once per job class"
|
30
|
+
end
|
31
|
+
|
32
|
+
begin
|
33
|
+
require "sidekiq_unique_jobs"
|
34
|
+
rescue LoadError
|
35
|
+
raise LoadError, "SidekiqUniqueJobs is required for ensure_uniqueness"
|
36
|
+
end
|
37
|
+
|
38
|
+
OnConflict.validate!(kwargs[:on_conflict], kwargs[:strategy]) if kwargs[:on_conflict].present?
|
39
|
+
|
40
|
+
kwargs[:scope] ||= :per_queue
|
41
|
+
kwargs[:ttl] ||= 30.days.to_i
|
42
|
+
kwargs[:timeout] ||= 0
|
43
|
+
kwargs[:limit] ||= 1
|
44
|
+
|
45
|
+
self.unique_job_options = kwargs
|
46
|
+
|
47
|
+
include UniqueJobMethods
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
module UniqueJobMethods
|
52
|
+
extend ActiveSupport::Concern
|
53
|
+
|
54
|
+
class_methods do
|
55
|
+
def unlock!(jid = nil, args: nil, kwargs: nil, queue: nil)
|
56
|
+
queue = self.try(:default_queue_name) || try(:queue)
|
57
|
+
raise ArgumentError, "Must specify queue:" unless queue.is_a?(String) || queue.is_a?(Symbol)
|
58
|
+
|
59
|
+
temp_context = LockContext.new({ job_clazz: self, jid: jid, queue: queue, args: args, kwargs: kwargs })
|
60
|
+
strategy = temp_context.lock_strategy
|
61
|
+
locksmith = strategy.send(:locksmith)
|
62
|
+
|
63
|
+
if jid
|
64
|
+
locksmith.unlock!()
|
65
|
+
else
|
66
|
+
locksmith.locked_jids.each do |jid|
|
67
|
+
unlock!(jid, args: args, kwargs: kwargs, queue: queue)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
# def unlock_all!(queue: :all)
|
73
|
+
# # TODO Public API to manually remove all locks for this class
|
74
|
+
# end
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -34,7 +34,7 @@ module CanvasSync
|
|
34
34
|
# TODO Support globalid
|
35
35
|
model_class = load_constant(job[:model])
|
36
36
|
find_by = job[:find_by]
|
37
|
-
target = find_by.is_a?(Hash) ? model_class.find_by(find_by) : model_class.find_by(id: find_by)
|
37
|
+
target = find_by.is_a?(Hash) ? model_class.find_by(**find_by) : model_class.find_by(id: find_by)
|
38
38
|
target.send(job[:method], *job_args, **job_kwargs)
|
39
39
|
elsif job[:class]
|
40
40
|
target = load_constant(job[:class])
|
data/lib/canvas_sync/version.rb
CHANGED
data/lib/canvas_sync.rb
CHANGED
@@ -16,6 +16,7 @@ require "canvas_sync/batch_processor"
|
|
16
16
|
require "canvas_sync/config"
|
17
17
|
|
18
18
|
require "canvas_sync/job_batches/batch"
|
19
|
+
require "canvas_sync/job_uniqueness/job_uniqueness"
|
19
20
|
|
20
21
|
Dir[File.dirname(__FILE__) + "/canvas_sync/jobs/*.rb"].each { |file| require file }
|
21
22
|
Dir[File.dirname(__FILE__) + "/canvas_sync/processors/*.rb"].each { |file| require file }
|
@@ -249,7 +250,7 @@ module CanvasSync
|
|
249
250
|
global_options[:account_id] = account_id if account_id.present?
|
250
251
|
global_options.merge!(globals) if globals
|
251
252
|
|
252
|
-
JobBatches::ChainBuilder.build(CanvasSync::Jobs::BeginSyncChainJob, [], global_options, &blk)
|
253
|
+
JobBatches::ChainBuilder.build(CanvasSync::Jobs::BeginSyncChainJob, *[], **global_options, &blk)
|
253
254
|
end
|
254
255
|
|
255
256
|
def group_by_job_options(model_list, options_hash, only_split: nil, default_key: :provisioning)
|
@@ -338,7 +339,7 @@ module CanvasSync
|
|
338
339
|
def logger
|
339
340
|
return @logger if defined? @logger
|
340
341
|
@logger = Logger.new(STDOUT)
|
341
|
-
@logger.level = Logger::
|
342
|
+
@logger.level = Logger::WARN
|
342
343
|
@logger
|
343
344
|
end
|
344
345
|
|
@@ -347,7 +348,7 @@ module CanvasSync
|
|
347
348
|
end
|
348
349
|
|
349
350
|
def redis_prefix
|
350
|
-
pfx =
|
351
|
+
pfx = config.redis_key_prefix
|
351
352
|
pfx = "#{Apartment::Tenant.current}:#{pfx}" if defined?(Apartment)
|
352
353
|
pfx
|
353
354
|
end
|
@@ -17,8 +17,9 @@ class Rubric < ApplicationRecord
|
|
17
17
|
belongs_to :rubric, primary_key: :canvas_id, foreign_key: :canvas_rubric_id , optional: true # based on another rubric
|
18
18
|
belongs_to :context, polymorphic: true, primary_key: :canvas_id, foreign_key: :canvas_context_id, foreign_type: :canvas_context_type, optional: true
|
19
19
|
|
20
|
-
has_many :rubric_associations, primary_key: :canvas_id, foreign_key: :canvas_association_id, foreign_type: :canvas_association_type
|
21
20
|
has_many :rubric_assessments, through: :rubric_associations, dependent: :destroy
|
21
|
+
has_many :rubric_associations, primary_key: :canvas_id, foreign_key: :canvas_rubric_id
|
22
|
+
has_many :associated_objects, through: :rubric_associations, source: :association_object
|
22
23
|
|
23
24
|
api_syncable({
|
24
25
|
canvas_id: :id,
|
@@ -5,7 +5,7 @@ Rails.application.configure do
|
|
5
5
|
# test suite. You never need to work with it otherwise. Remember that
|
6
6
|
# your test database is "scratch space" for the test suite and is wiped
|
7
7
|
# and recreated between test runs. Don't rely on the data there!
|
8
|
-
config.cache_classes =
|
8
|
+
config.cache_classes = false
|
9
9
|
|
10
10
|
# Do not eager load code on boot. This avoids loading your whole application
|
11
11
|
# just for the purpose of running a single test. If you are using a tool that
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
class
|
3
|
+
class BatchTestWorker < BatchTestJobBase
|
4
4
|
def perform
|
5
5
|
end
|
6
6
|
end
|
@@ -173,6 +173,48 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
173
173
|
end
|
174
174
|
end
|
175
175
|
|
176
|
+
describe '#process_dead_job' do
|
177
|
+
let(:batch) { CanvasSync::JobBatches::Batch.new }
|
178
|
+
let(:bid) { batch.bid }
|
179
|
+
let(:jid) { 'ABCD' }
|
180
|
+
before { CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } }
|
181
|
+
|
182
|
+
context 'dead' do
|
183
|
+
let(:failed_jid) { 'xxx' }
|
184
|
+
|
185
|
+
it 'tries to call death callback' do
|
186
|
+
allow(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:stagnated, bid)
|
187
|
+
expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:death, bid)
|
188
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
189
|
+
end
|
190
|
+
|
191
|
+
it 'add job to failed list' do
|
192
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, 'failed-job-id')
|
193
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
194
|
+
failed = CanvasSync::JobBatches::Batch.redis { |r| r.smembers("BID-#{bid}-dead") }
|
195
|
+
expect(failed).to contain_exactly('xxx', 'failed-job-id')
|
196
|
+
end
|
197
|
+
|
198
|
+
it 'automattically triggers the :stagnated callback' do
|
199
|
+
expect(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:stagnated, bid)
|
200
|
+
allow(CanvasSync::JobBatches::Batch).to receive(:enqueue_callbacks).with(:death, bid)
|
201
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
202
|
+
end
|
203
|
+
|
204
|
+
it 'does not trigger :stagnated if pending jobs are still present' do
|
205
|
+
CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 2) }
|
206
|
+
expect(CanvasSync::JobBatches::Batch).to_not receive(:enqueue_callbacks).with(:stagnated, bid)
|
207
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
208
|
+
end
|
209
|
+
|
210
|
+
it 'does not trigger :stagnated if pending batches are still present' do
|
211
|
+
CanvasSync::JobBatches::Batch.redis { |r| r.hset("BID-#{bid}", 'pending', 2) }
|
212
|
+
expect(CanvasSync::JobBatches::Batch).to_not receive(:enqueue_callbacks).with(:stagnated, bid)
|
213
|
+
CanvasSync::JobBatches::Batch.process_dead_job(bid, failed_jid)
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
176
218
|
describe '#process_successful_job' do
|
177
219
|
let(:batch) { CanvasSync::JobBatches::Batch.new }
|
178
220
|
let(:bid) { batch.bid }
|
@@ -182,7 +224,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
182
224
|
context 'complete' do
|
183
225
|
before { batch.on(:complete, Object) }
|
184
226
|
# before { batch.increment_job_queue(bid) }
|
185
|
-
# before { batch.jobs do
|
227
|
+
# before { batch.jobs do BatchTestWorker.perform_async end }
|
186
228
|
# before { CanvasSync::JobBatches::Batch.process_failed_job(bid, 'failed-job-id') }
|
187
229
|
|
188
230
|
it 'tries to call complete callback' do
|
@@ -211,7 +253,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
211
253
|
|
212
254
|
it 'triggers callbacks as expected' do
|
213
255
|
ActiveJob::Base.queue_adapter = :sidekiq
|
214
|
-
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
|
256
|
+
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Compat::Sidekiq::SidekiqCallbackWorker
|
215
257
|
|
216
258
|
callback_instance = double('SampleCallback')
|
217
259
|
expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
|
@@ -229,7 +271,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
229
271
|
|
230
272
|
it 'delays triggering callbacks if keep_open is set' do
|
231
273
|
ActiveJob::Base.queue_adapter = :sidekiq
|
232
|
-
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
|
274
|
+
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Compat::Sidekiq::SidekiqCallbackWorker
|
233
275
|
|
234
276
|
callback_instance = double('SampleCallback')
|
235
277
|
expect(SampleCallback).to receive(:new).at_least(1).times.and_return(callback_instance)
|
@@ -303,7 +345,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
303
345
|
let(:batch) { CanvasSync::JobBatches::Batch.new }
|
304
346
|
|
305
347
|
it 'increments pending' do
|
306
|
-
batch.jobs do
|
348
|
+
batch.jobs do BatchTestWorker.perform_async end
|
307
349
|
pending = CanvasSync::JobBatches::Batch.redis { |r| r.hget("BID-#{batch.bid}", 'pending') }
|
308
350
|
expect(pending).to eq('1')
|
309
351
|
end
|
@@ -338,7 +380,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
338
380
|
|
339
381
|
context 'With ActiveJob Adapter' do
|
340
382
|
around(:all) do |block|
|
341
|
-
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::ActiveJob::ActiveJobCallbackWorker
|
383
|
+
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Compat::ActiveJob::ActiveJobCallbackWorker
|
342
384
|
block.run
|
343
385
|
end
|
344
386
|
|
@@ -396,7 +438,7 @@ RSpec.describe CanvasSync::JobBatches::Batch do
|
|
396
438
|
|
397
439
|
context 'With Sidekiq Adapter' do
|
398
440
|
around(:all) do |block|
|
399
|
-
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Sidekiq::SidekiqCallbackWorker
|
441
|
+
CanvasSync::JobBatches::Batch::Callback.worker_class = CanvasSync::JobBatches::Compat::Sidekiq::SidekiqCallbackWorker
|
400
442
|
block.run
|
401
443
|
end
|
402
444
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
RSpec.describe CanvasSync::JobBatches::ActiveJob do
|
4
|
-
describe CanvasSync::JobBatches::ActiveJob::BatchAwareJob do
|
3
|
+
RSpec.describe CanvasSync::JobBatches::Compat::ActiveJob do
|
4
|
+
describe CanvasSync::JobBatches::Compat::ActiveJob::BatchAwareJob do
|
5
5
|
include ActiveJob::TestHelper
|
6
6
|
|
7
7
|
after do
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
4
|
-
describe CanvasSync::JobBatches::Sidekiq::ServerMiddleware do
|
3
|
+
RSpec.describe CanvasSync::JobBatches::Compat::Sidekiq do
|
4
|
+
describe CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware do
|
5
5
|
context 'when without batch' do
|
6
6
|
it 'just yields' do
|
7
7
|
yielded = false
|
@@ -44,7 +44,7 @@ RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
47
|
-
describe CanvasSync::JobBatches::Sidekiq::ClientMiddleware do
|
47
|
+
describe CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware do
|
48
48
|
context 'when without batch' do
|
49
49
|
it 'just yields' do
|
50
50
|
yielded = false
|
@@ -80,16 +80,17 @@ RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
|
80
80
|
end
|
81
81
|
end
|
82
82
|
|
83
|
-
RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
84
|
-
let(:config) { class_double(Sidekiq) }
|
83
|
+
RSpec.describe CanvasSync::JobBatches::Compat::Sidekiq do
|
84
|
+
let(:config) { defined?(Sidekiq::Config) ? double(Sidekiq::Config) : class_double(Sidekiq) }
|
85
85
|
let(:client_middleware) { double(Sidekiq::Middleware::Chain) }
|
86
86
|
|
87
87
|
context 'client' do
|
88
88
|
it 'adds client middleware' do
|
89
|
-
|
89
|
+
allow(Sidekiq).to receive(:configure_client).and_yield(config)
|
90
90
|
expect(config).to receive(:client_middleware).and_yield(client_middleware)
|
91
|
-
expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
|
92
|
-
CanvasSync::JobBatches::Sidekiq.
|
91
|
+
expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware)
|
92
|
+
CanvasSync::JobBatches::Compat::Sidekiq.instance_variable_set(:@already_configured, false)
|
93
|
+
CanvasSync::JobBatches::Compat::Sidekiq.configure
|
93
94
|
end
|
94
95
|
end
|
95
96
|
|
@@ -98,14 +99,15 @@ RSpec.describe CanvasSync::JobBatches::Sidekiq do
|
|
98
99
|
let(:death_handlers) { double(Array) }
|
99
100
|
|
100
101
|
it 'adds client and server middleware' do
|
101
|
-
|
102
|
+
allow(Sidekiq).to receive(:configure_server).and_yield(config)
|
102
103
|
expect(config).to receive(:client_middleware).and_yield(client_middleware)
|
103
104
|
expect(config).to receive(:server_middleware).and_yield(server_middleware)
|
104
105
|
expect(config).to receive(:death_handlers).and_return(death_handlers)
|
105
|
-
expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ClientMiddleware)
|
106
|
-
expect(server_middleware).to receive(:add).with(CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
|
106
|
+
expect(client_middleware).to receive(:add).with(CanvasSync::JobBatches::Compat::Sidekiq::ClientMiddleware)
|
107
|
+
expect(server_middleware).to receive(:add).with(CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware)
|
107
108
|
expect(death_handlers).to receive(:<<)
|
108
|
-
CanvasSync::JobBatches::Sidekiq.
|
109
|
+
CanvasSync::JobBatches::Compat::Sidekiq.instance_variable_set(:@already_configured, false)
|
110
|
+
CanvasSync::JobBatches::Compat::Sidekiq.configure
|
109
111
|
end
|
110
112
|
end
|
111
113
|
|
@@ -2,7 +2,7 @@ require 'spec_helper'
|
|
2
2
|
require 'sidekiq/testing'
|
3
3
|
|
4
4
|
Sidekiq::Testing.server_middleware do |chain|
|
5
|
-
chain.add CanvasSync::JobBatches::Sidekiq::ServerMiddleware
|
5
|
+
chain.add CanvasSync::JobBatches::Compat::Sidekiq::ServerMiddleware
|
6
6
|
end
|
7
7
|
|
8
8
|
CanvasSync.redis { |r| r.flushdb }
|
@@ -19,7 +19,7 @@ RSpec.describe CanvasSync::JobBatches::Batch::Status do
|
|
19
19
|
end
|
20
20
|
|
21
21
|
context 'when more than 0' do
|
22
|
-
before { batch.jobs do
|
22
|
+
before { batch.jobs do BatchTestWorker.perform_async end }
|
23
23
|
it 'returns pending jobs' do
|
24
24
|
expect(subject.pending).to eq(1)
|
25
25
|
end
|
@@ -68,7 +68,7 @@ RSpec.describe CanvasSync::JobBatches::Batch::Status do
|
|
68
68
|
describe '#created_at' do
|
69
69
|
it 'returns time' do
|
70
70
|
batch = CanvasSync::JobBatches::Batch.new
|
71
|
-
batch.jobs do
|
71
|
+
batch.jobs do BatchTestWorker.perform_async end
|
72
72
|
status = described_class.new(batch.bid)
|
73
73
|
expect(status.created_at).not_to be_nil
|
74
74
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'job_uniqueness/spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe CanvasSync::JobUniqueness::Compat::ActiveJob do
|
4
|
+
context 'Job Extension' do
|
5
|
+
it 'includes UniqueJobExtension' do
|
6
|
+
expect(ActiveJob::Base < CanvasSync::JobUniqueness::Compat::ActiveJob::JobExtension).to be true
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'has the ensure_uniqueness method' do
|
10
|
+
expect(ActiveJob::Base.method(:ensure_uniqueness)).to be_present
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
context 'Job' do
|
15
|
+
let(:test_job) do
|
16
|
+
Class.new(ActiveJob::Base) do
|
17
|
+
ensure_uniqueness(
|
18
|
+
strategy: :until_executed,
|
19
|
+
)
|
20
|
+
def perform; end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
before(:each) do
|
25
|
+
stub_const('TestJob', test_job)
|
26
|
+
ActiveJob::Base.queue_adapter = :sidekiq
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'runs as expected' do
|
30
|
+
strategy = CanvasSync::JobUniqueness::Strategy::UntilExecuted.new(nil)
|
31
|
+
allow_any_instance_of(CanvasSync::JobUniqueness::LockContext).to receive(:lock_strategy) do |lock_context|
|
32
|
+
strategy.instance_variable_set(:@lock_context, lock_context)
|
33
|
+
strategy
|
34
|
+
end
|
35
|
+
allow(CanvasSync::JobUniqueness::Strategy::UntilExecuted).to receive(:new).and_return(strategy)
|
36
|
+
|
37
|
+
expect(strategy).to receive(:on_enqueue).and_call_original
|
38
|
+
expect(strategy).to receive(:on_perform).and_call_original
|
39
|
+
expect(strategy).to receive(:batch_callback).with(:complete, anything).and_call_original
|
40
|
+
expect(strategy).to receive(:batch_callback).with(:success, anything).and_call_original
|
41
|
+
|
42
|
+
expect_any_instance_of(test_job).to receive(:perform)
|
43
|
+
|
44
|
+
Sidekiq::Testing.inline! do
|
45
|
+
test_job.perform_later
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|