canvas_sync 0.21.1 → 0.22.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/canvas_sync/concerns/auto_relations.rb +11 -0
- data/lib/canvas_sync/config.rb +3 -5
- data/lib/canvas_sync/generators/templates/models/rubric.rb +2 -1
- data/lib/canvas_sync/job_batches/batch.rb +432 -402
- data/lib/canvas_sync/job_batches/callback.rb +100 -114
- data/lib/canvas_sync/job_batches/chain_builder.rb +194 -196
- data/lib/canvas_sync/job_batches/{active_job.rb → compat/active_job.rb} +2 -2
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/helpers.rb +1 -1
- data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web.rb +3 -3
- data/lib/canvas_sync/job_batches/{sidekiq.rb → compat/sidekiq.rb} +35 -22
- data/lib/canvas_sync/job_batches/compat.rb +20 -0
- data/lib/canvas_sync/job_batches/context_hash.rb +124 -126
- data/lib/canvas_sync/job_batches/jobs/base_job.rb +2 -4
- data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +125 -127
- data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +14 -16
- data/lib/canvas_sync/job_batches/pool.rb +193 -195
- data/lib/canvas_sync/job_batches/redis_model.rb +50 -52
- data/lib/canvas_sync/job_batches/redis_script.rb +129 -131
- data/lib/canvas_sync/job_batches/status.rb +85 -87
- data/lib/canvas_sync/job_uniqueness/compat/active_job.rb +75 -0
- data/lib/canvas_sync/job_uniqueness/compat/sidekiq.rb +135 -0
- data/lib/canvas_sync/job_uniqueness/compat.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/configuration.rb +25 -0
- data/lib/canvas_sync/job_uniqueness/job_uniqueness.rb +47 -0
- data/lib/canvas_sync/job_uniqueness/lock_context.rb +171 -0
- data/lib/canvas_sync/job_uniqueness/locksmith.rb +92 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/base.rb +32 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/log.rb +13 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/null_strategy.rb +9 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/raise.rb +11 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reject.rb +21 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict/reschedule.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/on_conflict.rb +41 -0
- data/lib/canvas_sync/job_uniqueness/strategy/base.rb +104 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_and_while_executing.rb +35 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executed.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_executing.rb +20 -0
- data/lib/canvas_sync/job_uniqueness/strategy/until_expired.rb +16 -0
- data/lib/canvas_sync/job_uniqueness/strategy/while_executing.rb +26 -0
- data/lib/canvas_sync/job_uniqueness/strategy.rb +27 -0
- data/lib/canvas_sync/job_uniqueness/unique_job_common.rb +79 -0
- data/lib/canvas_sync/misc_helper.rb +1 -1
- data/lib/canvas_sync/version.rb +1 -1
- data/lib/canvas_sync.rb +4 -3
- data/spec/dummy/app/models/rubric.rb +2 -1
- data/spec/dummy/config/environments/test.rb +1 -1
- data/spec/job_batching/batch_spec.rb +49 -7
- data/spec/job_batching/{active_job_spec.rb → compat/active_job_spec.rb} +2 -2
- data/spec/job_batching/{sidekiq_spec.rb → compat/sidekiq_spec.rb} +14 -12
- data/spec/job_batching/flow_spec.rb +1 -1
- data/spec/job_batching/integration_helper.rb +1 -1
- data/spec/job_batching/status_spec.rb +2 -2
- data/spec/job_uniqueness/compat/active_job_spec.rb +49 -0
- data/spec/job_uniqueness/compat/sidekiq_spec.rb +68 -0
- data/spec/job_uniqueness/lock_context_spec.rb +95 -0
- data/spec/job_uniqueness/on_conflict/log_spec.rb +11 -0
- data/spec/job_uniqueness/on_conflict/raise_spec.rb +10 -0
- data/spec/job_uniqueness/on_conflict/reschedule_spec.rb +24 -0
- data/spec/job_uniqueness/on_conflict_spec.rb +16 -0
- data/spec/job_uniqueness/spec_helper.rb +14 -0
- data/spec/job_uniqueness/strategy/base_spec.rb +100 -0
- data/spec/job_uniqueness/strategy/until_and_while_executing_spec.rb +48 -0
- data/spec/job_uniqueness/strategy/until_executed_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_executing_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/until_expired_spec.rb +23 -0
- data/spec/job_uniqueness/strategy/while_executing_spec.rb +33 -0
- data/spec/job_uniqueness/support/lock_strategy.rb +28 -0
- data/spec/job_uniqueness/support/on_conflict.rb +24 -0
- data/spec/job_uniqueness/support/test_worker.rb +19 -0
- data/spec/job_uniqueness/unique_job_common_spec.rb +45 -0
- data/spec/spec_helper.rb +1 -1
- metadata +278 -204
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/css/styles.less +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/batch_tree.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/batches_assets/js/util.js +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batch_tree.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_batches_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_common.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_jobs_table.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/_pagination.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batch.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/batches.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pool.erb +0 -0
- /data/lib/canvas_sync/job_batches/{sidekiq → compat/sidekiq}/web/views/pools.erb +0 -0
@@ -1,146 +1,132 @@
|
|
1
|
-
module CanvasSync
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
if clazz.
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
target = :class
|
22
|
-
end
|
1
|
+
module CanvasSync::JobBatches
|
2
|
+
class Batch
|
3
|
+
module Callback
|
4
|
+
mattr_accessor :worker_class
|
5
|
+
|
6
|
+
VALID_CALLBACKS = %i[success complete death stagnated].freeze
|
7
|
+
|
8
|
+
module CallbackWorkerCommon
|
9
|
+
def perform(definition, event, opts, bid, parent_bid)
|
10
|
+
return unless VALID_CALLBACKS.include?(event.to_sym)
|
11
|
+
|
12
|
+
method = nil
|
13
|
+
target = :instance
|
14
|
+
clazz = definition
|
15
|
+
if clazz.is_a?(String)
|
16
|
+
if clazz.include?('#')
|
17
|
+
clazz, method = clazz.split("#")
|
18
|
+
elsif clazz.include?('.')
|
19
|
+
clazz, method = clazz.split(".")
|
20
|
+
target = :class
|
23
21
|
end
|
22
|
+
end
|
24
23
|
|
25
|
-
|
26
|
-
|
24
|
+
method ||= "on_#{event}"
|
25
|
+
status = Batch::Status.new(bid)
|
27
26
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
else
|
33
|
-
Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")
|
34
|
-
end
|
27
|
+
if clazz && object = Object.const_get(clazz)
|
28
|
+
target = target == :instance ? object.new : object
|
29
|
+
if target.respond_to?(method, true)
|
30
|
+
target.send(method, status, opts.with_indifferent_access)
|
35
31
|
else
|
36
|
-
Batch.logger.warn("Invalid callback method #{definition} -
|
32
|
+
Batch.logger.warn("Invalid callback method #{definition} - #{target.to_s} does not respond to #{method}")
|
37
33
|
end
|
34
|
+
else
|
35
|
+
Batch.logger.warn("Invalid callback method #{definition} - Class #{clazz} not found")
|
38
36
|
end
|
39
37
|
end
|
38
|
+
end
|
40
39
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
40
|
+
class Finalize
|
41
|
+
# The methods in this class are called after all same-named callbacks have been
|
42
|
+
# completed for the passed Batch.
|
43
|
+
# These methods mainly handle bubbling events up to the parent Batch
|
44
|
+
# You could say that they are the callbacks for callbacks.
|
45
45
|
|
46
|
-
|
46
|
+
def dispatch(status, opts)
|
47
|
+
bid = opts["bid"]
|
48
|
+
event = opts["event"].to_sym
|
47
49
|
|
48
|
-
|
49
|
-
send(event, bid, batch_status, batch_status.parent_bid)
|
50
|
+
Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}"}
|
50
51
|
|
51
|
-
|
52
|
-
|
53
|
-
|
52
|
+
batch_status = Status.new bid
|
53
|
+
send(event, bid, batch_status, batch_status.parent_bid)
|
54
|
+
|
55
|
+
Batch.redis do |r|
|
56
|
+
r.srem("BID-#{bid}-pending_callbacks", "#{event}-finalize")
|
57
|
+
end
|
54
58
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
end
|
59
|
+
if event == :success
|
60
|
+
if opts['origin'].present?
|
61
|
+
# This is a callback for a callback. In this case we need to check if we should cleanup the original bid.
|
62
|
+
origin_bid = opts['origin']['for_bid']
|
63
|
+
_, pending, success_ran = Batch.redis do |r|
|
64
|
+
r.multi do |r|
|
65
|
+
r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
|
66
|
+
r.scard("BID-#{origin_bid}-pending_callbacks")
|
67
|
+
r.hget("BID-#{origin_bid}", "success")
|
65
68
|
end
|
66
|
-
Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
|
67
69
|
end
|
70
|
+
Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
|
71
|
+
end
|
68
72
|
|
69
|
-
|
70
|
-
|
71
|
-
end
|
73
|
+
if (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
|
74
|
+
Batch.cleanup_redis(bid)
|
72
75
|
end
|
73
76
|
end
|
77
|
+
end
|
74
78
|
|
75
|
-
|
76
|
-
|
79
|
+
def success(bid, status, parent_bid)
|
80
|
+
return unless parent_bid
|
77
81
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
82
|
+
Batch.with_callback_check(parent_bid) do |r|
|
83
|
+
r.sadd("BID-#{parent_bid}-batches-success", bid)
|
84
|
+
r.srem("BID-#{parent_bid}-batches-failed", bid)
|
85
|
+
r.sadd("BID-#{parent_bid}-batches-complete", bid)
|
86
|
+
end
|
87
|
+
end
|
83
88
|
|
84
|
-
|
85
|
-
|
86
|
-
r.scard("BID-#{parent_bid}-batches-complete")
|
89
|
+
def complete(bid, status, parent_bid)
|
90
|
+
return unless parent_bid
|
87
91
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
92
|
+
pending, children, success = Batch.redis do |r|
|
93
|
+
r.multi do |r|
|
94
|
+
r.hincrby("BID-#{bid}", "pending", 0)
|
95
|
+
r.hincrby("BID-#{bid}", "children", 0)
|
96
|
+
r.scard("BID-#{bid}-batches-success")
|
93
97
|
end
|
98
|
+
end
|
94
99
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
100
|
+
if !(pending.to_i.zero? && children == success)
|
101
|
+
# If batch was not successfull check and see if its parent is complete
|
102
|
+
# if the parent is complete we can trigger its complete callback.
|
103
|
+
#
|
104
|
+
# Otherwise, we don't want to to trigger the parent's :complete here (and
|
105
|
+
# we instead opt to have success tigger parent :complete) - this
|
106
|
+
# allows the success callback to add additional jobs to the parent batch
|
107
|
+
# before triggering :complete.
|
108
|
+
|
109
|
+
Batch.with_callback_check(parent_bid, except: [:success]) do |r|
|
110
|
+
r.sadd("BID-#{parent_bid}-batches-complete", bid)
|
111
|
+
r.sadd("BID-#{parent_bid}-batches-failed", bid)
|
102
112
|
end
|
103
113
|
end
|
114
|
+
end
|
104
115
|
|
105
|
-
|
106
|
-
|
107
|
-
r.multi do |r|
|
108
|
-
r.hincrby("BID-#{bid}", "pending", 0)
|
109
|
-
r.hincrby("BID-#{bid}", "children", 0)
|
110
|
-
r.scard("BID-#{bid}-batches-success")
|
111
|
-
end
|
112
|
-
end
|
116
|
+
def death(bid, status, parent_bid)
|
117
|
+
return unless parent_bid
|
113
118
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
# Otherwise, we don't want to to trigger the parent's :complete here (and
|
119
|
-
# instead opt to have success tigger parent :complete) - this
|
120
|
-
# allows the success callback to add additional jobs to the parent batch
|
121
|
-
# before triggering :complete.
|
122
|
-
|
123
|
-
Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
|
124
|
-
_, _, complete, pending, children, failure = Batch.redis do |r|
|
125
|
-
r.multi do |r|
|
126
|
-
r.sadd("BID-#{parent_bid}-batches-complete", bid)
|
127
|
-
r.sadd("BID-#{parent_bid}-batches-failed", bid)
|
128
|
-
r.scard("BID-#{parent_bid}-batches-complete")
|
129
|
-
r.hincrby("BID-#{parent_bid}", "pending", 0)
|
130
|
-
r.hincrby("BID-#{parent_bid}", "children", 0)
|
131
|
-
r.scard("BID-#{parent_bid}-failed")
|
132
|
-
end
|
133
|
-
end
|
134
|
-
if complete == children && pending == failure
|
135
|
-
Batch.enqueue_callbacks(:complete, parent_bid)
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
119
|
+
# We only need to bubble the event here - other events (eg stagnation) will be checked and bubbled elsewhere.
|
120
|
+
|
121
|
+
Batch.enqueue_callbacks(:death, parent_bid)
|
122
|
+
end
|
139
123
|
|
140
|
-
|
141
|
-
|
124
|
+
def stagnated(bid, status, parent_bid)
|
125
|
+
return unless parent_bid
|
142
126
|
|
143
|
-
|
127
|
+
Batch.with_callback_check(parent_bid) do |r|
|
128
|
+
r.sadd("BID-#{parent_bid}-batches-stagnated", bid)
|
129
|
+
r.expire("BID-#{parent_bid}-batches-stagnated", BID_EXPIRE_TTL)
|
144
130
|
end
|
145
131
|
end
|
146
132
|
end
|