ruby_reactor 0.8.6 → 0.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/CHANGELOG.md +81 -0
- data/CLAUDE.md +1 -1
- data/README.md +42 -3
- data/lib/ruby_reactor/dsl/map_builder.rb +20 -1
- data/lib/ruby_reactor/error/recorded_failure.rb +18 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +27 -1
- data/lib/ruby_reactor/executor/result_handler.rb +16 -0
- data/lib/ruby_reactor/executor/retry_manager.rb +2 -2
- data/lib/ruby_reactor/executor/step_executor.rb +20 -0
- data/lib/ruby_reactor/executor.rb +62 -10
- data/lib/ruby_reactor/interrupt_claims.rb +81 -0
- data/lib/ruby_reactor/map/step_rollback.rb +121 -0
- data/lib/ruby_reactor/reactor.rb +270 -105
- data/lib/ruby_reactor/rspec/matchers.rb +64 -16
- data/lib/ruby_reactor/rspec/test_subject.rb +25 -24
- data/lib/ruby_reactor/step/compose_step.rb +10 -0
- data/lib/ruby_reactor/step/map_step.rb +12 -0
- data/lib/ruby_reactor/storage/adapter.rb +17 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +42 -0
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +17 -0
- data/lib/ruby_reactor/web/public/assets/{index-C6ejszCQ.js → index-DxjQlW7Q.js} +8 -8
- data/lib/ruby_reactor/web/public/index.html +1 -1
- data/lib/ruby_reactor/worker.rb +73 -6
- data/specs/010-interrupt-in-composed-child/checklists/requirements.md +43 -0
- data/specs/010-interrupt-in-composed-child/contracts/api-surface.md +75 -0
- data/specs/010-interrupt-in-composed-child/data-model.md +40 -0
- data/specs/010-interrupt-in-composed-child/plan.md +149 -0
- data/specs/010-interrupt-in-composed-child/quickstart.md +64 -0
- data/specs/010-interrupt-in-composed-child/research.md +207 -0
- data/specs/010-interrupt-in-composed-child/spec.md +267 -0
- data/specs/010-interrupt-in-composed-child/tasks.md +315 -0
- data/specs/010-rollback-follow-ups/checklists/requirements.md +47 -0
- data/specs/010-rollback-follow-ups/contracts/api-surface.md +123 -0
- data/specs/010-rollback-follow-ups/contracts/resume-protocol.md +132 -0
- data/specs/010-rollback-follow-ups/data-model.md +139 -0
- data/specs/010-rollback-follow-ups/plan.md +273 -0
- data/specs/010-rollback-follow-ups/pr-description.md +110 -0
- data/specs/010-rollback-follow-ups/quickstart.md +157 -0
- data/specs/010-rollback-follow-ups/research.md +440 -0
- data/specs/010-rollback-follow-ups/spec.md +489 -0
- data/specs/010-rollback-follow-ups/tasks.md +1022 -0
- data/specs/future_improvements.md +17 -14
- metadata +22 -2
|
@@ -115,6 +115,127 @@ module RubyReactor
|
|
|
115
115
|
missing.map { |index| rollback_entry(index, :context_unavailable, "context expired") }
|
|
116
116
|
end
|
|
117
117
|
|
|
118
|
+
# 010 R-13 (P-6): one call to the map's `undo_all` block, with the
|
|
119
|
+
# completed elements' results (index order, read lazily, never all at
|
|
120
|
+
# once), instead of replaying each element. A fan-out map reads its
|
|
121
|
+
# result slots and dispatches no rollback jobs; an inline map reads its
|
|
122
|
+
# element contexts one at a time, and an element an interruption left
|
|
123
|
+
# `aborted` still replays its own completed steps (it has no result).
|
|
124
|
+
def bulk_rollback(map_id, step_name, block)
|
|
125
|
+
failures, count, results =
|
|
126
|
+
dispatched_map_id(step_name) ? fan_out_completed(map_id) : inline_completed(map_id, step_name)
|
|
127
|
+
failures << call_undo_all(block, results, step_name, count) if count.positive?
|
|
128
|
+
finish_rollback(step_name, count, failures.compact)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def fan_out_completed(map_id)
|
|
132
|
+
meta = rollback_storage.retrieve_map_metadata(map_id, context.reactor_class.name)
|
|
133
|
+
return [[rollback_entry(nil, :context_unavailable, "map records expired")], 0, []] unless meta
|
|
134
|
+
|
|
135
|
+
total = meta["count"].to_i
|
|
136
|
+
failures = []
|
|
137
|
+
count = 0
|
|
138
|
+
each_result_slot(map_id, total) do |index, raw|
|
|
139
|
+
next failures << rollback_entry(index, :context_unavailable, "context expired") if raw.nil?
|
|
140
|
+
|
|
141
|
+
count += 1 if completed_slot?(raw)
|
|
142
|
+
end
|
|
143
|
+
results = Enumerator.new do |yielder|
|
|
144
|
+
each_result_slot(map_id, total) do |_index, raw|
|
|
145
|
+
yielder << ContextSerializer.deserialize_value(raw) if raw && completed_slot?(raw)
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
[failures, count, results.lazy]
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# ponytail: 1,000 slots per HMGET bounds memory; raise it if round trips dominate.
|
|
152
|
+
def each_result_slot(map_id, total, &block)
|
|
153
|
+
(0...total).each_slice(RubyReactor::Map::ResultEnumerator::DEFAULT_BATCH_SIZE) do |indexes|
|
|
154
|
+
slots = rollback_storage.retrieve_map_result_slots(map_id, context.reactor_class.name, indexes)
|
|
155
|
+
indexes.zip(slots).each(&block)
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def completed_slot?(raw)
|
|
160
|
+
!(raw.is_a?(Hash) && (raw.key?("_error") || raw.key?("_halt") || raw.key?("_skipped")))
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Pass 1 replays aborted elements and counts completed ones; the lazy
|
|
164
|
+
# pass 2 is what the block reads.
|
|
165
|
+
def inline_completed(map_id, step_name)
|
|
166
|
+
total = rollback_storage.count_map_element_context_ids(map_id, context.reactor_class.name)
|
|
167
|
+
failures = []
|
|
168
|
+
seen = Set.new
|
|
169
|
+
unnamed = 0
|
|
170
|
+
count = 0
|
|
171
|
+
each_element(map_id, total) do |id, element|
|
|
172
|
+
next unnamed += 1 unless element
|
|
173
|
+
|
|
174
|
+
seen << Utils::FetchIndifferent.call(element.map_metadata || {}, :index).to_i
|
|
175
|
+
case element.status.to_s
|
|
176
|
+
when "completed" then count += 1
|
|
177
|
+
when "aborted"
|
|
178
|
+
outcome = inline_element_rollback(map_id, id, step_name)
|
|
179
|
+
failures.concat(ContextSerializer.deserialize_value(outcome["failures"]))
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
unseen = ->(indexes) { indexes.reject { |index| seen.include?(index) } }
|
|
183
|
+
results = Enumerator.new do |yielder|
|
|
184
|
+
each_element(map_id, total) do |_id, element|
|
|
185
|
+
yielder << element_result(element) if element && element.status.to_s == "completed"
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
[unavailable_entries(step_name, unnamed, unseen) + failures, count, results.lazy]
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# Element contexts in index order (head of the index first), one loaded
|
|
192
|
+
# at a time, `Map::ROLLBACK_CHUNK` ids per read.
|
|
193
|
+
def each_element(map_id, total)
|
|
194
|
+
storage_name = RubyReactor.reactor_storage_name(element_class)
|
|
195
|
+
(0...total).each_slice(RubyReactor::Map::ROLLBACK_CHUNK) do |heads|
|
|
196
|
+
ids = rollback_storage.retrieve_map_element_context_ids_from_tail(
|
|
197
|
+
map_id, context.reactor_class.name, total - 1 - heads.last, heads.size, total: total
|
|
198
|
+
).reverse
|
|
199
|
+
ids.each do |id|
|
|
200
|
+
data = rollback_storage.retrieve_context(id, storage_name)
|
|
201
|
+
yield id, data && Context.deserialize_from_retry(data)
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# What the element returned, as the map collected it
|
|
207
|
+
# (`ResultHandler#final_result`): its `returns` step, else every result.
|
|
208
|
+
def element_result(element)
|
|
209
|
+
klass = element.reactor_class || element_class
|
|
210
|
+
klass.return_step ? element.get_result(klass.return_step) : element.intermediate_results
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def call_undo_all(block, results, step_name, count)
|
|
214
|
+
log_rollback("undo_all.started", step_name, count: count)
|
|
215
|
+
outcome = begin
|
|
216
|
+
block.call(results)
|
|
217
|
+
rescue RubyReactor::Error::Rescuable => e
|
|
218
|
+
e
|
|
219
|
+
end
|
|
220
|
+
failure = undo_all_failure(step_name, outcome)
|
|
221
|
+
context.append_execution_trace({ type: :undo_all, step: step_name, count: count, timestamp: Time.now,
|
|
222
|
+
result: failure ? failure[:message] : :ok })
|
|
223
|
+
log_rollback("undo_all.completed", step_name, count: count, failed: failure ? 1 : 0)
|
|
224
|
+
failure
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def undo_all_failure(step_name, outcome)
|
|
228
|
+
return unless outcome.is_a?(Exception) || outcome.is_a?(RubyReactor::Failure)
|
|
229
|
+
|
|
230
|
+
error = outcome.is_a?(RubyReactor::Failure) ? outcome.error : outcome
|
|
231
|
+
{ step: step_name.to_sym, kind: :undo_all, reason: outcome.is_a?(Exception) ? :raised : :returned_failure,
|
|
232
|
+
message: error.respond_to?(:message) ? error.message : error.to_s }
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def rollback_storage
|
|
236
|
+
RubyReactor.configuration.storage_adapter
|
|
237
|
+
end
|
|
238
|
+
|
|
118
239
|
def finish_rollback(step_name, total, failures)
|
|
119
240
|
log_rollback("completed", step_name, total: total, failed: failures.size)
|
|
120
241
|
return RubyReactor.Success() if failures.empty?
|
data/lib/ruby_reactor/reactor.rb
CHANGED
|
@@ -9,6 +9,12 @@ module RubyReactor
|
|
|
9
9
|
# Seconds a manual undo waits for the run's context lock (009 R-12).
|
|
10
10
|
UNDO_LOCK_WAIT = 5
|
|
11
11
|
|
|
12
|
+
# Statuses a resume is accepted in: a run paused at an interrupt (008
|
|
13
|
+
# FR-032), or one executing (another resume, its first run, a wait on
|
|
14
|
+
# background work) for an interrupt that has not run yet (010 FR-017). A
|
|
15
|
+
# finished, `aborted` or rolling-back run is never run forward.
|
|
16
|
+
RESUMABLE_STATUSES = %w[paused running].freeze
|
|
17
|
+
|
|
12
18
|
attr_reader :context, :result, :undo_trace, :execution_trace
|
|
13
19
|
|
|
14
20
|
def self.find(id)
|
|
@@ -38,6 +44,14 @@ module RubyReactor
|
|
|
38
44
|
find(context_id)
|
|
39
45
|
end
|
|
40
46
|
|
|
47
|
+
# How a resume names an interrupt: a Symbol for one of the reactor's own,
|
|
48
|
+
# or its step path from the root (an Array: the compose step names, then
|
|
49
|
+
# the interrupt) for one inside a composed child (010).
|
|
50
|
+
def self.interrupt_key(step_name)
|
|
51
|
+
path = Array(step_name).map(&:to_sym)
|
|
52
|
+
path.one? ? path.first : path
|
|
53
|
+
end
|
|
54
|
+
|
|
41
55
|
def self.continue(id:, payload:, step_name:, idempotency_key: nil)
|
|
42
56
|
reactor = find(id)
|
|
43
57
|
result = reactor.continue(payload: payload, step_name: step_name, idempotency_key: idempotency_key)
|
|
@@ -147,63 +161,36 @@ module RubyReactor
|
|
|
147
161
|
@result
|
|
148
162
|
end
|
|
149
163
|
|
|
164
|
+
# Accepting a resume (010 R-05, contracts/resume-protocol.md P-2–P-4):
|
|
165
|
+
#
|
|
166
|
+
# 1. checks on the loaded snapshot, then the payload is validated here, in
|
|
167
|
+
# the caller's process. An invalid payload stores and enqueues nothing;
|
|
168
|
+
# 2. the interrupt is claimed (`SET NX`): one resume per interrupt, in every
|
|
169
|
+
# execution mode. The claim holds the payload;
|
|
170
|
+
# 3. if the run's lock is free, this process owns the run: it reloads it and
|
|
171
|
+
# resumes inline, as before. Otherwise (another execution is live, the
|
|
172
|
+
# reactor's lock or semaphore is held, or a background-resume interrupt)
|
|
173
|
+
# the resume is handed to a Worker, which applies the claim under the
|
|
174
|
+
# run's lock, and a `DispatchResult` comes back.
|
|
175
|
+
#
|
|
176
|
+
# The payload is never validated again (J-7), so an accepted resume cannot
|
|
177
|
+
# fail validation later.
|
|
150
178
|
def continue(payload:, step_name:, idempotency_key: nil)
|
|
151
179
|
_ = idempotency_key
|
|
152
180
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
if @context.cancelled
|
|
158
|
-
raise Error::ValidationError,
|
|
159
|
-
"Cannot resume: reactor has been cancelled (Reason: #{@context.cancellation_reason})"
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
# Only a reactor paused at an interrupt takes a resume (008 FR-032): one
|
|
163
|
-
# that is executing or rolling back (`running`), finished, or `aborted`
|
|
164
|
-
# must not be run forward from its stored state.
|
|
165
|
-
unless @context.status.to_s == "paused"
|
|
166
|
-
raise Error::ValidationError,
|
|
167
|
-
"Cannot resume: the reactor is #{@context.status}, not paused at an interrupt"
|
|
168
|
-
end
|
|
169
|
-
|
|
170
|
-
validate_continue_step!(step_name)
|
|
171
|
-
|
|
172
|
-
if (failure = validate_continue_payload(payload, step_name))
|
|
173
|
-
return failure
|
|
174
|
-
end
|
|
175
|
-
|
|
176
|
-
target_step = step_name
|
|
177
|
-
@context.set_result(target_step, payload)
|
|
178
|
-
|
|
179
|
-
# `interrupt :x, resume: :background` — payload is validated and stored
|
|
180
|
-
# (above, in this process); the remaining work goes to a worker instead
|
|
181
|
-
# of running inline in the delivering process.
|
|
182
|
-
step_config = self.class.steps[step_name.to_sym]
|
|
183
|
-
if step_config.respond_to?(:background_resume?) && step_config.background_resume?
|
|
184
|
-
return @result = enqueue_background_resume
|
|
185
|
-
end
|
|
181
|
+
path = Array(self.class.interrupt_key(step_name))
|
|
182
|
+
_, step_config = ensure_resumable!(path)
|
|
186
183
|
|
|
187
|
-
#
|
|
188
|
-
#
|
|
189
|
-
@context
|
|
190
|
-
save_context
|
|
184
|
+
# A claimed resume is accepted: a later payload, valid or not, must not
|
|
185
|
+
# count against `max_attempts` and undo the run it is about to complete.
|
|
186
|
+
raise_already_resumed(path) if InterruptClaims.claimed?(@context, path)
|
|
191
187
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
@result = executor.resume_execution
|
|
188
|
+
failure = validate_continue_payload(payload, step_config, path)
|
|
189
|
+
return failure if failure
|
|
195
190
|
|
|
196
|
-
@context
|
|
197
|
-
@undo_trace = executor.undo_trace
|
|
198
|
-
@execution_trace = executor.execution_trace
|
|
191
|
+
raise_already_resumed(path) unless InterruptClaims.claim!(@context, path, payload)
|
|
199
192
|
|
|
200
|
-
|
|
201
|
-
rescue Lock::AcquisitionError, Semaphore::AcquisitionError => e
|
|
202
|
-
# Contended at the resume's own gates: nothing ran, so the run is still
|
|
203
|
-
# paused and the caller may retry, as with `Reactor.run`. A context-lock
|
|
204
|
-
# contention means a live resume holds the run: leave it alone.
|
|
205
|
-
reopen_paused unless executor&.past_gates? || e.is_a?(Lock::ContextLockContention)
|
|
206
|
-
raise
|
|
193
|
+
resume_claimed(path, step_config, payload)
|
|
207
194
|
rescue Error::InputValidationError => e
|
|
208
195
|
# This might catch other validations, but here we specifically want payload validation.
|
|
209
196
|
# The block above handles payload validation explicitly.
|
|
@@ -214,12 +201,31 @@ module RubyReactor
|
|
|
214
201
|
# resumes it meanwhile (009 R-12). Returns `:handed_off` when a fan-out
|
|
215
202
|
# map's rollback continues in its element jobs: the run is then
|
|
216
203
|
# `rolling_back`, and a worker finishes it as `cancelled`.
|
|
217
|
-
|
|
204
|
+
#
|
|
205
|
+
# Lock, then load (010 R-09): the run is reloaded once the lock is held, so
|
|
206
|
+
# an undo built from an older snapshot never undoes stale state. With
|
|
207
|
+
# `failure:` the run ends `failed` with that reason (an interrupt's payload
|
|
208
|
+
# attempts ran out, R-08) instead of being left for `cancel`.
|
|
209
|
+
def undo(failure: nil)
|
|
218
210
|
raise Error::ValidationError, "rollback already in progress" if @context.rolling_back?
|
|
219
211
|
|
|
220
212
|
lock = acquire_undo_lock
|
|
213
|
+
@context = self.class.find(@context.context_id).context
|
|
214
|
+
raise Error::ValidationError, "rollback already in progress" if @context.rolling_back?
|
|
215
|
+
|
|
221
216
|
executor = Executor.new(self.class, {}, @context)
|
|
217
|
+
# An aborted run may record a `compensate` the interruption cut off:
|
|
218
|
+
# keep it, so `undo_all` runs it first (010 R-10).
|
|
219
|
+
cut_off = if @context.status.to_s == "aborted" && @context.rollback.is_a?(Hash) &&
|
|
220
|
+
@context.rollback.key?("arguments")
|
|
221
|
+
@context.rollback
|
|
222
|
+
else
|
|
223
|
+
{}
|
|
224
|
+
end
|
|
222
225
|
@context.rollback = { "trigger" => "undo", "compensated" => true, "failures" => [] }
|
|
226
|
+
.merge(cut_off.slice("step", "compensated", "arguments", "error"))
|
|
227
|
+
# A hand-off finishes in a worker (`Executor#finish_undo`), which applies it.
|
|
228
|
+
@context.rollback["failure_reason"] = failure if failure
|
|
223
229
|
begin
|
|
224
230
|
executor.undo_all
|
|
225
231
|
rescue Error::RollbackHandedOff => e
|
|
@@ -229,6 +235,10 @@ module RubyReactor
|
|
|
229
235
|
return :handed_off
|
|
230
236
|
end
|
|
231
237
|
@context.rollback = nil
|
|
238
|
+
if failure
|
|
239
|
+
@context.status = "failed"
|
|
240
|
+
@context.failure_reason = failure
|
|
241
|
+
end
|
|
232
242
|
executor.save_context
|
|
233
243
|
ensure
|
|
234
244
|
lock&.release
|
|
@@ -244,6 +254,15 @@ module RubyReactor
|
|
|
244
254
|
save_context
|
|
245
255
|
end
|
|
246
256
|
|
|
257
|
+
# The interrupts this paused run can be resumed at (010 R-05): a Symbol
|
|
258
|
+
# for one of this reactor's own, and the step path from here (the compose
|
|
259
|
+
# step names, then the interrupt) for one inside a composed child.
|
|
260
|
+
def ready_interrupt_steps
|
|
261
|
+
return [] unless RESUMABLE_STATUSES.include?(@context.status.to_s)
|
|
262
|
+
|
|
263
|
+
pending_interrupts(self.class, @context, [])
|
|
264
|
+
end
|
|
265
|
+
|
|
247
266
|
def validate!
|
|
248
267
|
# Validate reactor configuration
|
|
249
268
|
validate_steps!
|
|
@@ -257,15 +276,154 @@ module RubyReactor
|
|
|
257
276
|
RubyReactor::Configuration.instance
|
|
258
277
|
end
|
|
259
278
|
|
|
260
|
-
|
|
261
|
-
|
|
279
|
+
# Returns the context that paused at the interrupt `path` and its config.
|
|
280
|
+
def ensure_resumable!(path)
|
|
281
|
+
if @context.status.to_s == "paused" && !@context.current_step
|
|
282
|
+
raise Error::ValidationError, "Cannot resume: context does not have a current step (was it interrupted?)"
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# A composed child pauses only inside its root, which owns the run (010 R-07).
|
|
286
|
+
if @context.private_data[:composed] || @context.private_data["composed"]
|
|
287
|
+
raise Error::ValidationError, "Cannot resume: #{self.class.name} is a composed child; continue its root run"
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
if @context.cancelled
|
|
291
|
+
raise Error::ValidationError,
|
|
292
|
+
"Cannot resume: reactor has been cancelled (Reason: #{@context.cancellation_reason})"
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
unless RESUMABLE_STATUSES.include?(@context.status.to_s)
|
|
296
|
+
raise Error::ValidationError,
|
|
297
|
+
"Cannot resume: the reactor is #{@context.status}, not paused or running at an interrupt"
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
target_context, step_config = resolve_interrupt_target!(path)
|
|
301
|
+
raise_already_resumed(path) if target_context.has_result?(path.last)
|
|
302
|
+
|
|
303
|
+
[target_context, step_config]
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def raise_already_resumed(path)
|
|
307
|
+
raise Error::ValidationError, "Cannot resume: interrupt :#{path.join(".")} was already resumed"
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
# Owns a paused run if its lock is free (lock, then load), else hands off.
|
|
311
|
+
# A running run is another execution's: the resume joins it through a
|
|
312
|
+
# Worker and never runs it here. The lock is released before any hand-off
|
|
313
|
+
# enqueues, so the Worker never waits on it; a crash in between leaves a
|
|
314
|
+
# `running` run the sweeper recovers.
|
|
315
|
+
def resume_claimed(path, step_config, payload)
|
|
316
|
+
lock = @context.status.to_s == "paused" ? try_run_lock : :contended
|
|
317
|
+
return hand_off_resume(path, reason: :run_busy) if lock == :contended
|
|
318
|
+
|
|
319
|
+
begin
|
|
320
|
+
reload_for_resume!
|
|
321
|
+
# This process owns the run: apply the caller's payload as given, as
|
|
322
|
+
# an inline resume always did (the claim's copy went through
|
|
323
|
+
# serialization). Other claims are applied from storage at resume.
|
|
324
|
+
if @context.status.to_s == "paused"
|
|
325
|
+
path[0...-1].reduce(@context) { |context, name| paused_child(context, name) }
|
|
326
|
+
&.set_result(path.last, payload)
|
|
327
|
+
end
|
|
328
|
+
if @context.status.to_s != "paused"
|
|
329
|
+
reason = :run_busy
|
|
330
|
+
elsif background_resume?(step_config)
|
|
331
|
+
prepare_background_resume
|
|
332
|
+
reason = :background
|
|
333
|
+
else
|
|
334
|
+
return resume_inline(lock)
|
|
335
|
+
end
|
|
336
|
+
rescue Lock::AcquisitionError, Semaphore::AcquisitionError => e
|
|
337
|
+
# The run's own lock is held here, so this is the reactor's `with_lock`
|
|
338
|
+
# or `with_semaphore`: the resume, already saved `running` with its
|
|
339
|
+
# payload applied, waits in a worker instead (FR-009).
|
|
340
|
+
raise if e.is_a?(Lock::ContextLockContention)
|
|
341
|
+
|
|
342
|
+
reason = e.is_a?(Semaphore::AcquisitionError) ? :semaphore : :lock
|
|
343
|
+
ensure
|
|
344
|
+
lock&.release
|
|
345
|
+
end
|
|
346
|
+
hand_off_resume(path, reason: reason, key: contended_key(reason))
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
# nil in inline job-testing mode (no run lock there, as in the executor);
|
|
350
|
+
# :contended while another execution holds it.
|
|
351
|
+
def try_run_lock
|
|
352
|
+
return nil if inline_testing_mode?
|
|
353
|
+
|
|
354
|
+
lock = RubyReactor::Lock.new("async:#{@context.context_id}", owner: SecureRandom.uuid,
|
|
355
|
+
ttl: configuration.context_lock_ttl,
|
|
356
|
+
wait: 0, auto_extend: true)
|
|
357
|
+
lock.acquire
|
|
358
|
+
lock
|
|
359
|
+
rescue Lock::AcquisitionError
|
|
360
|
+
:contended
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
def reload_for_resume!
|
|
364
|
+
@context = self.class.find(@context.context_id).context
|
|
365
|
+
return unless @context.cancelled || @context.finished? || @context.rolling_back? ||
|
|
366
|
+
@context.status.to_s == "aborted"
|
|
367
|
+
|
|
368
|
+
raise Error::ValidationError,
|
|
369
|
+
"Cannot resume: the reactor is #{@context.status}, not paused at an interrupt"
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
def resume_inline(lock)
|
|
373
|
+
executor = Executor.new(self.class, {}, @context)
|
|
374
|
+
executor.context_lock_owner = lock&.owner
|
|
375
|
+
@result = executor.resume_execution
|
|
376
|
+
@context = executor.context
|
|
377
|
+
@undo_trace = executor.undo_trace
|
|
378
|
+
@execution_trace = executor.execution_trace
|
|
379
|
+
@result
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
def background_resume?(step_config)
|
|
383
|
+
step_config.respond_to?(:background_resume?) && step_config.background_resume?
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
# `interrupt :x, resume: :background`: the payload is applied and the run
|
|
387
|
+
# saved `running` under its lock; the remaining work goes to a worker.
|
|
388
|
+
def prepare_background_resume
|
|
389
|
+
InterruptClaims.apply!(@context)
|
|
390
|
+
@context.status = :running
|
|
391
|
+
Executor.middlewares_for(self.class).on(:before_async_enqueue, @context)
|
|
262
392
|
save_context
|
|
263
393
|
end
|
|
264
394
|
|
|
395
|
+
def hand_off_resume(path, reason:, key: nil)
|
|
396
|
+
log_resume_deferred(path, reason, key)
|
|
397
|
+
@result = configuration.async_router.perform_async(@context.context_id,
|
|
398
|
+
RubyReactor.reactor_storage_name(self.class),
|
|
399
|
+
intermediate_results: @context.intermediate_results)
|
|
400
|
+
check_for_inline_completion || @result
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
def log_resume_deferred(path, reason, key)
|
|
404
|
+
fields = { event: "ruby_reactor.resume.deferred", reactor: RubyReactor.reactor_storage_name(self.class),
|
|
405
|
+
context_id: @context.context_id, step: path.join("."), reason: reason.to_s, key: key }
|
|
406
|
+
configuration.logger.info(fields.map { |k, v| "#{k}=#{v.inspect}" }.join(" "))
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
def contended_key(reason)
|
|
410
|
+
config = case reason
|
|
411
|
+
when :lock then self.class.respond_to?(:lock_config) && self.class.lock_config
|
|
412
|
+
when :semaphore then self.class.respond_to?(:semaphore_config) && self.class.semaphore_config
|
|
413
|
+
end
|
|
414
|
+
config ? config[:key_proc].call(@context.inputs).to_s : nil
|
|
415
|
+
rescue StandardError
|
|
416
|
+
nil
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
def inline_testing_mode?
|
|
420
|
+
defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
|
|
421
|
+
end
|
|
422
|
+
|
|
265
423
|
# Raises `Lock::AcquisitionError` while a live run or rollback holds it.
|
|
266
424
|
# Skipped in inline job-testing mode, as `Executor#acquire_context_lock`.
|
|
267
425
|
def acquire_undo_lock
|
|
268
|
-
return if
|
|
426
|
+
return if inline_testing_mode?
|
|
269
427
|
|
|
270
428
|
lock = RubyReactor::Lock.new("async:#{@context.context_id}", owner: SecureRandom.uuid,
|
|
271
429
|
ttl: configuration.context_lock_ttl,
|
|
@@ -379,21 +537,6 @@ module RubyReactor
|
|
|
379
537
|
save_context
|
|
380
538
|
end
|
|
381
539
|
|
|
382
|
-
# Mirror of perform_async_run for the interrupt-resume path: persist the
|
|
383
|
-
# context (now carrying the validated payload) BEFORE enqueue — the job
|
|
384
|
-
# payload is identity-only (F2) — then hand the remainder to a worker.
|
|
385
|
-
def enqueue_background_resume
|
|
386
|
-
@context.status = :running
|
|
387
|
-
Executor.middlewares_for(self.class).on(:before_async_enqueue, @context)
|
|
388
|
-
save_context
|
|
389
|
-
|
|
390
|
-
@result = configuration.async_router.perform_async(@context.context_id,
|
|
391
|
-
RubyReactor.reactor_storage_name(self.class),
|
|
392
|
-
intermediate_results: @context.intermediate_results)
|
|
393
|
-
|
|
394
|
-
check_for_inline_completion || @result
|
|
395
|
-
end
|
|
396
|
-
|
|
397
540
|
def perform_async_run
|
|
398
541
|
@context.status = :running
|
|
399
542
|
# Persist BEFORE enqueue — the job payload is identity-only (F2).
|
|
@@ -434,58 +577,80 @@ module RubyReactor
|
|
|
434
577
|
@undo_trace = executor.undo_trace
|
|
435
578
|
end
|
|
436
579
|
|
|
437
|
-
|
|
438
|
-
|
|
580
|
+
# The context that paused at the interrupt `path` names, and that
|
|
581
|
+
# interrupt's config. A one-name path is a step of this reactor; a longer
|
|
582
|
+
# one names compose steps from here down, then the interrupt (010 R-05).
|
|
583
|
+
def resolve_interrupt_target!(path)
|
|
584
|
+
pending = ready_interrupt_steps
|
|
585
|
+
key = path.one? ? path.first : path
|
|
586
|
+
unless pending.include?(key)
|
|
587
|
+
raise Error::ValidationError,
|
|
588
|
+
"Cannot resume: expected step '#{@context.current_step}' " \
|
|
589
|
+
"or ready steps #{pending.inspect} but got '#{key}'"
|
|
590
|
+
end
|
|
591
|
+
|
|
592
|
+
target = path[0...-1].reduce(@context) { |context, name| paused_child(context, name) }
|
|
593
|
+
[target, target.reactor_class.steps[path.last]]
|
|
594
|
+
end
|
|
439
595
|
|
|
440
|
-
|
|
441
|
-
graph_manager = Executor::GraphManager.new(
|
|
596
|
+
def pending_interrupts(reactor_class, context, prefix)
|
|
597
|
+
graph_manager = Executor::GraphManager.new(reactor_class, DependencyGraph.new, context)
|
|
442
598
|
graph_manager.build_and_validate!
|
|
443
599
|
graph_manager.mark_completed_steps_from_context
|
|
444
|
-
ready_steps = graph_manager.dependency_graph.ready_steps.map(&:name).map(&:to_s)
|
|
445
600
|
|
|
446
|
-
|
|
601
|
+
# The interrupt a context paused at stays pending even once a resume
|
|
602
|
+
# stored its payload: that resume was contended and never ran.
|
|
603
|
+
ready = graph_manager.dependency_graph.ready_steps
|
|
604
|
+
paused_at = reactor_class.steps[context.current_step&.to_sym]
|
|
605
|
+
ready += [paused_at] if paused_at&.interrupt? && ready.none? { |step| step.name == paused_at.name }
|
|
606
|
+
|
|
607
|
+
ready.flat_map do |step_config|
|
|
608
|
+
name = step_config.name.to_sym
|
|
609
|
+
if step_config.respond_to?(:interrupt?) && step_config.interrupt?
|
|
610
|
+
[prefix.empty? ? name : prefix + [name]]
|
|
611
|
+
elsif (child = paused_child(context, name))
|
|
612
|
+
pending_interrupts(child.reactor_class, child, prefix + [name])
|
|
613
|
+
else
|
|
614
|
+
[]
|
|
615
|
+
end
|
|
616
|
+
end
|
|
617
|
+
end
|
|
447
618
|
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
619
|
+
def paused_child(context, step_name)
|
|
620
|
+
entry = context.composed_contexts[step_name] || context.composed_contexts[step_name.to_s]
|
|
621
|
+
child = entry.is_a?(Hash) && (entry[:context] || entry["context"])
|
|
622
|
+
child if child.is_a?(Context) && child.status.to_s == "paused"
|
|
451
623
|
end
|
|
452
624
|
|
|
453
|
-
def validate_continue_payload(payload,
|
|
454
|
-
step_config = self.class.steps[step_name.to_sym]
|
|
625
|
+
def validate_continue_payload(payload, step_config, path)
|
|
455
626
|
return unless step_config&.validation_schema
|
|
456
627
|
|
|
628
|
+
# A nested interrupt counts apart from a root step of the same name.
|
|
629
|
+
step_name = path.one? ? path.first : path.join(".").to_sym
|
|
630
|
+
|
|
457
631
|
validation = step_config.validation_schema.call(payload)
|
|
458
632
|
|
|
459
633
|
return unless validation.failure?
|
|
460
634
|
|
|
461
|
-
#
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
save_context # Persist the attempt count
|
|
468
|
-
|
|
469
|
-
current_attempts = @context.private_data[:interrupt_attempts][step_key]
|
|
635
|
+
# Counted in its own record (010 R-08): writing the snapshot here could
|
|
636
|
+
# overwrite a resume that is executing the run right now.
|
|
637
|
+
current_attempts = configuration.storage_adapter.increment_interrupt_attempts(
|
|
638
|
+
@context.context_id, RubyReactor.reactor_storage_name(self.class), step_name
|
|
639
|
+
)
|
|
470
640
|
max_attempts = step_config.max_attempts
|
|
471
641
|
|
|
472
642
|
if max_attempts != :infinity && current_attempts >= max_attempts
|
|
473
|
-
# Max attempts reached
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
attempts: current_attempts,
|
|
485
|
-
validation_errors: validation.errors.to_h
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
save_context
|
|
643
|
+
# Max attempts reached: undo, and mark the run failed (not cancelled)
|
|
644
|
+
# so it shows up as failed in the UI. Both under the run's lock (R-09).
|
|
645
|
+
undo(failure: {
|
|
646
|
+
message: "Validation failed after #{max_attempts} attempts",
|
|
647
|
+
step_name: step_name,
|
|
648
|
+
errors: validation.errors.to_h,
|
|
649
|
+
payload: payload,
|
|
650
|
+
step_arguments: payload,
|
|
651
|
+
attempts: current_attempts,
|
|
652
|
+
validation_errors: validation.errors.to_h
|
|
653
|
+
})
|
|
489
654
|
|
|
490
655
|
return RubyReactor::Failure(
|
|
491
656
|
"Validation failed after #{max_attempts} attempts",
|