ruby_reactor 0.8.6 → 0.8.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.release-please-manifest.json +1 -1
  3. data/.specify/feature.json +1 -1
  4. data/CHANGELOG.md +81 -0
  5. data/CLAUDE.md +1 -1
  6. data/README.md +42 -3
  7. data/lib/ruby_reactor/dsl/map_builder.rb +20 -1
  8. data/lib/ruby_reactor/error/recorded_failure.rb +18 -0
  9. data/lib/ruby_reactor/executor/compensation_manager.rb +27 -1
  10. data/lib/ruby_reactor/executor/result_handler.rb +16 -0
  11. data/lib/ruby_reactor/executor/retry_manager.rb +2 -2
  12. data/lib/ruby_reactor/executor/step_executor.rb +20 -0
  13. data/lib/ruby_reactor/executor.rb +62 -10
  14. data/lib/ruby_reactor/interrupt_claims.rb +81 -0
  15. data/lib/ruby_reactor/map/step_rollback.rb +121 -0
  16. data/lib/ruby_reactor/reactor.rb +270 -105
  17. data/lib/ruby_reactor/rspec/matchers.rb +64 -16
  18. data/lib/ruby_reactor/rspec/test_subject.rb +25 -24
  19. data/lib/ruby_reactor/step/compose_step.rb +10 -0
  20. data/lib/ruby_reactor/step/map_step.rb +12 -0
  21. data/lib/ruby_reactor/storage/adapter.rb +17 -0
  22. data/lib/ruby_reactor/storage/redis_adapter.rb +42 -0
  23. data/lib/ruby_reactor/version.rb +1 -1
  24. data/lib/ruby_reactor/web/api.rb +17 -0
  25. data/lib/ruby_reactor/web/public/assets/{index-C6ejszCQ.js → index-DxjQlW7Q.js} +8 -8
  26. data/lib/ruby_reactor/web/public/index.html +1 -1
  27. data/lib/ruby_reactor/worker.rb +73 -6
  28. data/specs/010-interrupt-in-composed-child/checklists/requirements.md +43 -0
  29. data/specs/010-interrupt-in-composed-child/contracts/api-surface.md +75 -0
  30. data/specs/010-interrupt-in-composed-child/data-model.md +40 -0
  31. data/specs/010-interrupt-in-composed-child/plan.md +149 -0
  32. data/specs/010-interrupt-in-composed-child/quickstart.md +64 -0
  33. data/specs/010-interrupt-in-composed-child/research.md +207 -0
  34. data/specs/010-interrupt-in-composed-child/spec.md +267 -0
  35. data/specs/010-interrupt-in-composed-child/tasks.md +315 -0
  36. data/specs/010-rollback-follow-ups/checklists/requirements.md +47 -0
  37. data/specs/010-rollback-follow-ups/contracts/api-surface.md +123 -0
  38. data/specs/010-rollback-follow-ups/contracts/resume-protocol.md +132 -0
  39. data/specs/010-rollback-follow-ups/data-model.md +139 -0
  40. data/specs/010-rollback-follow-ups/plan.md +273 -0
  41. data/specs/010-rollback-follow-ups/pr-description.md +110 -0
  42. data/specs/010-rollback-follow-ups/quickstart.md +157 -0
  43. data/specs/010-rollback-follow-ups/research.md +440 -0
  44. data/specs/010-rollback-follow-ups/spec.md +489 -0
  45. data/specs/010-rollback-follow-ups/tasks.md +1022 -0
  46. data/specs/future_improvements.md +17 -14
  47. metadata +22 -2
@@ -115,6 +115,127 @@ module RubyReactor
115
115
  missing.map { |index| rollback_entry(index, :context_unavailable, "context expired") }
116
116
  end
117
117
 
118
+ # 010 R-13 (P-6): one call to the map's `undo_all` block, with the
119
+ # completed elements' results (index order, read lazily, never all at
120
+ # once), instead of replaying each element. A fan-out map reads its
121
+ # result slots and dispatches no rollback jobs; an inline map reads its
122
+ # element contexts one at a time, and an element an interruption left
123
+ # `aborted` still replays its own completed steps (it has no result).
124
+ def bulk_rollback(map_id, step_name, block)
125
+ failures, count, results =
126
+ dispatched_map_id(step_name) ? fan_out_completed(map_id) : inline_completed(map_id, step_name)
127
+ failures << call_undo_all(block, results, step_name, count) if count.positive?
128
+ finish_rollback(step_name, count, failures.compact)
129
+ end
130
+
131
+ def fan_out_completed(map_id)
132
+ meta = rollback_storage.retrieve_map_metadata(map_id, context.reactor_class.name)
133
+ return [[rollback_entry(nil, :context_unavailable, "map records expired")], 0, []] unless meta
134
+
135
+ total = meta["count"].to_i
136
+ failures = []
137
+ count = 0
138
+ each_result_slot(map_id, total) do |index, raw|
139
+ next failures << rollback_entry(index, :context_unavailable, "context expired") if raw.nil?
140
+
141
+ count += 1 if completed_slot?(raw)
142
+ end
143
+ results = Enumerator.new do |yielder|
144
+ each_result_slot(map_id, total) do |_index, raw|
145
+ yielder << ContextSerializer.deserialize_value(raw) if raw && completed_slot?(raw)
146
+ end
147
+ end
148
+ [failures, count, results.lazy]
149
+ end
150
+
151
+ # ponytail: 1,000 slots per HMGET bounds memory; raise it if round trips dominate.
152
+ def each_result_slot(map_id, total, &block)
153
+ (0...total).each_slice(RubyReactor::Map::ResultEnumerator::DEFAULT_BATCH_SIZE) do |indexes|
154
+ slots = rollback_storage.retrieve_map_result_slots(map_id, context.reactor_class.name, indexes)
155
+ indexes.zip(slots).each(&block)
156
+ end
157
+ end
158
+
159
+ def completed_slot?(raw)
160
+ !(raw.is_a?(Hash) && (raw.key?("_error") || raw.key?("_halt") || raw.key?("_skipped")))
161
+ end
162
+
163
+ # Pass 1 replays aborted elements and counts completed ones; the lazy
164
+ # pass 2 is what the block reads.
165
+ def inline_completed(map_id, step_name)
166
+ total = rollback_storage.count_map_element_context_ids(map_id, context.reactor_class.name)
167
+ failures = []
168
+ seen = Set.new
169
+ unnamed = 0
170
+ count = 0
171
+ each_element(map_id, total) do |id, element|
172
+ next unnamed += 1 unless element
173
+
174
+ seen << Utils::FetchIndifferent.call(element.map_metadata || {}, :index).to_i
175
+ case element.status.to_s
176
+ when "completed" then count += 1
177
+ when "aborted"
178
+ outcome = inline_element_rollback(map_id, id, step_name)
179
+ failures.concat(ContextSerializer.deserialize_value(outcome["failures"]))
180
+ end
181
+ end
182
+ unseen = ->(indexes) { indexes.reject { |index| seen.include?(index) } }
183
+ results = Enumerator.new do |yielder|
184
+ each_element(map_id, total) do |_id, element|
185
+ yielder << element_result(element) if element && element.status.to_s == "completed"
186
+ end
187
+ end
188
+ [unavailable_entries(step_name, unnamed, unseen) + failures, count, results.lazy]
189
+ end
190
+
191
+ # Element contexts in index order (head of the index first), one loaded
192
+ # at a time, `Map::ROLLBACK_CHUNK` ids per read.
193
+ def each_element(map_id, total)
194
+ storage_name = RubyReactor.reactor_storage_name(element_class)
195
+ (0...total).each_slice(RubyReactor::Map::ROLLBACK_CHUNK) do |heads|
196
+ ids = rollback_storage.retrieve_map_element_context_ids_from_tail(
197
+ map_id, context.reactor_class.name, total - 1 - heads.last, heads.size, total: total
198
+ ).reverse
199
+ ids.each do |id|
200
+ data = rollback_storage.retrieve_context(id, storage_name)
201
+ yield id, data && Context.deserialize_from_retry(data)
202
+ end
203
+ end
204
+ end
205
+
206
+ # What the element returned, as the map collected it
207
+ # (`ResultHandler#final_result`): its `returns` step, else every result.
208
+ def element_result(element)
209
+ klass = element.reactor_class || element_class
210
+ klass.return_step ? element.get_result(klass.return_step) : element.intermediate_results
211
+ end
212
+
213
+ def call_undo_all(block, results, step_name, count)
214
+ log_rollback("undo_all.started", step_name, count: count)
215
+ outcome = begin
216
+ block.call(results)
217
+ rescue RubyReactor::Error::Rescuable => e
218
+ e
219
+ end
220
+ failure = undo_all_failure(step_name, outcome)
221
+ context.append_execution_trace({ type: :undo_all, step: step_name, count: count, timestamp: Time.now,
222
+ result: failure ? failure[:message] : :ok })
223
+ log_rollback("undo_all.completed", step_name, count: count, failed: failure ? 1 : 0)
224
+ failure
225
+ end
226
+
227
+ def undo_all_failure(step_name, outcome)
228
+ return unless outcome.is_a?(Exception) || outcome.is_a?(RubyReactor::Failure)
229
+
230
+ error = outcome.is_a?(RubyReactor::Failure) ? outcome.error : outcome
231
+ { step: step_name.to_sym, kind: :undo_all, reason: outcome.is_a?(Exception) ? :raised : :returned_failure,
232
+ message: error.respond_to?(:message) ? error.message : error.to_s }
233
+ end
234
+
235
+ def rollback_storage
236
+ RubyReactor.configuration.storage_adapter
237
+ end
238
+
118
239
  def finish_rollback(step_name, total, failures)
119
240
  log_rollback("completed", step_name, total: total, failed: failures.size)
120
241
  return RubyReactor.Success() if failures.empty?
@@ -9,6 +9,12 @@ module RubyReactor
9
9
  # Seconds a manual undo waits for the run's context lock (009 R-12).
10
10
  UNDO_LOCK_WAIT = 5
11
11
 
12
+ # Statuses a resume is accepted in: a run paused at an interrupt (008
13
+ # FR-032), or one executing (another resume, its first run, a wait on
14
+ # background work) for an interrupt that has not run yet (010 FR-017). A
15
+ # finished, `aborted` or rolling-back run is never run forward.
16
+ RESUMABLE_STATUSES = %w[paused running].freeze
17
+
12
18
  attr_reader :context, :result, :undo_trace, :execution_trace
13
19
 
14
20
  def self.find(id)
@@ -38,6 +44,14 @@ module RubyReactor
38
44
  find(context_id)
39
45
  end
40
46
 
47
+ # How a resume names an interrupt: a Symbol for one of the reactor's own,
48
+ # or its step path from the root (an Array: the compose step names, then
49
+ # the interrupt) for one inside a composed child (010).
50
+ def self.interrupt_key(step_name)
51
+ path = Array(step_name).map(&:to_sym)
52
+ path.one? ? path.first : path
53
+ end
54
+
41
55
  def self.continue(id:, payload:, step_name:, idempotency_key: nil)
42
56
  reactor = find(id)
43
57
  result = reactor.continue(payload: payload, step_name: step_name, idempotency_key: idempotency_key)
@@ -147,63 +161,36 @@ module RubyReactor
147
161
  @result
148
162
  end
149
163
 
164
+ # Accepting a resume (010 R-05, contracts/resume-protocol.md P-2–P-4):
165
+ #
166
+ # 1. checks on the loaded snapshot, then the payload is validated here, in
167
+ # the caller's process. An invalid payload stores and enqueues nothing;
168
+ # 2. the interrupt is claimed (`SET NX`): one resume per interrupt, in every
169
+ # execution mode. The claim holds the payload;
170
+ # 3. if the run's lock is free, this process owns the run: it reloads it and
171
+ # resumes inline, as before. Otherwise (another execution is live, the
172
+ # reactor's lock or semaphore is held, or a background-resume interrupt)
173
+ # the resume is handed to a Worker, which applies the claim under the
174
+ # run's lock, and a `DispatchResult` comes back.
175
+ #
176
+ # The payload is never validated again (J-7), so an accepted resume cannot
177
+ # fail validation later.
150
178
  def continue(payload:, step_name:, idempotency_key: nil)
151
179
  _ = idempotency_key
152
180
 
153
- unless @context.current_step
154
- raise Error::ValidationError, "Cannot resume: context does not have a current step (was it interrupted?)"
155
- end
156
-
157
- if @context.cancelled
158
- raise Error::ValidationError,
159
- "Cannot resume: reactor has been cancelled (Reason: #{@context.cancellation_reason})"
160
- end
161
-
162
- # Only a reactor paused at an interrupt takes a resume (008 FR-032): one
163
- # that is executing or rolling back (`running`), finished, or `aborted`
164
- # must not be run forward from its stored state.
165
- unless @context.status.to_s == "paused"
166
- raise Error::ValidationError,
167
- "Cannot resume: the reactor is #{@context.status}, not paused at an interrupt"
168
- end
169
-
170
- validate_continue_step!(step_name)
171
-
172
- if (failure = validate_continue_payload(payload, step_name))
173
- return failure
174
- end
175
-
176
- target_step = step_name
177
- @context.set_result(target_step, payload)
178
-
179
- # `interrupt :x, resume: :background` — payload is validated and stored
180
- # (above, in this process); the remaining work goes to a worker instead
181
- # of running inline in the delivering process.
182
- step_config = self.class.steps[step_name.to_sym]
183
- if step_config.respond_to?(:background_resume?) && step_config.background_resume?
184
- return @result = enqueue_background_resume
185
- end
181
+ path = Array(self.class.interrupt_key(step_name))
182
+ _, step_config = ensure_resumable!(path)
186
183
 
187
- # Claim the resume before running it, so a `continue` that arrives while
188
- # this one executes or rolls back reads `running` and fails (FR-032).
189
- @context.status = :running
190
- save_context
184
+ # A claimed resume is accepted: a later payload, valid or not, must not
185
+ # count against `max_attempts` and undo the run it is about to complete.
186
+ raise_already_resumed(path) if InterruptClaims.claimed?(@context, path)
191
187
 
192
- # Resume execution
193
- executor = Executor.new(self.class, {}, @context)
194
- @result = executor.resume_execution
188
+ failure = validate_continue_payload(payload, step_config, path)
189
+ return failure if failure
195
190
 
196
- @context = executor.context
197
- @undo_trace = executor.undo_trace
198
- @execution_trace = executor.execution_trace
191
+ raise_already_resumed(path) unless InterruptClaims.claim!(@context, path, payload)
199
192
 
200
- @result
201
- rescue Lock::AcquisitionError, Semaphore::AcquisitionError => e
202
- # Contended at the resume's own gates: nothing ran, so the run is still
203
- # paused and the caller may retry, as with `Reactor.run`. A context-lock
204
- # contention means a live resume holds the run: leave it alone.
205
- reopen_paused unless executor&.past_gates? || e.is_a?(Lock::ContextLockContention)
206
- raise
193
+ resume_claimed(path, step_config, payload)
207
194
  rescue Error::InputValidationError => e
208
195
  # This might catch other validations, but here we specifically want payload validation.
209
196
  # The block above handles payload validation explicitly.
@@ -214,12 +201,31 @@ module RubyReactor
214
201
  # resumes it meanwhile (009 R-12). Returns `:handed_off` when a fan-out
215
202
  # map's rollback continues in its element jobs: the run is then
216
203
  # `rolling_back`, and a worker finishes it as `cancelled`.
217
- def undo
204
+ #
205
+ # Lock, then load (010 R-09): the run is reloaded once the lock is held, so
206
+ # an undo built from an older snapshot never undoes stale state. With
207
+ # `failure:` the run ends `failed` with that reason (an interrupt's payload
208
+ # attempts ran out, R-08) instead of being left for `cancel`.
209
+ def undo(failure: nil)
218
210
  raise Error::ValidationError, "rollback already in progress" if @context.rolling_back?
219
211
 
220
212
  lock = acquire_undo_lock
213
+ @context = self.class.find(@context.context_id).context
214
+ raise Error::ValidationError, "rollback already in progress" if @context.rolling_back?
215
+
221
216
  executor = Executor.new(self.class, {}, @context)
217
+ # An aborted run may record a `compensate` the interruption cut off:
218
+ # keep it, so `undo_all` runs it first (010 R-10).
219
+ cut_off = if @context.status.to_s == "aborted" && @context.rollback.is_a?(Hash) &&
220
+ @context.rollback.key?("arguments")
221
+ @context.rollback
222
+ else
223
+ {}
224
+ end
222
225
  @context.rollback = { "trigger" => "undo", "compensated" => true, "failures" => [] }
226
+ .merge(cut_off.slice("step", "compensated", "arguments", "error"))
227
+ # A hand-off finishes in a worker (`Executor#finish_undo`), which applies it.
228
+ @context.rollback["failure_reason"] = failure if failure
223
229
  begin
224
230
  executor.undo_all
225
231
  rescue Error::RollbackHandedOff => e
@@ -229,6 +235,10 @@ module RubyReactor
229
235
  return :handed_off
230
236
  end
231
237
  @context.rollback = nil
238
+ if failure
239
+ @context.status = "failed"
240
+ @context.failure_reason = failure
241
+ end
232
242
  executor.save_context
233
243
  ensure
234
244
  lock&.release
@@ -244,6 +254,15 @@ module RubyReactor
244
254
  save_context
245
255
  end
246
256
 
257
+ # The interrupts this paused run can be resumed at (010 R-05): a Symbol
258
+ # for one of this reactor's own, and the step path from here (the compose
259
+ # step names, then the interrupt) for one inside a composed child.
260
+ def ready_interrupt_steps
261
+ return [] unless RESUMABLE_STATUSES.include?(@context.status.to_s)
262
+
263
+ pending_interrupts(self.class, @context, [])
264
+ end
265
+
247
266
  def validate!
248
267
  # Validate reactor configuration
249
268
  validate_steps!
@@ -257,15 +276,154 @@ module RubyReactor
257
276
  RubyReactor::Configuration.instance
258
277
  end
259
278
 
260
- def reopen_paused
261
- @context.status = :paused
279
+ # Returns the context that paused at the interrupt `path` and its config.
280
+ def ensure_resumable!(path)
281
+ if @context.status.to_s == "paused" && !@context.current_step
282
+ raise Error::ValidationError, "Cannot resume: context does not have a current step (was it interrupted?)"
283
+ end
284
+
285
+ # A composed child pauses only inside its root, which owns the run (010 R-07).
286
+ if @context.private_data[:composed] || @context.private_data["composed"]
287
+ raise Error::ValidationError, "Cannot resume: #{self.class.name} is a composed child; continue its root run"
288
+ end
289
+
290
+ if @context.cancelled
291
+ raise Error::ValidationError,
292
+ "Cannot resume: reactor has been cancelled (Reason: #{@context.cancellation_reason})"
293
+ end
294
+
295
+ unless RESUMABLE_STATUSES.include?(@context.status.to_s)
296
+ raise Error::ValidationError,
297
+ "Cannot resume: the reactor is #{@context.status}, not paused or running at an interrupt"
298
+ end
299
+
300
+ target_context, step_config = resolve_interrupt_target!(path)
301
+ raise_already_resumed(path) if target_context.has_result?(path.last)
302
+
303
+ [target_context, step_config]
304
+ end
305
+
306
+ def raise_already_resumed(path)
307
+ raise Error::ValidationError, "Cannot resume: interrupt :#{path.join(".")} was already resumed"
308
+ end
309
+
310
+ # Owns a paused run if its lock is free (lock, then load), else hands off.
311
+ # A running run is another execution's: the resume joins it through a
312
+ # Worker and never runs it here. The lock is released before any hand-off
313
+ # enqueues, so the Worker never waits on it; a crash in between leaves a
314
+ # `running` run the sweeper recovers.
315
+ def resume_claimed(path, step_config, payload)
316
+ lock = @context.status.to_s == "paused" ? try_run_lock : :contended
317
+ return hand_off_resume(path, reason: :run_busy) if lock == :contended
318
+
319
+ begin
320
+ reload_for_resume!
321
+ # This process owns the run: apply the caller's payload as given, as
322
+ # an inline resume always did (the claim's copy went through
323
+ # serialization). Other claims are applied from storage at resume.
324
+ if @context.status.to_s == "paused"
325
+ path[0...-1].reduce(@context) { |context, name| paused_child(context, name) }
326
+ &.set_result(path.last, payload)
327
+ end
328
+ if @context.status.to_s != "paused"
329
+ reason = :run_busy
330
+ elsif background_resume?(step_config)
331
+ prepare_background_resume
332
+ reason = :background
333
+ else
334
+ return resume_inline(lock)
335
+ end
336
+ rescue Lock::AcquisitionError, Semaphore::AcquisitionError => e
337
+ # The run's own lock is held here, so this is the reactor's `with_lock`
338
+ # or `with_semaphore`: the resume, already saved `running` with its
339
+ # payload applied, waits in a worker instead (FR-009).
340
+ raise if e.is_a?(Lock::ContextLockContention)
341
+
342
+ reason = e.is_a?(Semaphore::AcquisitionError) ? :semaphore : :lock
343
+ ensure
344
+ lock&.release
345
+ end
346
+ hand_off_resume(path, reason: reason, key: contended_key(reason))
347
+ end
348
+
349
+ # nil in inline job-testing mode (no run lock there, as in the executor);
350
+ # :contended while another execution holds it.
351
+ def try_run_lock
352
+ return nil if inline_testing_mode?
353
+
354
+ lock = RubyReactor::Lock.new("async:#{@context.context_id}", owner: SecureRandom.uuid,
355
+ ttl: configuration.context_lock_ttl,
356
+ wait: 0, auto_extend: true)
357
+ lock.acquire
358
+ lock
359
+ rescue Lock::AcquisitionError
360
+ :contended
361
+ end
362
+
363
+ def reload_for_resume!
364
+ @context = self.class.find(@context.context_id).context
365
+ return unless @context.cancelled || @context.finished? || @context.rolling_back? ||
366
+ @context.status.to_s == "aborted"
367
+
368
+ raise Error::ValidationError,
369
+ "Cannot resume: the reactor is #{@context.status}, not paused at an interrupt"
370
+ end
371
+
372
+ def resume_inline(lock)
373
+ executor = Executor.new(self.class, {}, @context)
374
+ executor.context_lock_owner = lock&.owner
375
+ @result = executor.resume_execution
376
+ @context = executor.context
377
+ @undo_trace = executor.undo_trace
378
+ @execution_trace = executor.execution_trace
379
+ @result
380
+ end
381
+
382
+ def background_resume?(step_config)
383
+ step_config.respond_to?(:background_resume?) && step_config.background_resume?
384
+ end
385
+
386
+ # `interrupt :x, resume: :background`: the payload is applied and the run
387
+ # saved `running` under its lock; the remaining work goes to a worker.
388
+ def prepare_background_resume
389
+ InterruptClaims.apply!(@context)
390
+ @context.status = :running
391
+ Executor.middlewares_for(self.class).on(:before_async_enqueue, @context)
262
392
  save_context
263
393
  end
264
394
 
395
+ def hand_off_resume(path, reason:, key: nil)
396
+ log_resume_deferred(path, reason, key)
397
+ @result = configuration.async_router.perform_async(@context.context_id,
398
+ RubyReactor.reactor_storage_name(self.class),
399
+ intermediate_results: @context.intermediate_results)
400
+ check_for_inline_completion || @result
401
+ end
402
+
403
+ def log_resume_deferred(path, reason, key)
404
+ fields = { event: "ruby_reactor.resume.deferred", reactor: RubyReactor.reactor_storage_name(self.class),
405
+ context_id: @context.context_id, step: path.join("."), reason: reason.to_s, key: key }
406
+ configuration.logger.info(fields.map { |k, v| "#{k}=#{v.inspect}" }.join(" "))
407
+ end
408
+
409
+ def contended_key(reason)
410
+ config = case reason
411
+ when :lock then self.class.respond_to?(:lock_config) && self.class.lock_config
412
+ when :semaphore then self.class.respond_to?(:semaphore_config) && self.class.semaphore_config
413
+ end
414
+ config ? config[:key_proc].call(@context.inputs).to_s : nil
415
+ rescue StandardError
416
+ nil
417
+ end
418
+
419
+ def inline_testing_mode?
420
+ defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
421
+ end
422
+
265
423
  # Raises `Lock::AcquisitionError` while a live run or rollback holds it.
266
424
  # Skipped in inline job-testing mode, as `Executor#acquire_context_lock`.
267
425
  def acquire_undo_lock
268
- return if defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
426
+ return if inline_testing_mode?
269
427
 
270
428
  lock = RubyReactor::Lock.new("async:#{@context.context_id}", owner: SecureRandom.uuid,
271
429
  ttl: configuration.context_lock_ttl,
@@ -379,21 +537,6 @@ module RubyReactor
379
537
  save_context
380
538
  end
381
539
 
382
- # Mirror of perform_async_run for the interrupt-resume path: persist the
383
- # context (now carrying the validated payload) BEFORE enqueue — the job
384
- # payload is identity-only (F2) — then hand the remainder to a worker.
385
- def enqueue_background_resume
386
- @context.status = :running
387
- Executor.middlewares_for(self.class).on(:before_async_enqueue, @context)
388
- save_context
389
-
390
- @result = configuration.async_router.perform_async(@context.context_id,
391
- RubyReactor.reactor_storage_name(self.class),
392
- intermediate_results: @context.intermediate_results)
393
-
394
- check_for_inline_completion || @result
395
- end
396
-
397
540
  def perform_async_run
398
541
  @context.status = :running
399
542
  # Persist BEFORE enqueue — the job payload is identity-only (F2).
@@ -434,58 +577,80 @@ module RubyReactor
434
577
  @undo_trace = executor.undo_trace
435
578
  end
436
579
 
437
- def validate_continue_step!(step_name)
438
- return if step_name.to_s == @context.current_step.to_s
580
+ # The context that paused at the interrupt `path` names, and that
581
+ # interrupt's config. A one-name path is a step of this reactor; a longer
582
+ # one names compose steps from here down, then the interrupt (010 R-05).
583
+ def resolve_interrupt_target!(path)
584
+ pending = ready_interrupt_steps
585
+ key = path.one? ? path.first : path
586
+ unless pending.include?(key)
587
+ raise Error::ValidationError,
588
+ "Cannot resume: expected step '#{@context.current_step}' " \
589
+ "or ready steps #{pending.inspect} but got '#{key}'"
590
+ end
591
+
592
+ target = path[0...-1].reduce(@context) { |context, name| paused_child(context, name) }
593
+ [target, target.reactor_class.steps[path.last]]
594
+ end
439
595
 
440
- # Build graph to check if step is ready
441
- graph_manager = Executor::GraphManager.new(self.class, DependencyGraph.new, @context)
596
+ def pending_interrupts(reactor_class, context, prefix)
597
+ graph_manager = Executor::GraphManager.new(reactor_class, DependencyGraph.new, context)
442
598
  graph_manager.build_and_validate!
443
599
  graph_manager.mark_completed_steps_from_context
444
- ready_steps = graph_manager.dependency_graph.ready_steps.map(&:name).map(&:to_s)
445
600
 
446
- return if ready_steps.include?(step_name.to_s)
601
+ # The interrupt a context paused at stays pending even once a resume
602
+ # stored its payload: that resume was contended and never ran.
603
+ ready = graph_manager.dependency_graph.ready_steps
604
+ paused_at = reactor_class.steps[context.current_step&.to_sym]
605
+ ready += [paused_at] if paused_at&.interrupt? && ready.none? { |step| step.name == paused_at.name }
606
+
607
+ ready.flat_map do |step_config|
608
+ name = step_config.name.to_sym
609
+ if step_config.respond_to?(:interrupt?) && step_config.interrupt?
610
+ [prefix.empty? ? name : prefix + [name]]
611
+ elsif (child = paused_child(context, name))
612
+ pending_interrupts(child.reactor_class, child, prefix + [name])
613
+ else
614
+ []
615
+ end
616
+ end
617
+ end
447
618
 
448
- raise Error::ValidationError,
449
- "Cannot resume: expected step '#{@context.current_step}' " \
450
- "or ready steps #{ready_steps} but got '#{step_name}'"
619
+ def paused_child(context, step_name)
620
+ entry = context.composed_contexts[step_name] || context.composed_contexts[step_name.to_s]
621
+ child = entry.is_a?(Hash) && (entry[:context] || entry["context"])
622
+ child if child.is_a?(Context) && child.status.to_s == "paused"
451
623
  end
452
624
 
453
- def validate_continue_payload(payload, step_name)
454
- step_config = self.class.steps[step_name.to_sym]
625
+ def validate_continue_payload(payload, step_config, path)
455
626
  return unless step_config&.validation_schema
456
627
 
628
+ # A nested interrupt counts apart from a root step of the same name.
629
+ step_name = path.one? ? path.first : path.join(".").to_sym
630
+
457
631
  validation = step_config.validation_schema.call(payload)
458
632
 
459
633
  return unless validation.failure?
460
634
 
461
- # Track attempts
462
- step_key = step_name.to_sym
463
- @context.private_data[:interrupt_attempts] ||= {}
464
- @context.private_data[:interrupt_attempts][step_key] ||= 0
465
- @context.private_data[:interrupt_attempts][step_key] += 1
466
-
467
- save_context # Persist the attempt count
468
-
469
- current_attempts = @context.private_data[:interrupt_attempts][step_key]
635
+ # Counted in its own record (010 R-08): writing the snapshot here could
636
+ # overwrite a resume that is executing the run right now.
637
+ current_attempts = configuration.storage_adapter.increment_interrupt_attempts(
638
+ @context.context_id, RubyReactor.reactor_storage_name(self.class), step_name
639
+ )
470
640
  max_attempts = step_config.max_attempts
471
641
 
472
642
  if max_attempts != :infinity && current_attempts >= max_attempts
473
- # Max attempts reached - Fail and Compensate
474
- undo
475
-
476
- # Instead of cancelling, we mark as failed so it shows up as failed in UI
477
- @context.status = "failed"
478
- @context.failure_reason = {
479
- message: "Validation failed after #{max_attempts} attempts",
480
- step_name: step_name,
481
- errors: validation.errors.to_h,
482
- payload: payload,
483
- step_arguments: payload,
484
- attempts: current_attempts,
485
- validation_errors: validation.errors.to_h
486
- }
487
-
488
- save_context
643
+ # Max attempts reached: undo, and mark the run failed (not cancelled)
644
+ # so it shows up as failed in the UI. Both under the run's lock (R-09).
645
+ undo(failure: {
646
+ message: "Validation failed after #{max_attempts} attempts",
647
+ step_name: step_name,
648
+ errors: validation.errors.to_h,
649
+ payload: payload,
650
+ step_arguments: payload,
651
+ attempts: current_attempts,
652
+ validation_errors: validation.errors.to_h
653
+ })
489
654
 
490
655
  return RubyReactor::Failure(
491
656
  "Validation failed after #{max_attempts} attempts",