ruby_reactor 0.8.5 → 0.8.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.release-please-manifest.json +1 -1
  3. data/.specify/feature.json +1 -1
  4. data/.specify/memory/constitution.md +17 -14
  5. data/CHANGELOG.md +58 -0
  6. data/CLAUDE.md +1 -1
  7. data/README.md +23 -8
  8. data/lib/ruby_reactor/adapters/active_job/map_element_rollback_worker.rb +20 -0
  9. data/lib/ruby_reactor/adapters/active_job/router.rb +32 -4
  10. data/lib/ruby_reactor/adapters/sidekiq/map_element_rollback_worker.rb +16 -0
  11. data/lib/ruby_reactor/adapters/sidekiq/router.rb +32 -4
  12. data/lib/ruby_reactor/context.rb +14 -1
  13. data/lib/ruby_reactor/dsl/definition_warnings.rb +30 -0
  14. data/lib/ruby_reactor/dsl/map_builder.rb +27 -7
  15. data/lib/ruby_reactor/dsl/step_builder.rb +8 -21
  16. data/lib/ruby_reactor/error/rescuable.rb +6 -1
  17. data/lib/ruby_reactor/error/rollback_handed_off.rb +27 -0
  18. data/lib/ruby_reactor/executor/compensation_manager.rb +33 -9
  19. data/lib/ruby_reactor/executor/result_handler.rb +72 -52
  20. data/lib/ruby_reactor/executor/retry_manager.rb +4 -3
  21. data/lib/ruby_reactor/executor/step_executor.rb +3 -1
  22. data/lib/ruby_reactor/executor.rb +186 -6
  23. data/lib/ruby_reactor/map/collector.rb +25 -93
  24. data/lib/ruby_reactor/map/dispatcher.rb +38 -7
  25. data/lib/ruby_reactor/map/element_executor.rb +14 -10
  26. data/lib/ruby_reactor/map/element_rollback.rb +158 -0
  27. data/lib/ruby_reactor/map/helpers.rb +11 -155
  28. data/lib/ruby_reactor/map/step_rollback.rb +136 -0
  29. data/lib/ruby_reactor/map/sweeper.rb +66 -8
  30. data/lib/ruby_reactor/map.rb +17 -0
  31. data/lib/ruby_reactor/max_retries_exhausted_failure.rb +4 -2
  32. data/lib/ruby_reactor/reactor.rb +41 -2
  33. data/lib/ruby_reactor/rspec/active_job_helpers.rb +13 -2
  34. data/lib/ruby_reactor/rspec/matchers.rb +40 -21
  35. data/lib/ruby_reactor/rspec/sidekiq_helpers.rb +7 -3
  36. data/lib/ruby_reactor/rspec/test_subject.rb +13 -3
  37. data/lib/ruby_reactor/rspec.rb +3 -6
  38. data/lib/ruby_reactor/step/compose_step.rb +3 -0
  39. data/lib/ruby_reactor/step/map_step.rb +117 -114
  40. data/lib/ruby_reactor/storage/adapter.rb +14 -0
  41. data/lib/ruby_reactor/storage/redis_adapter.rb +18 -2
  42. data/lib/ruby_reactor/storage/redis_map_rollback.rb +153 -0
  43. data/lib/ruby_reactor/storage/redis_reactor_scan.rb +1 -1
  44. data/lib/ruby_reactor/sweeper.rb +3 -1
  45. data/lib/ruby_reactor/version.rb +1 -1
  46. data/lib/ruby_reactor/web/api.rb +11 -1
  47. data/lib/ruby_reactor/web/public/assets/{index-CQbgHtd0.js → index-C6ejszCQ.js} +12 -12
  48. data/lib/ruby_reactor/web/public/assets/index-DM5darhy.css +1 -0
  49. data/lib/ruby_reactor/web/public/index.html +2 -2
  50. data/lib/ruby_reactor/worker.rb +4 -2
  51. data/specs/007-execution-flow-analysis/evidence/harness.rb +2 -0
  52. data/specs/007-execution-flow-analysis/evidence/probes/03_map.rb +4 -1
  53. data/specs/009-distributed-map-undo/checklists/requirements.md +53 -0
  54. data/specs/009-distributed-map-undo/contracts/api-surface.md +125 -0
  55. data/specs/009-distributed-map-undo/contracts/rollback-protocol.md +157 -0
  56. data/specs/009-distributed-map-undo/data-model.md +151 -0
  57. data/specs/009-distributed-map-undo/plan.md +323 -0
  58. data/specs/009-distributed-map-undo/pr-description.md +78 -0
  59. data/specs/009-distributed-map-undo/quickstart.md +154 -0
  60. data/specs/009-distributed-map-undo/research.md +585 -0
  61. data/specs/009-distributed-map-undo/spec.md +360 -0
  62. data/specs/009-distributed-map-undo/tasks.md +1060 -0
  63. data/specs/future_improvements.md +18 -30
  64. metadata +21 -3
  65. data/lib/ruby_reactor/web/public/assets/index-BQvIWPdx.css +0 -1
@@ -12,11 +12,16 @@ module RubyReactor
12
12
  # raises — rescuing that would stop the caller's timeout from ever firing.
13
13
  # Running rollback code during any of them is unsafe; the executor marks
14
14
  # the run `aborted` for a manual undo instead.
15
+ #
16
+ # Nor does it match `RollbackHandedOff`, a control signal (009 R-04): it
17
+ # must pass the `rescue Error::Rescuable` sites between `MapStep#undo` and
18
+ # the executor (`CompensationManager#undo_step`, `StepExecutor`), which
19
+ # would otherwise record it as an undo failure.
15
20
  module Rescuable
16
21
  INTERRUPTIONS = [SignalException, SystemExit, NoMemoryError].freeze
17
22
 
18
23
  def self.===(exception)
19
- exception.is_a?(Exception) && !interruption?(exception)
24
+ exception.is_a?(Exception) && !interruption?(exception) && !exception.is_a?(RollbackHandedOff)
20
25
  end
21
26
 
22
27
  def self.interruption?(exception)
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyReactor
4
+ module Error
5
+ # A control signal, never a failure (009 R-04): a fan-out map started its
6
+ # distributed rollback and must wait for its element jobs. Raised from
7
+ # `MapStep#compensate` / `#undo`, it unwinds the rollback without popping
8
+ # the map's undo entry (the resume cursor), passes every `rescue
9
+ # Error::Rescuable` (see `Rescuable.===`), and is rescued only by
10
+ # `Executor` and `Reactor#undo`, which record where the rollback stands,
11
+ # save, and hand the run off as `rolling_back`.
12
+ #
13
+ # `failure` carries the Failure the run will end with, minus its rollback
14
+ # failures, set by the `ResultHandler` it passes through and consumed by
15
+ # the executor of that level.
16
+ class RollbackHandedOff < Base
17
+ attr_reader :map_id, :reactor_class_name
18
+ attr_accessor :failure
19
+
20
+ def initialize(map_id:, reactor_class_name: nil, message: "rollback handed off at map #{map_id}")
21
+ super(message)
22
+ @map_id = map_id
23
+ @reactor_class_name = reactor_class_name
24
+ end
25
+ end
26
+ end
27
+ end
@@ -30,11 +30,23 @@ module RubyReactor
30
30
  # (005 FR-004). `ResultHandler` attaches it to the final Failure.
31
31
  attr_reader :undo_trace, :rollback_failures
32
32
 
33
+ # The step failure being rolled back: `{ step:, error:, compensated:,
34
+ # compensation_error: }`. A rollback that hands off (009 R-04) records it,
35
+ # so the resume knows whether that step's own compensate still runs.
36
+ attr_reader :pending
37
+
38
+ # Rollback failures collected before a hand-off, saved serialized on
39
+ # the context's `rollback` and restored on resume.
40
+ def restore_rollback_failures(list)
41
+ @rollback_failures.replace(Array(ContextSerializer.deserialize_value(list)))
42
+ end
43
+
33
44
  def add_to_undo_stack(step_info)
34
45
  @context.undo_stack << step_info
35
46
  end
36
47
 
37
48
  def handle_step_failure(step_config, error, arguments)
49
+ @pending = { step: step_config.name, error: error, compensated: false }
38
50
  # A step whose OWN coordination acquisition failed (contention, or a
39
51
  # bad key proc) never ran its body — "no step compensates, the
40
52
  # contended step's work has not been attempted" (US3-1/T018), which
@@ -44,29 +56,38 @@ module RubyReactor
44
56
  # turning a plain contention failure into a confusing
45
57
  # CompensationError. Prior steps still roll back normally.
46
58
  if step_never_started?(error)
59
+ @pending[:compensated] = true
47
60
  rollback_completed_steps
48
- return RubyReactor.Failure("Step '#{step_config.name}' failed: #{error}")
61
+ return RubyReactor.Failure(step_failure_message(step_config, error))
49
62
  end
50
63
 
51
64
  # Try compensation
52
65
  compensation_result = compensate_step(step_config, error, arguments)
66
+ @pending[:compensated] = true
67
+ @pending[:compensation_error] = compensation_error_message(step_config, compensation_result)
53
68
  case compensation_result
54
69
  when RubyReactor::Success
55
70
  # Compensation succeeded, continue with rollback
56
71
  rollback_completed_steps
57
- RubyReactor.Failure("Step '#{step_config.name}' failed: #{error}")
72
+ RubyReactor.Failure(step_failure_message(step_config, error))
58
73
  when RubyReactor::Failure
59
74
  # Compensation failed, this is more serious
60
75
  rollback_completed_steps
61
- raise Error::CompensationError.new(
62
- "Compensation for step '#{step_config.name}' failed: #{compensation_result.error}",
63
- step: step_config.name,
64
- context: @context,
65
- original_error: error
66
- )
76
+ raise Error::CompensationError.new(@pending[:compensation_error], step: step_config.name, context: @context,
77
+ original_error: error)
67
78
  end
68
79
  end
69
80
 
81
+ def step_failure_message(step_config, error)
82
+ "Step '#{step_config.name}' failed: #{error}"
83
+ end
84
+
85
+ def compensation_error_message(step_config, compensation_result)
86
+ return unless compensation_result.is_a?(RubyReactor::Failure)
87
+
88
+ "Compensation for step '#{step_config.name}' failed: #{compensation_result.error}"
89
+ end
90
+
70
91
  # A unit that compensates itself outside the executor loop (StepWorker's
71
92
  # `async_step`): the same coordination re-take, trace, middleware events
72
93
  # and `rollback_failures` as a step compensated here.
@@ -77,7 +98,9 @@ module RubyReactor
77
98
  # Newest first. Each entry leaves the stack only once its undo returned,
78
99
  # so an interruption mid-rollback leaves exactly the entries still to
79
100
  # undo — the interrupted one included — for a manual undo (008 R-16).
80
- def rollback_completed_steps
101
+ # `after_pop` runs after each pop: an element rollback job saves there,
102
+ # so a redelivery resumes after the last undone entry (009 R-07).
103
+ def rollback_completed_steps(&after_pop)
81
104
  until undo_stack.empty?
82
105
  step_info = undo_stack.last
83
106
  result = @context.with_step(step_info[:step].name) do
@@ -86,6 +109,7 @@ module RubyReactor
86
109
  undo_stack.pop
87
110
  @undo_trace << { type: :undo, step: step_info[:step].name, result: result,
88
111
  arguments: step_info[:arguments] }
112
+ after_pop&.call
89
113
  end
90
114
  end
91
115
 
@@ -35,9 +35,13 @@ module RubyReactor
35
35
  # Every reactor-level Failure that follows a rollback is built here, so
36
36
  # this is the one place `rollback_failures` is attached (005 R-08).
37
37
  def handle_execution_error(error)
38
- failure = build_execution_failure(error)
39
- failure.rollback_failures.concat(@compensation_manager.rollback_failures) if failure.is_a?(RubyReactor::Failure)
40
- failure
38
+ with_rollback_failures(build_execution_failure(error))
39
+ end
40
+
41
+ # The end of a rollback that handed off (009 R-04): the Failure recorded
42
+ # at the hand-off, with every rollback failure collected since.
43
+ def handed_off_failure(failure)
44
+ with_rollback_failures(failure)
41
45
  end
42
46
 
43
47
  def final_result(reactor_class)
@@ -51,34 +55,62 @@ module RubyReactor
51
55
 
52
56
  private
53
57
 
58
+ def with_rollback_failures(failure)
59
+ failure.rollback_failures.concat(@compensation_manager.rollback_failures) if failure.is_a?(RubyReactor::Failure)
60
+ failure
61
+ end
62
+
63
+ # Every error rolls back the completed steps first (Constitution II), then
64
+ # becomes the run's Failure.
54
65
  def build_execution_failure(error)
66
+ if error.is_a?(Error::StepFailureError)
67
+ current_context = error.context || @context
68
+ current_context.current_step = error.step
69
+ store_failed_map_context(current_context) if current_context.map_metadata
70
+ end
71
+ handing_off_as(error) { @compensation_manager.rollback_completed_steps }
72
+ failure_for(error)
73
+ end
74
+
75
+ # A rollback that hands off mid-way (009 R-04) carries the Failure this
76
+ # run will end with, minus its rollback failures, to the executor that
77
+ # records the hand-off. `error` is what the run fails with once the
78
+ # rollback is done (a lambda when it is costly or not built yet).
79
+ def handing_off_as(error)
80
+ yield
81
+ rescue Error::RollbackHandedOff => e
82
+ e.failure ||= failure_for(error.respond_to?(:call) ? error.call : error)
83
+ raise
84
+ end
85
+
86
+ def failure_for(error)
55
87
  case error
56
88
  when Error::StepFailureError
57
- handle_step_failure_error(error)
89
+ create_failure_from_error(error, redacted_input_names(error.context))
58
90
  when Error::InputValidationError
59
91
  # Unified validation failure shape (inputs, step args, step output).
60
- # Roll back any completed steps so saga semantics hold for mid-reactor
61
- # validation failures (a no-op for input validation at reactor start).
62
- @compensation_manager.rollback_completed_steps
63
92
  build_validation_failure(error)
64
93
  when Error::Base
65
- # Other errors need rollback. A `CompensationError` names the step
66
- # whose compensation failed.
67
- @compensation_manager.rollback_completed_steps
94
+ # A `CompensationError` names the step whose compensation failed.
68
95
  RubyReactor.Failure("Execution error: #{error.message}", exception_class: error.class.name,
69
96
  step_name: error.step || @context.current_step,
70
97
  reactor_name: @context.reactor_class&.name)
71
98
  else
72
99
  # Any other StandardError after completed work (a checkpoint write,
73
- # a hook) still leaves a partial saga: roll it back like any failure
74
- # (Constitution II, 008 R-07) and name the step that was executing.
75
- @compensation_manager.rollback_completed_steps
100
+ # a hook) still leaves a partial saga, rolled back like any failure
101
+ # (Constitution II, 008 R-07); name the step that was executing.
76
102
  RubyReactor.Failure("Execution failed: #{error.message}", exception_class: error.class.name,
77
103
  step_name: @context.current_step,
78
104
  reactor_name: @context.reactor_class&.name)
79
105
  end
80
106
  end
81
107
 
108
+ def redacted_input_names(context)
109
+ return [] unless context&.reactor_class
110
+
111
+ context.reactor_class.inputs.select { |_, config| config[:redact] }.keys
112
+ end
113
+
82
114
  # Failure for a validation error (reactor inputs, step arguments, or
83
115
  # step output), carrying both the structured field errors and the step/
84
116
  # reactor attribution stamped at the raise site (nil step_name for
@@ -139,7 +171,8 @@ module RubyReactor
139
171
  validate_step_output(step_config, result.value, resolved_arguments)
140
172
  @step_results[step_config.name] = result
141
173
  if step_config.rollback_tracked?
142
- @compensation_manager.add_to_undo_stack({ step: step_config, arguments: resolved_arguments,
174
+ @compensation_manager.add_to_undo_stack({ step: step_config,
175
+ arguments: step_config.rollback_arguments(resolved_arguments),
143
176
  result: result })
144
177
  end
145
178
  @context.set_result(step_config.name, result.value)
@@ -156,36 +189,38 @@ module RubyReactor
156
189
 
157
190
  def handle_retries_exhausted(step_config, result, resolved_arguments)
158
191
  adopt_rollback_failures(result)
159
- @compensation_manager.handle_step_failure(step_config, result.original_error, resolved_arguments)
160
- orig_err = result.original_error.is_a?(Exception) ? result.original_error : nil
161
- error = Error::StepFailureError.new(result.error, step: step_config.name, context: @context,
162
- original_error: orig_err,
163
- step_arguments: resolved_arguments,
164
- validation_errors: result.validation_errors)
165
- if result.respond_to?(:backtrace) && result.backtrace
166
- error.set_backtrace(result.backtrace)
167
- elsif orig_err
168
- error.set_backtrace(orig_err.backtrace)
192
+ error = step_failure_error(step_config, result.error, result, resolved_arguments, cause: result.original_error)
193
+ handing_off_as(error) do
194
+ @compensation_manager.handle_step_failure(step_config, result.original_error, resolved_arguments)
169
195
  end
170
196
  raise error
171
197
  end
172
198
 
173
199
  def handle_failure(step_config, result, resolved_arguments)
174
200
  adopt_rollback_failures(result)
175
- failure_result = @compensation_manager.handle_step_failure(step_config, result.error, resolved_arguments)
176
- orig_err = result.error.is_a?(Exception) ? result.error : nil
177
- # A step that propagates another unit's validation failure (an
178
- # async_step reader) keeps its field errors on the reactor's failure.
179
- error = Error::StepFailureError.new(failure_result.error, step: step_config.name, context: @context,
180
- original_error: orig_err,
181
- step_arguments: resolved_arguments,
182
- validation_errors: result.validation_errors)
201
+ message = @compensation_manager.step_failure_message(step_config, result.error)
202
+ failure_result = handing_off_as(-> { step_failure_error(step_config, message, result, resolved_arguments) }) do
203
+ @compensation_manager.handle_step_failure(step_config, result.error, resolved_arguments)
204
+ end
205
+ raise step_failure_error(step_config, failure_result.error, result, resolved_arguments)
206
+ end
207
+
208
+ # A step that propagates another unit's failure (an async_step reader, a
209
+ # map adopting a fan-out element's Failure) keeps its field errors and
210
+ # exception class on the reactor's failure.
211
+ def step_failure_error(step_config, message, result, resolved_arguments, cause: result.error)
212
+ orig_err = cause.is_a?(Exception) ? cause : nil
213
+ error = Error::StepFailureError.new(message, step: step_config.name, context: @context,
214
+ original_error: orig_err,
215
+ step_arguments: resolved_arguments,
216
+ exception_class: (result.exception_class unless orig_err),
217
+ validation_errors: result.validation_errors)
183
218
  if result.respond_to?(:backtrace) && result.backtrace
184
219
  error.set_backtrace(result.backtrace)
185
220
  elsif orig_err
186
221
  error.set_backtrace(orig_err.backtrace)
187
222
  end
188
- raise error
223
+ error
189
224
  end
190
225
 
191
226
  # Wrapped first, so a returned `Step::Inputs` is validated and stored as its Hash.
@@ -193,31 +228,16 @@ module RubyReactor
193
228
  success_result = RubyReactor.Success(result)
194
229
  validate_step_output(step_config, success_result.value, resolved_arguments)
195
230
  @step_results[step_config.name] = success_result
196
- @compensation_manager.add_to_undo_stack({ step: step_config, arguments: resolved_arguments,
231
+ @compensation_manager.add_to_undo_stack({ step: step_config,
232
+ arguments: step_config.rollback_arguments(resolved_arguments),
197
233
  result: success_result })
198
234
  @context.set_result(step_config.name, success_result.value)
199
235
  @dependency_graph.complete_step(step_config.name)
200
236
  end
201
237
 
202
- def handle_step_failure_error(error)
203
- current_context = error.context || @context
204
- current_context.current_step = error.step
205
-
206
- store_failed_map_context(current_context) if current_context.map_metadata
207
-
208
- @compensation_manager.rollback_completed_steps
209
-
210
- redact_inputs = []
211
- if error.context&.reactor_class
212
- redact_inputs = error.context.reactor_class.inputs.select { |_, config| config[:redact] }.keys
213
- end
214
-
215
- create_failure_from_error(error, redact_inputs)
216
- end
217
-
218
238
  def store_failed_map_context(context)
219
239
  return unless context.map_metadata && context.map_metadata[:map_id]
220
- return unless context.map_metadata[:fail_fast]
240
+ return unless Map::Helpers.normalize_arguments(context.map_metadata)[:atomic]
221
241
 
222
242
  storage = RubyReactor.configuration.storage_adapter
223
243
  storage.store_map_failed_context_id(
@@ -276,7 +296,7 @@ module RubyReactor
276
296
  # compensation and roll back prior steps, so the side effect is not
277
297
  # orphaned. Then surface the structured validation error (the later
278
298
  # rollback in handle_execution_error is a no-op — stack already clear).
279
- @compensation_manager.handle_step_failure(step_config, error, resolved_arguments)
299
+ handing_off_as(error) { @compensation_manager.handle_step_failure(step_config, error, resolved_arguments) }
280
300
  raise error
281
301
  end
282
302
 
@@ -61,7 +61,7 @@ module RubyReactor
61
61
  serialized_context = ContextSerializer.serialize(context_to_serialize)
62
62
 
63
63
  if @context.map_metadata
64
- map_args = @context.map_metadata.transform_keys(&:to_sym)
64
+ map_args = Map::Helpers.normalize_arguments(@context.map_metadata)
65
65
  configuration.async_router.perform_map_element_in(
66
66
  delay,
67
67
  map_id: map_args[:map_id],
@@ -75,7 +75,7 @@ module RubyReactor
75
75
  step_name: map_args[:step_name],
76
76
  batch_size: map_args[:batch_size],
77
77
  serialized_context: serialized_context,
78
- fail_fast: map_args[:fail_fast]
78
+ atomic: map_args[:atomic]
79
79
  )
80
80
  else
81
81
  # Persist BEFORE enqueue — the job payload is identity-only (F2). The
@@ -193,7 +193,8 @@ module RubyReactor
193
193
  reactor_name: reactor_class.name,
194
194
  step_arguments: result.respond_to?(:step_arguments) ? result.step_arguments : {},
195
195
  validation_errors: result.validation_errors,
196
- rollback_failures: result.rollback_failures
196
+ rollback_failures: result.rollback_failures,
197
+ exception_class: result.exception_class
197
198
  )
198
199
  end
199
200
 
@@ -133,9 +133,11 @@ module RubyReactor
133
133
  # the stack. A worker run is redelivered instead, and re-runs the step.
134
134
  def track_interrupted_construct(step_config, arguments, error)
135
135
  return if @context.inline_async_execution || Error::Rescuable === error # rubocop:disable Style/CaseEquality
136
+ return if error.is_a?(Error::RollbackHandedOff) # a rollback signal, not an interruption
136
137
  return unless step_config.undoes_partial_run?
137
138
 
138
- @compensation_manager.add_to_undo_stack({ step: step_config, arguments: arguments,
139
+ @compensation_manager.add_to_undo_stack({ step: step_config,
140
+ arguments: step_config.rollback_arguments(arguments),
139
141
  result: RubyReactor.Success(nil) })
140
142
  end
141
143
 
@@ -135,6 +135,10 @@ module RubyReactor
135
135
  handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
136
136
  completed = true
137
137
  @result
138
+ rescue Error::RollbackHandedOff => e
139
+ record_rollback_handoff(e)
140
+ completed = true
141
+ @result
138
142
  rescue RubyReactor::Lock::AcquisitionError,
139
143
  RubyReactor::Semaphore::AcquisitionError,
140
144
  RubyReactor::RateLimit::ExceededError,
@@ -153,7 +157,7 @@ module RubyReactor
153
157
  @contention_snooze = true
154
158
  raise
155
159
  rescue Error::Rescuable => e
156
- @result = aborting_on_interruption { @result_handler.handle_execution_error(e) }
160
+ @result = handing_off { aborting_on_interruption { @result_handler.handle_execution_error(e) } }
157
161
  update_context_status(@result)
158
162
  completed = true
159
163
  @result
@@ -184,6 +188,9 @@ module RubyReactor
184
188
  end
185
189
 
186
190
  def resume_execution # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity
191
+ # A composed child re-entered while its own rollback is handed off.
192
+ return resume_rollback if @context.rolling_back?
193
+
187
194
  middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
188
195
  completed = false
189
196
 
@@ -256,6 +263,10 @@ module RubyReactor
256
263
  handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
257
264
  completed = true
258
265
  @result
266
+ rescue Error::RollbackHandedOff => e
267
+ record_rollback_handoff(e)
268
+ completed = true
269
+ @result
259
270
  rescue RubyReactor::Lock::AcquisitionError,
260
271
  RubyReactor::Semaphore::AcquisitionError,
261
272
  RubyReactor::RateLimit::ExceededError,
@@ -274,7 +285,7 @@ module RubyReactor
274
285
  @contention_snooze = true
275
286
  raise e
276
287
  rescue Error::Rescuable => e
277
- aborting_on_interruption { handle_resume_error(e) }
288
+ handing_off { aborting_on_interruption { handle_resume_error(e) } }
278
289
  update_context_status(@result)
279
290
  completed = true
280
291
  @result
@@ -283,16 +294,100 @@ module RubyReactor
283
294
  raise
284
295
  ensure
285
296
  release_locks unless @parked
297
+ # Saved while the context lock is still held (009 R-13): released first,
298
+ # a worker resuming this run could load the pre-save blob, and this save
299
+ # would then overwrite its progress.
300
+ save_context unless skip_context_persist?
286
301
  @acquired_context_lock&.release
287
302
  @acquired_context_lock = nil
288
303
  leave_ordered_lock_scope
304
+
305
+ emit_lifecycle_completion(completed)
306
+ end
307
+
308
+ # Undoes every completed step (manual undo, a composed child's undo, a
309
+ # map element's rollback). A rollback that hands off here (009 R-04)
310
+ # leaves this level's rollback failures on its context, and the next
311
+ # `undo_all` of the same context picks them up, so none is lost though
312
+ # the executor is rebuilt on resume (G2).
313
+ def undo_all(&after_pop)
314
+ saved = @context.rollback && @context.rollback["failures"]
315
+ @compensation_manager.restore_rollback_failures(saved) if saved
316
+ @compensation_manager.rollback_completed_steps(&after_pop)
317
+ clear_saved_rollback_failures
318
+ rescue Error::RollbackHandedOff
319
+ (@context.rollback ||= {})["failures"] =
320
+ ContextSerializer.serialize_value(@compensation_manager.rollback_failures)
321
+ raise
322
+ end
323
+
324
+ # Finishes a rollback that handed off at a fan-out map (009 R-04), once
325
+ # the map's element rollbacks have all reported: the Worker of a
326
+ # `rolling_back` run, or a composed child re-entered by its root. Under
327
+ # the run's context lock, as `resume_execution`.
328
+ def resume_rollback
329
+ state = @context.rollback || {}
330
+ # The hand-off came out of a running step (a composed child rolling
331
+ # back inside `ComposeStep#run`): resume forward, and that step
332
+ # finishes its child's rollback.
333
+ if state["trigger"] == "step" || state.empty?
334
+ @context.rollback = nil
335
+ @context.status = :running
336
+ return resume_execution
337
+ end
338
+
339
+ middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
340
+ completed = false
341
+ enter_ordered_lock_scope
342
+ acquire_context_lock
343
+ @result = finish_rollback(state)
344
+ @context.rollback = nil
345
+ update_context_status(@result)
346
+ completed = true
347
+ @result
348
+ rescue Error::RollbackHandedOff => e
349
+ record_rollback_handoff(e)
350
+ completed = true
351
+ @result
352
+ rescue RubyReactor::Lock::AcquisitionError => e
353
+ @contention_snooze = true
354
+ raise composed_contention_park(e) || e
355
+ rescue Error::Rescuable => e
356
+ handing_off { aborting_on_interruption { handle_resume_error(e) } }
357
+ update_context_status(@result)
358
+ completed = true
359
+ @result
360
+ rescue Exception # rubocop:disable Lint/RescueException
361
+ mark_aborted
362
+ raise
363
+ ensure
289
364
  save_context unless skip_context_persist?
365
+ @acquired_context_lock&.release
366
+ @acquired_context_lock = nil
367
+ leave_ordered_lock_scope
290
368
 
291
369
  emit_lifecycle_completion(completed)
292
370
  end
293
371
 
294
- def undo_all
295
- @compensation_manager.rollback_completed_steps
372
+ # The rollback's other half (009 R-05): marks the hand-off saved, then
373
+ # enqueues the owner's Worker if the map rollback already settled and this
374
+ # side claims the signal. Whichever of this and the last element rollback
375
+ # job sees both second enqueues the owner, once. Called right after the
376
+ # `rolling_back` save, while the context lock is still held.
377
+ def hand_off_rollback!(handed_off)
378
+ storage = RubyReactor.configuration.storage_adapter
379
+ map_id = handed_off.map_id
380
+ map_class = handed_off.reactor_class_name
381
+ storage.mark_map_rollback_handed_off(map_id, map_class)
382
+ RubyReactor.configuration.logger.info(
383
+ "event=ruby_reactor.rollback.handed_off context_id=#{@context.context_id.inspect} map_id=#{map_id.inspect}"
384
+ )
385
+ meta = storage.retrieve_map_rollback_metadata(map_id, map_class)
386
+ return unless meta && storage.count_map_rollback_outcomes(map_id, map_class) >= meta["total"].to_i
387
+ return unless storage.claim_map_rollback_signal(map_id, map_class)
388
+
389
+ RubyReactor.configuration.async_router.perform_async(@context.context_id,
390
+ RubyReactor.reactor_storage_name(@reactor_class))
296
391
  end
297
392
 
298
393
  # True once `resume_execution` is past its reactor-level gates (context
@@ -318,10 +413,18 @@ module RubyReactor
318
413
  def aborting_on_interruption
319
414
  yield
320
415
  rescue Exception => e # rubocop:disable Lint/RescueException
321
- mark_aborted unless Error::Rescuable === e # rubocop:disable Style/CaseEquality
416
+ mark_aborted unless Error::Rescuable === e || e.is_a?(Error::RollbackHandedOff) # rubocop:disable Style/CaseEquality
322
417
  raise
323
418
  end
324
419
 
420
+ # A rollback run from a `rescue Error::Rescuable` body (the error's own
421
+ # rollback) is outside the sibling `rescue Error::RollbackHandedOff`.
422
+ def handing_off
423
+ yield
424
+ rescue Error::RollbackHandedOff => e
425
+ record_rollback_handoff(e)
426
+ end
427
+
325
428
  def undo_stack
326
429
  @compensation_manager.undo_stack
327
430
  end
@@ -768,7 +871,8 @@ module RubyReactor
768
871
 
769
872
  case result
770
873
  when RubyReactor::DispatchResult
771
- @context.status = :running
874
+ # A rollback hand-off returns one too, and stays `rolling_back`.
875
+ @context.status = :running unless @context.rolling_back?
772
876
  when RubyReactor::Halt
773
877
  @context.status = :halted
774
878
  when RubyReactor::Success
@@ -833,6 +937,82 @@ module RubyReactor
833
937
  @result
834
938
  end
835
939
 
940
+ # Where this level's rollback stands when it hands off (009 DM §1). A
941
+ # composed child keeps it on its own context, inside the root's blob, and
942
+ # re-raises; the top-level run saves it as `rolling_back` while it still
943
+ # holds the context lock (I-6), runs the handshake, and returns a
944
+ # `DispatchResult`, as a forward hand-off does.
945
+ def record_rollback_handoff(handed_off)
946
+ failure = handed_off.failure
947
+ handed_off.failure = nil # this level's Failure; the level above builds its own
948
+ @context.rollback = rollback_state(failure)
949
+ @context.status = :rolling_back
950
+ raise handed_off if @context.root_context
951
+
952
+ save_context
953
+ hand_off_rollback!(handed_off)
954
+ @result = RubyReactor::DispatchResult.new(job_id: "map_rollback:#{handed_off.map_id}",
955
+ intermediate_results: @context.intermediate_results,
956
+ execution_id: @context.context_id)
957
+ end
958
+
959
+ # `trigger`: `failure` (a step failure is rolling back), `undo` (set by
960
+ # `Reactor#undo`), or `step` (a running step's child handed off, and no
961
+ # rollback started at this level). A later hand-off of the same rollback
962
+ # keeps what an earlier one recorded.
963
+ def rollback_state(failure)
964
+ state = (@context.rollback || {}).dup
965
+ pending = @compensation_manager.pending
966
+ state["trigger"] ||= failure || pending ? "failure" : "step"
967
+ if pending
968
+ state["step"] = pending[:step].to_s
969
+ state["compensated"] = pending[:compensated]
970
+ state["compensation_error"] = pending[:compensation_error]
971
+ end
972
+ state["failure"] = ContextSerializer.serialize_value(failure) if failure
973
+ state["failures"] = ContextSerializer.serialize_value(@compensation_manager.rollback_failures)
974
+ state
975
+ end
976
+
977
+ # The rest of a handed-off rollback: the failing step's own compensate if
978
+ # it had not finished (a map or compose adopts its settled state), the
979
+ # steps still on the undo stack, then the outcome the inline rollback
980
+ # gives (I-7).
981
+ def finish_rollback(state)
982
+ @compensation_manager.restore_rollback_failures(state["failures"])
983
+ return finish_undo if state["trigger"] == "undo"
984
+
985
+ failure = ContextSerializer.deserialize_value(state["failure"])
986
+ step_config = state["step"] && @reactor_class.steps[state["step"].to_sym]
987
+ if step_config && !state["compensated"]
988
+ @compensation_manager.handle_step_failure(step_config, failure.error, {})
989
+ else
990
+ @compensation_manager.rollback_completed_steps
991
+ if state["compensation_error"]
992
+ raise Error::CompensationError.new(state["compensation_error"], step: step_config&.name, context: @context)
993
+ end
994
+ end
995
+ @context.current_step = failure.step_name&.to_sym
996
+ @result_handler.handed_off_failure(failure)
997
+ rescue Error::CompensationError => e
998
+ @result_handler.handle_execution_error(e)
999
+ end
1000
+
1001
+ def finish_undo
1002
+ @compensation_manager.rollback_completed_steps
1003
+ @context.cancelled = true
1004
+ @context.cancellation_reason = "Undo triggered"
1005
+ @context.status = "cancelled"
1006
+ nil
1007
+ end
1008
+
1009
+ def clear_saved_rollback_failures
1010
+ return unless @context.rollback
1011
+
1012
+ @context.rollback.delete("failures")
1013
+ @context.rollback = nil if @context.rollback.empty?
1014
+ end
1015
+
836
1016
  def handle_interrupt(interrupt_result)
837
1017
  save_context
838
1018