ruby_reactor 0.8.2 → 0.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. checksums.yaml +4 -4
  2. data/.claude/skills/speckit-review/SKILL.md +324 -0
  3. data/.release-please-manifest.json +1 -1
  4. data/.specify/extensions.yml +10 -0
  5. data/.specify/feature.json +1 -1
  6. data/.specify/workflows/speckit/workflow.yml +13 -1
  7. data/.specify/workflows/workflow-registry.json +2 -2
  8. data/CHANGELOG.md +94 -0
  9. data/CLAUDE.md +2 -2
  10. data/README.md +128 -93
  11. data/lib/ruby_reactor/adapters/active_job/router.rb +19 -0
  12. data/lib/ruby_reactor/adapters/sidekiq/router.rb +21 -0
  13. data/lib/ruby_reactor/context.rb +26 -0
  14. data/lib/ruby_reactor/context_serializer.rb +4 -2
  15. data/lib/ruby_reactor/dsl/async_reactor_builder.rb +3 -6
  16. data/lib/ruby_reactor/dsl/compose_builder.rb +3 -10
  17. data/lib/ruby_reactor/dsl/interrupt_builder.rb +14 -0
  18. data/lib/ruby_reactor/dsl/lockable.rb +76 -21
  19. data/lib/ruby_reactor/dsl/reactor.rb +8 -11
  20. data/lib/ruby_reactor/dsl/retryable.rb +45 -0
  21. data/lib/ruby_reactor/dsl/step_builder.rb +150 -13
  22. data/lib/ruby_reactor/error/async_result_pending.rb +1 -1
  23. data/lib/ruby_reactor/error/execution_parked.rb +16 -0
  24. data/lib/ruby_reactor/error/reactor_contention_park.rb +26 -0
  25. data/lib/ruby_reactor/error/step_contention_park.rb +26 -0
  26. data/lib/ruby_reactor/error/undeclared_input_error.rb +13 -0
  27. data/lib/ruby_reactor/executor/async_step_dispatch.rb +109 -3
  28. data/lib/ruby_reactor/executor/compensation_manager.rb +99 -17
  29. data/lib/ruby_reactor/executor/ordered_lock_support.rb +76 -44
  30. data/lib/ruby_reactor/executor/result_handler.rb +34 -13
  31. data/lib/ruby_reactor/executor/retry_manager.rb +20 -13
  32. data/lib/ruby_reactor/executor/step_coordination.rb +788 -0
  33. data/lib/ruby_reactor/executor/step_executor.rb +115 -11
  34. data/lib/ruby_reactor/executor.rb +90 -20
  35. data/lib/ruby_reactor/map/element_executor.rb +24 -2
  36. data/lib/ruby_reactor/map/helpers.rb +35 -11
  37. data/lib/ruby_reactor/max_retries_exhausted_failure.rb +2 -2
  38. data/lib/ruby_reactor/open_telemetry.rb +61 -24
  39. data/lib/ruby_reactor/retry_context.rb +31 -2
  40. data/lib/ruby_reactor/rspec/helpers.rb +15 -0
  41. data/lib/ruby_reactor/rspec/matchers.rb +92 -0
  42. data/lib/ruby_reactor/rspec/test_subject.rb +9 -5
  43. data/lib/ruby_reactor/step/async_reactor_step.rb +46 -26
  44. data/lib/ruby_reactor/step/compose_step.rb +22 -7
  45. data/lib/ruby_reactor/step/input_contract.rb +5 -0
  46. data/lib/ruby_reactor/step/inputs.rb +57 -0
  47. data/lib/ruby_reactor/step/map_step.rb +32 -22
  48. data/lib/ruby_reactor/step.rb +59 -11
  49. data/lib/ruby_reactor/step_sweeper.rb +29 -1
  50. data/lib/ruby_reactor/step_worker.rb +260 -37
  51. data/lib/ruby_reactor/version.rb +1 -1
  52. data/lib/ruby_reactor/web/api.rb +72 -7
  53. data/lib/ruby_reactor/web/coordination_serializer.rb +120 -2
  54. data/lib/ruby_reactor/web/public/assets/{index-Dw4KV4QY.js → index-CeZU-ESu.js} +9 -9
  55. data/lib/ruby_reactor/web/public/index.html +1 -1
  56. data/lib/ruby_reactor/worker.rb +56 -30
  57. data/lib/ruby_reactor.rb +30 -6
  58. data/specs/{004-inheritable-step-class → 006-step-retry-declarations}/checklists/requirements.md +10 -9
  59. data/specs/006-step-retry-declarations/contracts/dsl-surface.md +89 -0
  60. data/specs/006-step-retry-declarations/data-model.md +58 -0
  61. data/specs/006-step-retry-declarations/plan.md +187 -0
  62. data/specs/006-step-retry-declarations/quickstart.md +80 -0
  63. data/specs/006-step-retry-declarations/research.md +194 -0
  64. data/specs/006-step-retry-declarations/spec.md +453 -0
  65. data/specs/006-step-retry-declarations/tasks.md +382 -0
  66. data/specs/future_improvements.md +250 -0
  67. data/specs/specs-inputs-by-method-md-piped-wigderson.md +77 -0
  68. metadata +20 -28
  69. data/specs/002-step-input-contracts/checklists/requirements.md +0 -49
  70. data/specs/002-step-input-contracts/contracts/dsl-surface.md +0 -193
  71. data/specs/002-step-input-contracts/data-model.md +0 -115
  72. data/specs/002-step-input-contracts/plan.md +0 -165
  73. data/specs/002-step-input-contracts/quickstart.md +0 -170
  74. data/specs/002-step-input-contracts/research.md +0 -233
  75. data/specs/002-step-input-contracts/spec.md +0 -359
  76. data/specs/002-step-input-contracts/tasks.md +0 -367
  77. data/specs/004-inheritable-step-class/contracts/step-lifecycle.md +0 -85
  78. data/specs/004-inheritable-step-class/data-model.md +0 -116
  79. data/specs/004-inheritable-step-class/plan.md +0 -174
  80. data/specs/004-inheritable-step-class/quickstart.md +0 -112
  81. data/specs/004-inheritable-step-class/research.md +0 -308
  82. data/specs/004-inheritable-step-class/spec.md +0 -316
  83. data/specs/004-inheritable-step-class/tasks.md +0 -258
  84. data/specs/active_job.md +0 -259
  85. data/specs/deferred-003-step-lock-declarations/checklists/requirements.md +0 -51
  86. data/specs/deferred-003-step-lock-declarations/contracts/dsl-surface.md +0 -154
  87. data/specs/deferred-003-step-lock-declarations/data-model.md +0 -131
  88. data/specs/deferred-003-step-lock-declarations/plan.md +0 -166
  89. data/specs/deferred-003-step-lock-declarations/quickstart.md +0 -169
  90. data/specs/deferred-003-step-lock-declarations/research.md +0 -196
  91. data/specs/deferred-003-step-lock-declarations/spec.md +0 -447
  92. data/specs/deferred-003-step-lock-declarations/tasks.md +0 -572
  93. data/specs/possible_feature.md +0 -22
@@ -28,6 +28,14 @@ module RubyReactor
28
28
  return RubyReactor.Success(nil)
29
29
  end
30
30
 
31
+ deadlock = check_async_step_deadlock(step_config)
32
+ # Through the normal step-failure handler, never returned bare: a bare
33
+ # Failure goes straight back to `Executor#execute`, which marks the
34
+ # context failed WITHOUT rolling back — leaving an earlier step's side
35
+ # effect uncompensated. `handle_step_result` compensates and raises the
36
+ # `StepFailureError` the executor's own rescue chain expects.
37
+ return @result_handler.handle_step_result(step_config, deadlock, {}) if deadlock
38
+
31
39
  record_async_step_dispatch(step_config)
32
40
  enqueue_async_step(step_config)
33
41
 
@@ -43,6 +51,101 @@ module RubyReactor
43
51
  !storage.retrieve_step_result(@context.context_id, step_config.name, async_step_class_name).nil?
44
52
  end
45
53
 
54
+ # US4/T034: an `async_step` whose step class declares a key this
55
+ # execution currently holds would deadlock exactly like `async_reactor`
56
+ # dispatching into one — refuse before the durable record is written or
57
+ # the job is enqueued, naming the key (Step::AsyncReactorStep's guard
58
+ # already covers `async_reactor`; this reuses its message and registry).
59
+ def check_async_step_deadlock(step_config)
60
+ held = Step::AsyncReactorStep.held_lock_keys(@context)
61
+ return nil if held.empty?
62
+
63
+ # Only lock and a limit-1 semaphore have the circular-wait shape this
64
+ # guard defends against (same registry `with_lock`/limit-1
65
+ # `with_semaphore` push to, T032). Rate limit, period, and the
66
+ # ordered lock never enter `held_lock_keys`, so they cannot deadlock
67
+ # a hand-off and are deliberately not checked here.
68
+ lock_config = step_config.lock_config
69
+ semaphore_config = step_config.semaphore_config
70
+ return nil unless lock_config || (semaphore_config && semaphore_config[:limit] == 1)
71
+
72
+ args = resolve_args_for_deadlock_check(step_config)
73
+ return args if args.is_a?(RubyReactor::Failure) # KeyError or "skip, can't resolve without blocking"
74
+ return nil if args.nil? # skipped — non-blocking resolution was not possible
75
+
76
+ collision = async_step_lock_keys(lock_config, semaphore_config, args, step_config)
77
+ .find { |key| held.include?(key) }
78
+ return nil unless collision
79
+
80
+ # A never-started error, not a bare message: the step was neither
81
+ # dispatched nor run, so rollback must not compensate it.
82
+ message = Step::AsyncReactorStep.deadlock_message(collision, "#{@reactor_class&.name}##{step_config.name}",
83
+ @context, kind: "async_step")
84
+ never_started_failure(Executor::StepCoordination::DispatchRefused.new(message, step: step_config.name),
85
+ step_config)
86
+ rescue Executor::StepCoordination::KeyError => e
87
+ # A guard key that cannot be computed is the same non-retryable step
88
+ # failure the real acquisition would raise (FR-007) — never a generic
89
+ # execution error, which would skip rollback of the earlier steps.
90
+ never_started_failure(e, step_config)
91
+ end
92
+
93
+ # Through `StepCoordination.resolve_key`, so a nil/empty or raising key
94
+ # proc fails here exactly as it would at acquisition time.
95
+ def async_step_lock_keys(lock_config, semaphore_config, args, step_config)
96
+ keys = []
97
+ keys << Executor::StepCoordination.resolve_key(lock_config, args, step_config.name) if lock_config
98
+ if semaphore_config && semaphore_config[:limit] == 1
99
+ keys << Executor::StepCoordination.resolve_key(semaphore_config, args, step_config.name)
100
+ end
101
+ keys.compact
102
+ end
103
+
104
+ def never_started_failure(error, step_config)
105
+ RubyReactor::Failure(error, step_name: step_config.name, reactor_name: @reactor_class&.name,
106
+ retryable: false)
107
+ end
108
+
109
+ # Finding 5: `async_step` defers argument resolution to the worker, so
110
+ # computing the key here means resolving early — safe UNLESS an
111
+ # argument reads a still-pending async result, which would BLOCK (or
112
+ # park) this dispatching step just to run a guard check. Detect that
113
+ # case without calling `.resolve` at all, skip the guard, and log it.
114
+ def resolve_args_for_deadlock_check(step_config)
115
+ if step_config.arguments.values.any? { |cfg| pending_async_source?(cfg[:source]) }
116
+ log_guard_skipped(step_config)
117
+ return nil
118
+ end
119
+
120
+ resolved = {}
121
+ step_config.arguments.each do |name, cfg|
122
+ value = cfg[:source].resolve(@context)
123
+ value = cfg[:transform].call(value) if cfg[:transform]
124
+ resolved[name] = value
125
+ end
126
+ # The worker keys off `coordination_arguments` of these same values;
127
+ # anything else here would let a self-deadlocking job through.
128
+ step_config.coordination_arguments(resolved, @context.inputs)
129
+ rescue Executor::StepCoordination::KeyError => e
130
+ never_started_failure(e, step_config)
131
+ end
132
+
133
+ def pending_async_source?(source)
134
+ return false unless source.is_a?(RubyReactor::Template::Result)
135
+ return false if @context.intermediate_results.key?(source.step_name.to_sym) ||
136
+ @context.intermediate_results.key?(source.step_name.to_s)
137
+
138
+ ref = @context.composed_contexts[source.step_name] || @context.composed_contexts[source.step_name.to_s]
139
+ ref.is_a?(Hash) && %i[async_step_ref async_reactor_ref].include?(ref[:type]&.to_sym)
140
+ end
141
+
142
+ def log_guard_skipped(step_config)
143
+ configuration.logger.info(
144
+ "event=\"ruby_reactor.step_coordination.guard_skipped\" reactor=#{@reactor_class&.name.inspect} " \
145
+ "step=#{step_config.name.inspect} execution_id=#{@context.context_id.inspect}"
146
+ )
147
+ end
148
+
46
149
  def record_async_step_dispatch(step_config)
47
150
  root = @context.root_context || @context
48
151
  @context.composed_contexts[step_config.name] = {
@@ -98,11 +201,14 @@ module RubyReactor
98
201
  end
99
202
 
100
203
  # One machine-parseable line per hand-off / dispatch, carrying the
101
- # three identifiers needed to correlate it with everything else.
102
- def log_async_event(event, step_name)
204
+ # three identifiers needed to correlate it with everything else, plus
205
+ # any extra key=value fields (e.g. a park's key/primitive/attempt),
206
+ # inserted before execution_id so it always trails the line.
207
+ def log_async_event(event, step_name, **fields)
208
+ extra = fields.map { |k, v| " #{k}=#{v.inspect}" }.join
103
209
  configuration.logger.info(
104
210
  "event=\"ruby_reactor.#{event}\" reactor=#{@reactor_class&.name.inspect} " \
105
- "step=#{step_name.inspect} execution_id=#{@context.context_id.inspect}"
211
+ "step=#{step_name.inspect}#{extra} execution_id=#{@context.context_id.inspect}"
106
212
  )
107
213
  end
108
214
  end
@@ -3,22 +3,49 @@
3
3
  module RubyReactor
4
4
  class Executor
5
5
  class CompensationManager
6
+ # Raised ONLY by the step's own coordination, before its body: a
7
+ # coordination error raised from inside a body (a nested direct
8
+ # `Step.run`) arrives as `StepCoordination::NestedCoordinationError`, and
9
+ # a bare `Lock::AcquisitionError` etc. may come from a nested
10
+ # `Reactor.run` — both mean the body ran, so neither is listed here.
11
+ NEVER_STARTED_ERROR_CLASSES = [
12
+ RubyReactor::Executor::StepCoordination::Contended,
13
+ RubyReactor::Executor::StepCoordination::KeyError,
14
+ RubyReactor::Executor::StepCoordination::DispatchRefused
15
+ ].freeze
16
+
6
17
  def initialize(context)
7
18
  @context = context
8
19
  @undo_trace = []
20
+ @rollback_failures = []
9
21
  end
10
22
 
11
23
  def undo_stack
12
24
  @context.undo_stack
13
25
  end
14
26
 
15
- attr_reader :undo_trace
27
+ # Every undo/compensation that did not complete, in rollback order
28
+ # (005 FR-004). `ResultHandler` attaches it to the final Failure.
29
+ attr_reader :undo_trace, :rollback_failures
16
30
 
17
31
  def add_to_undo_stack(step_info)
18
32
  @context.undo_stack << step_info
19
33
  end
20
34
 
21
35
  def handle_step_failure(step_config, error, arguments)
36
+ # A step whose OWN coordination acquisition failed (contention, or a
37
+ # bad key proc) never ran its body — "no step compensates, the
38
+ # contended step's work has not been attempted" (US3-1/T018), which
39
+ # applies here exactly as it does to a worker park: compensating a
40
+ # step that never started is meaningless, and attempting one would
41
+ # try to re-acquire the very key that is (usually) still contended,
42
+ # turning a plain contention failure into a confusing
43
+ # CompensationError. Prior steps still roll back normally.
44
+ if step_never_started?(error)
45
+ rollback_completed_steps
46
+ return RubyReactor.Failure("Step '#{step_config.name}' failed: #{error}")
47
+ end
48
+
22
49
  # Try compensation
23
50
  compensation_result = compensate_step(step_config, error, arguments)
24
51
  case compensation_result
@@ -70,16 +97,38 @@ module RubyReactor
70
97
  result.respond_to?(:skipped?) && result.skipped?
71
98
  end
72
99
 
100
+ def step_never_started?(error)
101
+ NEVER_STARTED_ERROR_CLASSES.any? { |klass| error.is_a?(klass) }
102
+ end
103
+
104
+ # US6/T047: re-take a step's own lock/semaphore around its compensate
105
+ # or undo body, so a concurrent forward execution cannot enter the
106
+ # step's critical section while rollback is undoing what it protected.
107
+ # A no-op when the step declares no coordination.
108
+ def coordinated_rollback(step_config, arguments, &block)
109
+ return block.call if Executor::StepCoordination.none?(step_config)
110
+
111
+ # The undo stack stores the RESOLVED arguments; the forward key was
112
+ # computed from `coordination_arguments` of them, so rollback does the
113
+ # same and re-takes the very key the forward run held.
114
+ Executor::StepCoordination.new(
115
+ step_config: step_config, arguments: step_config.coordination_arguments(arguments, @context.inputs),
116
+ context: @context, reactor_class: @context.reactor_class, middlewares: middlewares
117
+ ).around_rollback(&block)
118
+ end
119
+
73
120
  def compensate_step(step_config, error, arguments)
74
121
  middlewares.on(:start_compensation, step_config.name, error, arguments, @context)
75
122
  begin
76
- compensate_result = catch(StepSignals::TAG) do
77
- if step_config.compensate_block
78
- step_config.compensate_block.call(error, arguments, @context)
79
- elsif step_config.has_impl?
80
- step_config.impl.compensate(error, arguments, @context)
81
- else
82
- RubyReactor.Skipped() # Default: nothing defined, rollback continues
123
+ compensate_result = coordinated_rollback(step_config, arguments) do
124
+ catch(StepSignals::TAG) do
125
+ if step_config.compensate_block
126
+ step_config.compensate_block.call(error, step_config.wrap_inputs(arguments), @context)
127
+ elsif step_config.has_impl?
128
+ step_config.impl.compensate(error, arguments, @context)
129
+ else
130
+ RubyReactor.Skipped() # Default: nothing defined, rollback continues
131
+ end
83
132
  end
84
133
  end
85
134
 
@@ -96,6 +145,7 @@ module RubyReactor
96
145
  @undo_trace << { type: :compensation, step: step_config.name, error: error, arguments: arguments }
97
146
 
98
147
  if compensate_result.is_a?(RubyReactor::Failure)
148
+ record_rollback_failure(step_config.name, :compensate, compensate_result)
99
149
  middlewares.on(:failed_compensation, step_config.name, compensate_result, @context)
100
150
  else
101
151
  middlewares.on(:complete_compensation, step_config.name, compensate_result, @context)
@@ -103,21 +153,27 @@ module RubyReactor
103
153
 
104
154
  compensate_result
105
155
  rescue StandardError => e
156
+ record_rollback_failure(step_config.name, :compensate, e)
106
157
  middlewares.on(:failed_compensation, step_config.name, e, @context)
107
- raise e
158
+ # A raise is a compensation failure like a returned Failure: the
159
+ # caller still rolls back the completed steps, then raises
160
+ # `CompensationError`. Re-raising here skipped both.
161
+ RubyReactor.Failure(e)
108
162
  end
109
163
  end
110
164
 
111
- def undo_step(step_config, result, arguments)
165
+ def undo_step(step_config, result, arguments) # rubocop:disable Metrics/MethodLength
112
166
  middlewares.on(:start_undo, step_config.name, result, arguments, @context)
113
167
  begin
114
- undo_result = catch(StepSignals::TAG) do
115
- if step_config.undo_block
116
- step_config.undo_block.call(result.value, arguments, @context)
117
- elsif step_config.has_impl?
118
- step_config.impl.undo(result.value, arguments, @context)
119
- else
120
- RubyReactor.Skipped() # Default: nothing defined, rollback continues
168
+ undo_result = coordinated_rollback(step_config, arguments) do
169
+ catch(StepSignals::TAG) do
170
+ if step_config.undo_block
171
+ step_config.undo_block.call(result.value, step_config.wrap_inputs(arguments), @context)
172
+ elsif step_config.has_impl?
173
+ step_config.impl.undo(result.value, arguments, @context)
174
+ else
175
+ RubyReactor.Skipped() # Default: nothing defined, rollback continues
176
+ end
121
177
  end
122
178
  end
123
179
 
@@ -133,6 +189,7 @@ module RubyReactor
133
189
  )
134
190
 
135
191
  if undo_result.is_a?(RubyReactor::Failure)
192
+ record_rollback_failure(step_config.name, :undo, undo_result)
136
193
  middlewares.on(:failed_undo, step_config.name, undo_result, @context)
137
194
  else
138
195
  middlewares.on(:complete_undo, step_config.name, undo_result, @context)
@@ -140,6 +197,7 @@ module RubyReactor
140
197
 
141
198
  undo_result
142
199
  rescue StandardError => e
200
+ record_rollback_failure(step_config.name, :undo, e)
143
201
  middlewares.on(:failed_undo, step_config.name, e, @context)
144
202
  # Log undo failure but don't halt the rollback process
145
203
  @context.append_execution_trace(
@@ -148,6 +206,30 @@ module RubyReactor
148
206
  RubyReactor.Failure(e)
149
207
  end
150
208
  end
209
+
210
+ # `outcome` is the Failure an undo/compensate returned, or the exception
211
+ # it raised. A composed child's Failure already carries its own list —
212
+ # flatten it instead of adding one opaque entry for the compose step.
213
+ def record_rollback_failure(step_name, kind, outcome)
214
+ if outcome.is_a?(RubyReactor::Failure) && outcome.rollback_failures.any?
215
+ @rollback_failures.concat(outcome.rollback_failures)
216
+ return
217
+ end
218
+
219
+ error = outcome.is_a?(RubyReactor::Failure) ? outcome.error : outcome
220
+ contended = error.is_a?(StepCoordination::Contended)
221
+ reason = if contended
222
+ :coordination_unavailable
223
+ elsif outcome.is_a?(Exception)
224
+ :raised
225
+ else
226
+ :returned_failure
227
+ end
228
+ @rollback_failures << {
229
+ step: step_name.to_sym, kind: kind, key: (error.key if contended), reason: reason,
230
+ message: error.respond_to?(:message) ? error.message : error.to_s
231
+ }
232
+ end
151
233
  end
152
234
  end
153
235
  end
@@ -45,27 +45,39 @@ module RubyReactor
45
45
  }
46
46
  end
47
47
 
48
- # Strict-ordering gate. Runs BEFORE rate-limit / lock / semaphore so a
49
- # waiting nonce never holds any other primitive — preventing
50
- # hold-and-wait deadlocks when `with_lock` and `with_ordered_lock`
51
- # share inputs. Raises {OrderedLock::WaitError}; the Sidekiq worker
52
- # rescues and snoozes.
53
- def check_ordered_lock_gate
54
- info = ordered_lock_info
55
- return :go unless info
56
-
57
- OrderedLock.new(
58
- info.fetch(:key),
59
- nonce: info.fetch(:nonce),
60
- epoch: info.fetch(:epoch),
61
- poison_pill_timeout: info.fetch(:poison_pill_timeout),
62
- strict: info.fetch(:strict)
48
+ # THE strict-ordering gate classifier, shared by the reactor level
49
+ # (`enter_ordered_lock_scope`) and the step level
50
+ # (`StepCoordination#ordered_lock_gate`), so a gate state one level
51
+ # handles cannot fall through to "run" at the other (005 R-06, F7).
52
+ # Exhaustive: a state nobody mapped raises instead of running.
53
+ #
54
+ # :go — proceed (includes a poison advance past a dead blocker)
55
+ # :skip_chain — strict chain already failed, and this run is `fresh`
56
+ # (an in-flight run that paused completes regardless)
57
+ # :stale — the position belongs to a drained, reused generation
58
+ # :drained — the batch drained and GC'd while this caller slept
59
+ #
60
+ # Raises {OrderedLock::WaitError} when it is not this nonce's turn.
61
+ def self.gate(info, fresh:)
62
+ state = OrderedLock.new(
63
+ info.fetch(:key), nonce: info.fetch(:nonce), epoch: info.fetch(:epoch),
64
+ poison_pill_timeout: info.fetch(:poison_pill_timeout), strict: info.fetch(:strict)
63
65
  ).check!
66
+
67
+ case state
68
+ when :go then :go
69
+ when :skip_chain_failed then fresh ? :skip_chain : :go
70
+ when :stale_batch then :stale
71
+ when :drained_go then :drained
72
+ else raise ArgumentError, "unhandled ordered-lock gate state #{state.inspect}"
73
+ end
64
74
  end
65
75
 
66
76
  # Combined gate-check + thread-local push. Call at the top of
67
77
  # `execute` / `resume_execution`. Pair with `leave_ordered_lock_scope`
68
- # in `ensure`.
78
+ # in `ensure`. The gate runs BEFORE rate-limit / lock / semaphore so a
79
+ # waiting nonce never holds any other primitive; a `WaitError` goes to
80
+ # the worker, which snoozes.
69
81
  #
70
82
  # The strict-mode chain-skip only fires on a *fresh* start (no step
71
83
  # has run yet on this context). This lets an in-flight run that paused
@@ -74,20 +86,19 @@ module RubyReactor
74
86
  # strict to a fresh Sidekiq job (which enters via `resume_execution`
75
87
  # but has no prior step state).
76
88
  def enter_ordered_lock_scope
77
- gate = check_ordered_lock_gate
89
+ info = ordered_lock_info
90
+ outcome = info ? OrderedLockSupport.gate(info, fresh: fresh_ordered_lock_start?) : :go
78
91
  # A stale-batch run never participates regardless of fresh/resume state —
79
- # its numbering belongs to a drained generation. Chain-skip stays gated
80
- # on a fresh start so an in-flight paused run still completes on resume.
81
- @ordered_lock_stale_batch = gate == :stale_batch
82
- @ordered_lock_chain_skip = fresh_ordered_lock_start? && gate == :skip_chain_failed
92
+ # its numbering belongs to a drained generation.
93
+ @ordered_lock_stale_batch = outcome == :stale
94
+ @ordered_lock_chain_skip = outcome == :skip_chain
83
95
 
84
96
  # Drained-batch gate: the batch GC'd while this caller slept. A genuine
85
97
  # late straggler runs (poison semantics); a Sidekiq redelivery of an
86
98
  # ALREADY-terminal context must not re-execute its steps. Only the
87
99
  # latter — confirmed by a terminal stored status — is short-circuited.
88
- @ordered_lock_drained_replay = gate == :drained_go && stored_status_terminal?
100
+ @ordered_lock_drained_replay = outcome == :drained && stored_status_terminal?
89
101
 
90
- info = ordered_lock_info
91
102
  return unless info
92
103
 
93
104
  OrderedLockSupport.active_keys << info[:key]
@@ -101,8 +112,9 @@ module RubyReactor
101
112
  start_ordered_lock_heartbeat(info)
102
113
  end
103
114
 
115
+ # Same predicate as `Executor#first_execution?` (005 R-03).
104
116
  def fresh_ordered_lock_start?
105
- @context.intermediate_results.empty? && @context.current_step.nil?
117
+ first_execution?
106
118
  end
107
119
 
108
120
  def ordered_lock_chain_skip?
@@ -215,17 +227,49 @@ module RubyReactor
215
227
  def start_ordered_lock_heartbeat(info)
216
228
  return if @ordered_lock_heartbeat_running
217
229
 
230
+ @ordered_lock_heartbeat_running = true
231
+ @ordered_lock_heartbeat = OrderedLockSupport.start_heartbeat(info)
232
+ end
233
+
234
+ def stop_ordered_lock_heartbeat
235
+ return unless @ordered_lock_heartbeat_running
236
+
237
+ @ordered_lock_heartbeat_running = false
238
+ @ordered_lock_heartbeat&.stop
239
+ @ordered_lock_heartbeat = nil
240
+ end
241
+
242
+ # Value object so a caller can `heartbeat.stop` without reaching into
243
+ # the thread/flag it wraps. Returned by `.start_heartbeat`.
244
+ class Heartbeat
245
+ def initialize(thread, running)
246
+ @thread = thread
247
+ @running = running
248
+ end
249
+
250
+ def stop
251
+ @running[0] = false
252
+ @thread.wakeup if @thread.alive?
253
+ @thread.join(0.1)
254
+ rescue StandardError
255
+ # Best-effort shutdown; never let heartbeat teardown break the caller's ensure chain.
256
+ end
257
+ end
258
+
259
+ # Shared by the reactor-level heartbeat above (`#start_ordered_lock_heartbeat`)
260
+ # and the step-level gate (`StepCoordination#ordered_lock_gate`, T061) — the
261
+ # restamp logic and interval math are identical, only the caller's own
262
+ # on/off bookkeeping differs.
263
+ def self.start_heartbeat(info)
218
264
  pp = info[:poison_pill_timeout].to_f
219
265
  interval = [pp / 3.0, HEARTBEAT_MIN_INTERVAL].max
220
- @ordered_lock_heartbeat_running = true
221
- lock = OrderedLock.new(
222
- info.fetch(:key), nonce: info.fetch(:nonce), epoch: info.fetch(:epoch)
223
- )
266
+ running = [true]
267
+ lock = OrderedLock.new(info.fetch(:key), nonce: info.fetch(:nonce), epoch: info.fetch(:epoch))
224
268
 
225
- @ordered_lock_heartbeat = Thread.new do
226
- while @ordered_lock_heartbeat_running
269
+ thread = Thread.new do
270
+ while running[0]
227
271
  sleep interval
228
- break unless @ordered_lock_heartbeat_running
272
+ break unless running[0]
229
273
 
230
274
  begin
231
275
  lock.heartbeat!
@@ -238,20 +282,8 @@ module RubyReactor
238
282
  end
239
283
  end
240
284
  end
241
- end
242
285
 
243
- def stop_ordered_lock_heartbeat
244
- return unless @ordered_lock_heartbeat_running
245
-
246
- @ordered_lock_heartbeat_running = false
247
- thread = @ordered_lock_heartbeat
248
- @ordered_lock_heartbeat = nil
249
- return unless thread
250
-
251
- thread.wakeup if thread.alive?
252
- thread.join(0.1)
253
- rescue StandardError
254
- # Best-effort shutdown; never let heartbeat teardown break the ensure chain.
286
+ Heartbeat.new(thread, running)
255
287
  end
256
288
 
257
289
  # Advance the cursor when this run reached a *terminal* status.
@@ -32,7 +32,26 @@ module RubyReactor
32
32
  end
33
33
  end
34
34
 
35
+ # Every reactor-level Failure that follows a rollback is built here, so
36
+ # this is the one place `rollback_failures` is attached (005 R-08).
35
37
  def handle_execution_error(error)
38
+ failure = build_execution_failure(error)
39
+ failure.rollback_failures.concat(@compensation_manager.rollback_failures) if failure.is_a?(RubyReactor::Failure)
40
+ failure
41
+ end
42
+
43
+ def final_result(reactor_class)
44
+ if reactor_class.return_step
45
+ result_value = @context.get_result(reactor_class.return_step)
46
+ RubyReactor.Success(result_value)
47
+ else
48
+ RubyReactor.Success(@context.intermediate_results)
49
+ end
50
+ end
51
+
52
+ private
53
+
54
+ def build_execution_failure(error)
36
55
  case error
37
56
  when Error::StepFailureError
38
57
  handle_step_failure_error(error)
@@ -52,17 +71,6 @@ module RubyReactor
52
71
  end
53
72
  end
54
73
 
55
- def final_result(reactor_class)
56
- if reactor_class.return_step
57
- result_value = @context.get_result(reactor_class.return_step)
58
- RubyReactor.Success(result_value)
59
- else
60
- RubyReactor.Success(@context.intermediate_results)
61
- end
62
- end
63
-
64
- private
65
-
66
74
  # Failure for a validation error (reactor inputs, step arguments, or
67
75
  # step output), carrying both the structured field errors and the step/
68
76
  # reactor attribution stamped at the raise site (nil step_name for
@@ -138,7 +146,16 @@ module RubyReactor
138
146
  step_config.respond_to?(:async_dispatch?) && step_config.async_dispatch?
139
147
  end
140
148
 
149
+ # A composed child's Failure carries the child's own rollback failures;
150
+ # fold them in BEFORE this level rolls back, so they come first.
151
+ def adopt_rollback_failures(result)
152
+ return unless result.respond_to?(:rollback_failures)
153
+
154
+ @compensation_manager.rollback_failures.concat(result.rollback_failures)
155
+ end
156
+
141
157
  def handle_retries_exhausted(step_config, result, resolved_arguments)
158
+ adopt_rollback_failures(result)
142
159
  @compensation_manager.handle_step_failure(step_config, result.original_error, resolved_arguments)
143
160
  orig_err = result.original_error.is_a?(Exception) ? result.original_error : nil
144
161
  error = Error::StepFailureError.new(result.error, step: step_config.name, context: @context,
@@ -154,6 +171,7 @@ module RubyReactor
154
171
  end
155
172
 
156
173
  def handle_failure(step_config, result, resolved_arguments)
174
+ adopt_rollback_failures(result)
157
175
  failure_result = @compensation_manager.handle_step_failure(step_config, result.error, resolved_arguments)
158
176
  orig_err = result.error.is_a?(Exception) ? result.error : nil
159
177
  # A step that propagates another unit's validation failure (an
@@ -170,13 +188,14 @@ module RubyReactor
170
188
  raise error
171
189
  end
172
190
 
191
+ # Wrapped first, so a returned `Step::Inputs` is validated and stored as its Hash.
173
192
  def handle_unknown_result(step_config, result, resolved_arguments)
174
- validate_step_output(step_config, result, resolved_arguments)
175
193
  success_result = RubyReactor.Success(result)
194
+ validate_step_output(step_config, success_result.value, resolved_arguments)
176
195
  @step_results[step_config.name] = success_result
177
196
  @compensation_manager.add_to_undo_stack({ step: step_config, arguments: resolved_arguments,
178
197
  result: success_result })
179
- @context.set_result(step_config.name, result)
198
+ @context.set_result(step_config.name, success_result.value)
180
199
  @dependency_graph.complete_step(step_config.name)
181
200
  end
182
201
 
@@ -233,6 +252,8 @@ module RubyReactor
233
252
  end
234
253
 
235
254
  def resolve_exception_class(original_error, error)
255
+ # A step's own contention is reported by its cause (Lock::AcquisitionError, ...).
256
+ original_error = original_error.original if original_error.is_a?(StepCoordination::Contended)
236
257
  return original_error.class.name if original_error
237
258
 
238
259
  error.respond_to?(:exception_class) ? error.exception_class : nil
@@ -24,20 +24,26 @@ module RubyReactor
24
24
  step_config.retry_config[:max_attempts])
25
25
  end
26
26
 
27
- def calculate_backoff_delay(step_config, _error, reactor_class)
27
+ def calculate_backoff_delay(step_config)
28
28
  attempt_number = @context.retry_context.attempts_for_step(step_config.name)
29
- backoff_strategy = step_config.retry_config[:backoff] || reactor_class.retry_defaults[:backoff]
30
- base_delay = step_config.retry_config[:base_delay] || reactor_class.retry_defaults[:base_delay]
29
+ retry_config = step_config.retry_config
31
30
 
32
- delay = RetryContext.calculate_backoff_delay(attempt_number, backoff_strategy, base_delay)
31
+ delay = RetryContext.calculate_backoff_delay(attempt_number, retry_config[:backoff], retry_config[:base_delay])
33
32
  @context.retry_context.next_retry_at = Time.now + delay
34
33
  delay
35
34
  end
36
35
 
37
- def requeue_job_for_step_retry(step_config, error, reactor_class)
36
+ def requeue_job_for_step_retry(step_config)
38
37
  @context.current_step = step_config.name
39
- delay = calculate_backoff_delay(step_config, error, reactor_class)
38
+ delay = calculate_backoff_delay(step_config)
40
39
 
40
+ requeue_job(step_config, delay)
41
+ end
42
+
43
+ # The requeue for a failure retry, given an already-decided backoff
44
+ # delay. (A contention park never comes through here: it raises
45
+ # `Error::StepContentionPark` and is requeued by the worker, 005 R-01.)
46
+ def requeue_job(_step_config, delay)
41
47
  # Serialize context and requeue the job
42
48
  # Use root context if available to ensure we serialize the full tree
43
49
  # BUT for map elements (which have map_metadata), we must serialize the element context itself
@@ -140,14 +146,14 @@ module RubyReactor
140
146
 
141
147
  # Always try async retry if configured
142
148
  if is_async
143
- handle_async_retry(step_config, reactor_class, result)
149
+ handle_async_retry(step_config)
144
150
  else
145
- handle_sync_retry(step_config, reactor_class, result)
151
+ handle_sync_retry(step_config)
146
152
  end
147
153
  end
148
154
 
149
- def handle_async_retry(step_config, reactor_class, result)
150
- requeue_result = requeue_job_for_step_retry(step_config, result.error, reactor_class)
155
+ def handle_async_retry(step_config)
156
+ requeue_result = requeue_job_for_step_retry(step_config)
151
157
 
152
158
  # If it returned an DispatchResult, we are truly async.
153
159
  # Otherwise, it ran inline and we should return the result of that execution.
@@ -162,8 +168,8 @@ module RubyReactor
162
168
  end
163
169
  end
164
170
 
165
- def handle_sync_retry(step_config, reactor_class, result)
166
- delay = calculate_backoff_delay(step_config, result.error, reactor_class)
171
+ def handle_sync_retry(step_config)
172
+ delay = calculate_backoff_delay(step_config)
167
173
  sleep(delay)
168
174
  nil # continue loop
169
175
  end
@@ -186,7 +192,8 @@ module RubyReactor
186
192
  end,
187
193
  reactor_name: reactor_class.name,
188
194
  step_arguments: result.respond_to?(:step_arguments) ? result.step_arguments : {},
189
- validation_errors: result.validation_errors
195
+ validation_errors: result.validation_errors,
196
+ rollback_failures: result.rollback_failures
190
197
  )
191
198
  end
192
199