ruby_reactor 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/.specify/memory/constitution.md +17 -14
- data/CHANGELOG.md +254 -0
- data/CLAUDE.md +1 -1
- data/README.md +66 -15
- data/lib/ruby_reactor/adapters/active_job/map_element_rollback_worker.rb +20 -0
- data/lib/ruby_reactor/adapters/active_job/router.rb +32 -4
- data/lib/ruby_reactor/adapters/sidekiq/map_element_rollback_worker.rb +16 -0
- data/lib/ruby_reactor/adapters/sidekiq/router.rb +32 -4
- data/lib/ruby_reactor/context.rb +14 -1
- data/lib/ruby_reactor/dsl/async_macros.rb +30 -1
- data/lib/ruby_reactor/dsl/async_reactor_builder.rb +12 -6
- data/lib/ruby_reactor/dsl/compose_builder.rb +12 -6
- data/lib/ruby_reactor/dsl/definition_warnings.rb +30 -0
- data/lib/ruby_reactor/dsl/interrupt_builder.rb +1 -3
- data/lib/ruby_reactor/dsl/map_builder.rb +27 -9
- data/lib/ruby_reactor/dsl/step_builder.rb +93 -34
- data/lib/ruby_reactor/error/argument_resolution_error.rb +19 -0
- data/lib/ruby_reactor/error/rescuable.rb +33 -0
- data/lib/ruby_reactor/error/rollback_handed_off.rb +27 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +63 -35
- data/lib/ruby_reactor/executor/result_handler.rb +85 -63
- data/lib/ruby_reactor/executor/retry_manager.rb +4 -3
- data/lib/ruby_reactor/executor/step_coordination.rb +11 -8
- data/lib/ruby_reactor/executor/step_executor.rb +61 -49
- data/lib/ruby_reactor/executor.rb +221 -7
- data/lib/ruby_reactor/map/collector.rb +31 -89
- data/lib/ruby_reactor/map/dispatcher.rb +65 -8
- data/lib/ruby_reactor/map/element_executor.rb +21 -11
- data/lib/ruby_reactor/map/element_rollback.rb +158 -0
- data/lib/ruby_reactor/map/helpers.rb +11 -125
- data/lib/ruby_reactor/map/result_enumerator.rb +18 -12
- data/lib/ruby_reactor/map/step_rollback.rb +136 -0
- data/lib/ruby_reactor/map/sweeper.rb +66 -8
- data/lib/ruby_reactor/map.rb +17 -0
- data/lib/ruby_reactor/max_retries_exhausted_failure.rb +4 -2
- data/lib/ruby_reactor/reactor.rb +65 -2
- data/lib/ruby_reactor/rspec/active_job_helpers.rb +13 -2
- data/lib/ruby_reactor/rspec/matchers.rb +59 -24
- data/lib/ruby_reactor/rspec/sidekiq_helpers.rb +7 -3
- data/lib/ruby_reactor/rspec/test_subject.rb +13 -3
- data/lib/ruby_reactor/rspec.rb +3 -6
- data/lib/ruby_reactor/step/compose_step.rb +10 -1
- data/lib/ruby_reactor/step/map_step.rb +144 -36
- data/lib/ruby_reactor/step.rb +7 -0
- data/lib/ruby_reactor/step_worker.rb +46 -22
- data/lib/ruby_reactor/storage/adapter.rb +18 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +27 -2
- data/lib/ruby_reactor/storage/redis_map_rollback.rb +153 -0
- data/lib/ruby_reactor/storage/redis_reactor_scan.rb +1 -1
- data/lib/ruby_reactor/sweeper.rb +3 -1
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +11 -1
- data/lib/ruby_reactor/web/public/assets/{index-CeZU-ESu.js → index-C6ejszCQ.js} +12 -12
- data/lib/ruby_reactor/web/public/assets/index-DM5darhy.css +1 -0
- data/lib/ruby_reactor/web/public/index.html +2 -2
- data/lib/ruby_reactor/worker.rb +7 -3
- data/lib/ruby_reactor.rb +17 -6
- data/specs/007-execution-flow-analysis/analysis/README.md +147 -0
- data/specs/007-execution-flow-analysis/analysis/execution-order.md +359 -0
- data/specs/007-execution-flow-analysis/analysis/findings-and-options.md +502 -0
- data/specs/007-execution-flow-analysis/analysis/invariants.md +109 -0
- data/specs/007-execution-flow-analysis/checklists/requirements.md +39 -0
- data/specs/007-execution-flow-analysis/contracts/report-structure.md +71 -0
- data/specs/007-execution-flow-analysis/data-model.md +83 -0
- data/specs/007-execution-flow-analysis/evidence/harness.rb +231 -0
- data/specs/007-execution-flow-analysis/evidence/output.txt +333 -0
- data/specs/007-execution-flow-analysis/evidence/probes/01_plain.rb +122 -0
- data/specs/007-execution-flow-analysis/evidence/probes/02_compose.rb +182 -0
- data/specs/007-execution-flow-analysis/evidence/probes/03_map.rb +235 -0
- data/specs/007-execution-flow-analysis/evidence/probes/04_async.rb +132 -0
- data/specs/007-execution-flow-analysis/evidence/probes/05_background.rb +58 -0
- data/specs/007-execution-flow-analysis/evidence/probes/06_coordination.rb +158 -0
- data/specs/007-execution-flow-analysis/evidence/probes/07_interrupts_manual.rb +185 -0
- data/specs/007-execution-flow-analysis/evidence/run.rb +15 -0
- data/specs/007-execution-flow-analysis/plan.md +127 -0
- data/specs/007-execution-flow-analysis/quickstart.md +51 -0
- data/specs/007-execution-flow-analysis/research.md +202 -0
- data/specs/007-execution-flow-analysis/spec.md +270 -0
- data/specs/007-execution-flow-analysis/tasks.md +257 -0
- data/specs/008-rollback-reliability/checklists/requirements.md +43 -0
- data/specs/008-rollback-reliability/contracts/api-surface.md +126 -0
- data/specs/008-rollback-reliability/contracts/rollback-semantics.md +76 -0
- data/specs/008-rollback-reliability/data-model.md +139 -0
- data/specs/008-rollback-reliability/plan.md +233 -0
- data/specs/008-rollback-reliability/quickstart.md +105 -0
- data/specs/008-rollback-reliability/research.md +653 -0
- data/specs/008-rollback-reliability/spec.md +561 -0
- data/specs/008-rollback-reliability/tasks.md +1110 -0
- data/specs/009-distributed-map-undo/checklists/requirements.md +53 -0
- data/specs/009-distributed-map-undo/contracts/api-surface.md +125 -0
- data/specs/009-distributed-map-undo/contracts/rollback-protocol.md +157 -0
- data/specs/009-distributed-map-undo/data-model.md +151 -0
- data/specs/009-distributed-map-undo/plan.md +323 -0
- data/specs/009-distributed-map-undo/pr-description.md +78 -0
- data/specs/009-distributed-map-undo/quickstart.md +154 -0
- data/specs/009-distributed-map-undo/research.md +585 -0
- data/specs/009-distributed-map-undo/spec.md +360 -0
- data/specs/009-distributed-map-undo/tasks.md +1060 -0
- data/specs/future_improvements.md +64 -28
- metadata +54 -3
- data/lib/ruby_reactor/web/public/assets/index-BQvIWPdx.css +0 -1
|
@@ -80,12 +80,18 @@ module RubyReactor
|
|
|
80
80
|
# it, not this one. (`async_reactor` still resolves here: its resolved
|
|
81
81
|
# values are the child's INPUTS, needed at dispatch.)
|
|
82
82
|
deferred_body = step_config.async_dispatch == :step || handoff_at?(step_config, :before)
|
|
83
|
-
resolved_arguments =
|
|
83
|
+
resolved_arguments, resolution_error = resolve_before_start(step_config, deferred_body)
|
|
84
84
|
|
|
85
85
|
@middlewares.on(:start_step, step_config.name, resolved_arguments, @context)
|
|
86
86
|
completed = false
|
|
87
87
|
begin
|
|
88
|
-
|
|
88
|
+
# A step whose arguments could not be resolved fails like any step
|
|
89
|
+
# (`:failed_step`, attribution) but never started: the result
|
|
90
|
+
# handler undoes the completed steps and does not compensate it.
|
|
91
|
+
result = if resolution_error
|
|
92
|
+
@result_handler.handle_step_result(step_config,
|
|
93
|
+
pre_body_failure(step_config, resolution_error), {})
|
|
94
|
+
elsif step_config.interrupt?
|
|
89
95
|
handle_interrupt_step(step_config)
|
|
90
96
|
elsif step_config.async_dispatch == :step
|
|
91
97
|
dispatch_async_step(step_config)
|
|
@@ -113,6 +119,7 @@ module RubyReactor
|
|
|
113
119
|
unless completed
|
|
114
120
|
event = e.is_a?(Error::ExecutionParked) ? :snooze_step : :failed_step
|
|
115
121
|
@middlewares.on(event, step_config.name, e, @context)
|
|
122
|
+
track_interrupted_construct(step_config, resolved_arguments, e)
|
|
116
123
|
end
|
|
117
124
|
raise
|
|
118
125
|
end
|
|
@@ -120,6 +127,30 @@ module RubyReactor
|
|
|
120
127
|
|
|
121
128
|
private
|
|
122
129
|
|
|
130
|
+
# An interruption leaves a caller-process run `aborted` for a manual
|
|
131
|
+
# undo, which replays the undo stack. A compose or map cut short already
|
|
132
|
+
# holds completed child work that only its own undo reverts, so it joins
|
|
133
|
+
# the stack. A worker run is redelivered instead, and re-runs the step.
|
|
134
|
+
def track_interrupted_construct(step_config, arguments, error)
|
|
135
|
+
return if @context.inline_async_execution || Error::Rescuable === error # rubocop:disable Style/CaseEquality
|
|
136
|
+
return if error.is_a?(Error::RollbackHandedOff) # a rollback signal, not an interruption
|
|
137
|
+
return unless step_config.undoes_partial_run?
|
|
138
|
+
|
|
139
|
+
@compensation_manager.add_to_undo_stack({ step: step_config,
|
|
140
|
+
arguments: step_config.rollback_arguments(arguments),
|
|
141
|
+
result: RubyReactor.Success(nil) })
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# `[arguments, nil]`, or `[{}, ArgumentResolutionError]` for the caller
|
|
145
|
+
# to fail the step with once its `:start_step` has fired.
|
|
146
|
+
def resolve_before_start(step_config, deferred_body)
|
|
147
|
+
return [{}, nil] if deferred_body
|
|
148
|
+
|
|
149
|
+
[step_config.resolve_arguments(@context), nil]
|
|
150
|
+
rescue Error::ArgumentResolutionError => e
|
|
151
|
+
[{}, e]
|
|
152
|
+
end
|
|
153
|
+
|
|
123
154
|
# The reactor's single hand-off point, `{ mode: :after|:before, step: }`.
|
|
124
155
|
# Nil for a reactor that never declares `background`.
|
|
125
156
|
def background_handoff
|
|
@@ -130,23 +161,20 @@ module RubyReactor
|
|
|
130
161
|
end
|
|
131
162
|
|
|
132
163
|
# Hand-off is keyed to REACHING the named step, not to where the
|
|
133
|
-
# declaration sits in the class body.
|
|
134
|
-
# must not run never triggers it — the hand-off only ever relocates work
|
|
135
|
-
# that is actually going to happen. Inside the worker the whole thing is
|
|
164
|
+
# declaration sits in the class body. Inside the worker the whole thing is
|
|
136
165
|
# suppressed (`inline_async_execution`) so it cannot re-trigger.
|
|
137
166
|
def handoff_at?(step_config, mode)
|
|
138
167
|
point = background_handoff
|
|
139
|
-
|
|
140
|
-
return false if @context.inline_async_execution
|
|
141
|
-
|
|
142
|
-
step_config.should_run?(@context)
|
|
168
|
+
point && point[:mode] == mode && point[:step] == step_config.name && !@context.inline_async_execution
|
|
143
169
|
end
|
|
144
170
|
|
|
145
|
-
# Post-execution trigger for `after:`. Only a
|
|
146
|
-
#
|
|
147
|
-
#
|
|
171
|
+
# Post-execution trigger for `after:`. Only a result the run continues
|
|
172
|
+
# from hands off: a Success, or a Skipped, which is only an
|
|
173
|
+
# instrumentation mark and never changes execution. A Halt stops the run,
|
|
174
|
+
# and a Failure, interrupt, queued retry or already-async result each own
|
|
175
|
+
# the flow instead.
|
|
148
176
|
def handoff_after?(step_config, result)
|
|
149
|
-
return false unless result.is_a?(RubyReactor::Success) && !result.is_a?(RubyReactor::
|
|
177
|
+
return false unless result.is_a?(RubyReactor::Success) && !result.is_a?(RubyReactor::Halt)
|
|
150
178
|
|
|
151
179
|
handoff_at?(step_config, :after)
|
|
152
180
|
end
|
|
@@ -175,7 +203,7 @@ module RubyReactor
|
|
|
175
203
|
end
|
|
176
204
|
|
|
177
205
|
def execute_step_with_retry(step_config, resolved_arguments = nil)
|
|
178
|
-
resolved_arguments ||= resolve_arguments(
|
|
206
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
179
207
|
result = @retry_manager.execute_with_retry(step_config, @reactor_class) do
|
|
180
208
|
safe_execute_step_sync(step_config, resolved_arguments)
|
|
181
209
|
end
|
|
@@ -188,7 +216,7 @@ module RubyReactor
|
|
|
188
216
|
end
|
|
189
217
|
|
|
190
218
|
def safe_execute_step_sync(step_config, resolved_arguments = nil)
|
|
191
|
-
resolved_arguments ||= resolve_arguments(
|
|
219
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
192
220
|
execute_step_sync_without_result_handling(step_config, resolved_arguments)
|
|
193
221
|
rescue Error::InputValidationError => e
|
|
194
222
|
# Validation failures are not retryable and must surface as a structured
|
|
@@ -216,10 +244,12 @@ module RubyReactor
|
|
|
216
244
|
rescue Error::ExecutionParked
|
|
217
245
|
@context.retry_context.decrement_attempt_for_step(step_config.name)
|
|
218
246
|
raise
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
247
|
+
# Arguments raised: the body never started. Not retried (the same
|
|
248
|
+
# inputs fail the same way), never compensated.
|
|
249
|
+
rescue Error::ArgumentResolutionError => e
|
|
250
|
+
pre_body_failure(step_config, e)
|
|
251
|
+
# Any other exception, standard or not, is this step's failure (008 R-16).
|
|
252
|
+
rescue Error::Rescuable => e
|
|
223
253
|
RubyReactor::Failure(
|
|
224
254
|
e,
|
|
225
255
|
step_name: step_config.name,
|
|
@@ -230,6 +260,16 @@ module RubyReactor
|
|
|
230
260
|
)
|
|
231
261
|
end
|
|
232
262
|
|
|
263
|
+
def pre_body_failure(step_config, error)
|
|
264
|
+
RubyReactor::Failure(error, step_name: step_config.name, reactor_name: @reactor_class.name,
|
|
265
|
+
inputs: @context.inputs, redact_inputs: redact_inputs, step_arguments: {},
|
|
266
|
+
retryable: false, exception_class: error.exception_class)
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def redact_inputs
|
|
270
|
+
@reactor_class.inputs.select { |_, config| config[:redact] }.keys
|
|
271
|
+
end
|
|
272
|
+
|
|
233
273
|
# Contention (US3). Synchronously there is no queue to park into: the
|
|
234
274
|
# step fails. In a worker the execution parks at this step — by raising
|
|
235
275
|
# `Error::StepContentionPark`, which every executor on the stack lets
|
|
@@ -315,14 +355,8 @@ module RubyReactor
|
|
|
315
355
|
|
|
316
356
|
def execute_step_sync(step_config, resolved_arguments = nil)
|
|
317
357
|
@context.with_step(step_config.name) do
|
|
318
|
-
# Check conditions and guards
|
|
319
|
-
unless step_config.should_run?(@context)
|
|
320
|
-
@dependency_graph.complete_step(step_config.name)
|
|
321
|
-
return RubyReactor.Success(nil)
|
|
322
|
-
end
|
|
323
|
-
|
|
324
358
|
# Resolve arguments
|
|
325
|
-
resolved_arguments ||= resolve_arguments(
|
|
359
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
326
360
|
|
|
327
361
|
# Validate arguments if validator is defined
|
|
328
362
|
validate_step_arguments(step_config, resolved_arguments)
|
|
@@ -338,14 +372,8 @@ module RubyReactor
|
|
|
338
372
|
# Execute step without handling the result (used during retries)
|
|
339
373
|
def execute_step_sync_without_result_handling(step_config, resolved_arguments = nil)
|
|
340
374
|
@context.with_step(step_config.name) do
|
|
341
|
-
# Check conditions and guards
|
|
342
|
-
unless step_config.should_run?(@context)
|
|
343
|
-
@dependency_graph.complete_step(step_config.name)
|
|
344
|
-
return RubyReactor.Success(nil)
|
|
345
|
-
end
|
|
346
|
-
|
|
347
375
|
# Resolve arguments
|
|
348
|
-
resolved_arguments ||= resolve_arguments(
|
|
376
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
349
377
|
|
|
350
378
|
yield resolved_arguments if block_given?
|
|
351
379
|
|
|
@@ -435,22 +463,6 @@ module RubyReactor
|
|
|
435
463
|
raise error
|
|
436
464
|
end
|
|
437
465
|
|
|
438
|
-
def resolve_arguments(step_config)
|
|
439
|
-
resolved = {}
|
|
440
|
-
|
|
441
|
-
step_config.arguments.each do |arg_name, arg_config|
|
|
442
|
-
source = arg_config[:source]
|
|
443
|
-
transform = arg_config[:transform]
|
|
444
|
-
|
|
445
|
-
value = source.resolve(@context)
|
|
446
|
-
value = transform.call(value) if transform
|
|
447
|
-
|
|
448
|
-
resolved[arg_name] = value
|
|
449
|
-
end
|
|
450
|
-
|
|
451
|
-
resolved
|
|
452
|
-
end
|
|
453
|
-
|
|
454
466
|
def run_step_implementation(step_config, arguments)
|
|
455
467
|
contract = step_config.input_contract
|
|
456
468
|
@context.append_execution_trace(
|
|
@@ -135,6 +135,10 @@ module RubyReactor
|
|
|
135
135
|
handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
|
|
136
136
|
completed = true
|
|
137
137
|
@result
|
|
138
|
+
rescue Error::RollbackHandedOff => e
|
|
139
|
+
record_rollback_handoff(e)
|
|
140
|
+
completed = true
|
|
141
|
+
@result
|
|
138
142
|
rescue RubyReactor::Lock::AcquisitionError,
|
|
139
143
|
RubyReactor::Semaphore::AcquisitionError,
|
|
140
144
|
RubyReactor::RateLimit::ExceededError,
|
|
@@ -152,11 +156,14 @@ module RubyReactor
|
|
|
152
156
|
park_held_primitives! if @context.inline_async_execution
|
|
153
157
|
@contention_snooze = true
|
|
154
158
|
raise
|
|
155
|
-
rescue
|
|
156
|
-
@result = @result_handler.handle_execution_error(e)
|
|
159
|
+
rescue Error::Rescuable => e
|
|
160
|
+
@result = handing_off { aborting_on_interruption { @result_handler.handle_execution_error(e) } }
|
|
157
161
|
update_context_status(@result)
|
|
158
162
|
completed = true
|
|
159
163
|
@result
|
|
164
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
165
|
+
mark_aborted
|
|
166
|
+
raise
|
|
160
167
|
ensure
|
|
161
168
|
release_locks unless @parked
|
|
162
169
|
leave_ordered_lock_scope
|
|
@@ -181,6 +188,9 @@ module RubyReactor
|
|
|
181
188
|
end
|
|
182
189
|
|
|
183
190
|
def resume_execution # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity
|
|
191
|
+
# A composed child re-entered while its own rollback is handed off.
|
|
192
|
+
return resume_rollback if @context.rolling_back?
|
|
193
|
+
|
|
184
194
|
middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
|
|
185
195
|
completed = false
|
|
186
196
|
|
|
@@ -239,6 +249,7 @@ module RubyReactor
|
|
|
239
249
|
@context.admit!
|
|
240
250
|
prepare_for_resume
|
|
241
251
|
save_context
|
|
252
|
+
@past_gates = true
|
|
242
253
|
|
|
243
254
|
@result = if @context.current_step
|
|
244
255
|
execute_current_step_and_continue
|
|
@@ -252,6 +263,10 @@ module RubyReactor
|
|
|
252
263
|
handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
|
|
253
264
|
completed = true
|
|
254
265
|
@result
|
|
266
|
+
rescue Error::RollbackHandedOff => e
|
|
267
|
+
record_rollback_handoff(e)
|
|
268
|
+
completed = true
|
|
269
|
+
@result
|
|
255
270
|
rescue RubyReactor::Lock::AcquisitionError,
|
|
256
271
|
RubyReactor::Semaphore::AcquisitionError,
|
|
257
272
|
RubyReactor::RateLimit::ExceededError,
|
|
@@ -269,23 +284,145 @@ module RubyReactor
|
|
|
269
284
|
park_held_primitives!
|
|
270
285
|
@contention_snooze = true
|
|
271
286
|
raise e
|
|
272
|
-
rescue
|
|
273
|
-
handle_resume_error(e)
|
|
287
|
+
rescue Error::Rescuable => e
|
|
288
|
+
handing_off { aborting_on_interruption { handle_resume_error(e) } }
|
|
274
289
|
update_context_status(@result)
|
|
275
290
|
completed = true
|
|
276
291
|
@result
|
|
292
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
293
|
+
mark_aborted
|
|
294
|
+
raise
|
|
277
295
|
ensure
|
|
278
296
|
release_locks unless @parked
|
|
297
|
+
# Saved while the context lock is still held (009 R-13): released first,
|
|
298
|
+
# a worker resuming this run could load the pre-save blob, and this save
|
|
299
|
+
# would then overwrite its progress.
|
|
300
|
+
save_context unless skip_context_persist?
|
|
279
301
|
@acquired_context_lock&.release
|
|
280
302
|
@acquired_context_lock = nil
|
|
281
303
|
leave_ordered_lock_scope
|
|
304
|
+
|
|
305
|
+
emit_lifecycle_completion(completed)
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Undoes every completed step (manual undo, a composed child's undo, a
|
|
309
|
+
# map element's rollback). A rollback that hands off here (009 R-04)
|
|
310
|
+
# leaves this level's rollback failures on its context, and the next
|
|
311
|
+
# `undo_all` of the same context picks them up, so none is lost though
|
|
312
|
+
# the executor is rebuilt on resume (G2).
|
|
313
|
+
def undo_all(&after_pop)
|
|
314
|
+
saved = @context.rollback && @context.rollback["failures"]
|
|
315
|
+
@compensation_manager.restore_rollback_failures(saved) if saved
|
|
316
|
+
@compensation_manager.rollback_completed_steps(&after_pop)
|
|
317
|
+
clear_saved_rollback_failures
|
|
318
|
+
rescue Error::RollbackHandedOff
|
|
319
|
+
(@context.rollback ||= {})["failures"] =
|
|
320
|
+
ContextSerializer.serialize_value(@compensation_manager.rollback_failures)
|
|
321
|
+
raise
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# Finishes a rollback that handed off at a fan-out map (009 R-04), once
|
|
325
|
+
# the map's element rollbacks have all reported: the Worker of a
|
|
326
|
+
# `rolling_back` run, or a composed child re-entered by its root. Under
|
|
327
|
+
# the run's context lock, as `resume_execution`.
|
|
328
|
+
def resume_rollback
|
|
329
|
+
state = @context.rollback || {}
|
|
330
|
+
# The hand-off came out of a running step (a composed child rolling
|
|
331
|
+
# back inside `ComposeStep#run`): resume forward, and that step
|
|
332
|
+
# finishes its child's rollback.
|
|
333
|
+
if state["trigger"] == "step" || state.empty?
|
|
334
|
+
@context.rollback = nil
|
|
335
|
+
@context.status = :running
|
|
336
|
+
return resume_execution
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
|
|
340
|
+
completed = false
|
|
341
|
+
enter_ordered_lock_scope
|
|
342
|
+
acquire_context_lock
|
|
343
|
+
@result = finish_rollback(state)
|
|
344
|
+
@context.rollback = nil
|
|
345
|
+
update_context_status(@result)
|
|
346
|
+
completed = true
|
|
347
|
+
@result
|
|
348
|
+
rescue Error::RollbackHandedOff => e
|
|
349
|
+
record_rollback_handoff(e)
|
|
350
|
+
completed = true
|
|
351
|
+
@result
|
|
352
|
+
rescue RubyReactor::Lock::AcquisitionError => e
|
|
353
|
+
@contention_snooze = true
|
|
354
|
+
raise composed_contention_park(e) || e
|
|
355
|
+
rescue Error::Rescuable => e
|
|
356
|
+
handing_off { aborting_on_interruption { handle_resume_error(e) } }
|
|
357
|
+
update_context_status(@result)
|
|
358
|
+
completed = true
|
|
359
|
+
@result
|
|
360
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
361
|
+
mark_aborted
|
|
362
|
+
raise
|
|
363
|
+
ensure
|
|
282
364
|
save_context unless skip_context_persist?
|
|
365
|
+
@acquired_context_lock&.release
|
|
366
|
+
@acquired_context_lock = nil
|
|
367
|
+
leave_ordered_lock_scope
|
|
283
368
|
|
|
284
369
|
emit_lifecycle_completion(completed)
|
|
285
370
|
end
|
|
286
371
|
|
|
287
|
-
|
|
288
|
-
|
|
372
|
+
# The rollback's other half (009 R-05): marks the hand-off saved, then
|
|
373
|
+
# enqueues the owner's Worker if the map rollback already settled and this
|
|
374
|
+
# side claims the signal. Whichever of this and the last element rollback
|
|
375
|
+
# job sees both second enqueues the owner, once. Called right after the
|
|
376
|
+
# `rolling_back` save, while the context lock is still held.
|
|
377
|
+
def hand_off_rollback!(handed_off)
|
|
378
|
+
storage = RubyReactor.configuration.storage_adapter
|
|
379
|
+
map_id = handed_off.map_id
|
|
380
|
+
map_class = handed_off.reactor_class_name
|
|
381
|
+
storage.mark_map_rollback_handed_off(map_id, map_class)
|
|
382
|
+
RubyReactor.configuration.logger.info(
|
|
383
|
+
"event=ruby_reactor.rollback.handed_off context_id=#{@context.context_id.inspect} map_id=#{map_id.inspect}"
|
|
384
|
+
)
|
|
385
|
+
meta = storage.retrieve_map_rollback_metadata(map_id, map_class)
|
|
386
|
+
return unless meta && storage.count_map_rollback_outcomes(map_id, map_class) >= meta["total"].to_i
|
|
387
|
+
return unless storage.claim_map_rollback_signal(map_id, map_class)
|
|
388
|
+
|
|
389
|
+
RubyReactor.configuration.async_router.perform_async(@context.context_id,
|
|
390
|
+
RubyReactor.reactor_storage_name(@reactor_class))
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
# True once `resume_execution` is past its reactor-level gates (context
|
|
394
|
+
# lock, lock, semaphore, period): from there on it may have run steps.
|
|
395
|
+
def past_gates?
|
|
396
|
+
@past_gates == true
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
# Reached only by an interruption — a signal, an exit, out of memory, an
|
|
400
|
+
# enclosing timeout (008 R-08, R-16); every other exception was rescued as
|
|
401
|
+
# `Error::Rescuable` and rolled back. Running user rollback code now is
|
|
402
|
+
# unsafe, so none runs: a run in the caller's process is recorded
|
|
403
|
+
# `aborted`, with the undo entries not yet undone kept, for a manual
|
|
404
|
+
# `Reactor#undo`; the `ensure` persists it, best effort. A worker run stays
|
|
405
|
+
# `running` and its job is redelivered.
|
|
406
|
+
def mark_aborted
|
|
407
|
+
@context.status = :aborted unless @context.inline_async_execution
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
# A rollback run from a `rescue Error::Rescuable` body is outside the
|
|
411
|
+
# sibling `rescue Exception`, which never sees what that body raises: an
|
|
412
|
+
# interruption there must mark the run here.
|
|
413
|
+
def aborting_on_interruption
|
|
414
|
+
yield
|
|
415
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
416
|
+
mark_aborted unless Error::Rescuable === e || e.is_a?(Error::RollbackHandedOff) # rubocop:disable Style/CaseEquality
|
|
417
|
+
raise
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
# A rollback run from a `rescue Error::Rescuable` body (the error's own
|
|
421
|
+
# rollback) is outside the sibling `rescue Error::RollbackHandedOff`.
|
|
422
|
+
def handing_off
|
|
423
|
+
yield
|
|
424
|
+
rescue Error::RollbackHandedOff => e
|
|
425
|
+
record_rollback_handoff(e)
|
|
289
426
|
end
|
|
290
427
|
|
|
291
428
|
def undo_stack
|
|
@@ -734,7 +871,8 @@ module RubyReactor
|
|
|
734
871
|
|
|
735
872
|
case result
|
|
736
873
|
when RubyReactor::DispatchResult
|
|
737
|
-
|
|
874
|
+
# A rollback hand-off returns one too, and stays `rolling_back`.
|
|
875
|
+
@context.status = :running unless @context.rolling_back?
|
|
738
876
|
when RubyReactor::Halt
|
|
739
877
|
@context.status = :halted
|
|
740
878
|
when RubyReactor::Success
|
|
@@ -799,6 +937,82 @@ module RubyReactor
|
|
|
799
937
|
@result
|
|
800
938
|
end
|
|
801
939
|
|
|
940
|
+
# Where this level's rollback stands when it hands off (009 DM §1). A
|
|
941
|
+
# composed child keeps it on its own context, inside the root's blob, and
|
|
942
|
+
# re-raises; the top-level run saves it as `rolling_back` while it still
|
|
943
|
+
# holds the context lock (I-6), runs the handshake, and returns a
|
|
944
|
+
# `DispatchResult`, as a forward hand-off does.
|
|
945
|
+
def record_rollback_handoff(handed_off)
|
|
946
|
+
failure = handed_off.failure
|
|
947
|
+
handed_off.failure = nil # this level's Failure; the level above builds its own
|
|
948
|
+
@context.rollback = rollback_state(failure)
|
|
949
|
+
@context.status = :rolling_back
|
|
950
|
+
raise handed_off if @context.root_context
|
|
951
|
+
|
|
952
|
+
save_context
|
|
953
|
+
hand_off_rollback!(handed_off)
|
|
954
|
+
@result = RubyReactor::DispatchResult.new(job_id: "map_rollback:#{handed_off.map_id}",
|
|
955
|
+
intermediate_results: @context.intermediate_results,
|
|
956
|
+
execution_id: @context.context_id)
|
|
957
|
+
end
|
|
958
|
+
|
|
959
|
+
# `trigger`: `failure` (a step failure is rolling back), `undo` (set by
|
|
960
|
+
# `Reactor#undo`), or `step` (a running step's child handed off, and no
|
|
961
|
+
# rollback started at this level). A later hand-off of the same rollback
|
|
962
|
+
# keeps what an earlier one recorded.
|
|
963
|
+
def rollback_state(failure)
|
|
964
|
+
state = (@context.rollback || {}).dup
|
|
965
|
+
pending = @compensation_manager.pending
|
|
966
|
+
state["trigger"] ||= failure || pending ? "failure" : "step"
|
|
967
|
+
if pending
|
|
968
|
+
state["step"] = pending[:step].to_s
|
|
969
|
+
state["compensated"] = pending[:compensated]
|
|
970
|
+
state["compensation_error"] = pending[:compensation_error]
|
|
971
|
+
end
|
|
972
|
+
state["failure"] = ContextSerializer.serialize_value(failure) if failure
|
|
973
|
+
state["failures"] = ContextSerializer.serialize_value(@compensation_manager.rollback_failures)
|
|
974
|
+
state
|
|
975
|
+
end
|
|
976
|
+
|
|
977
|
+
# The rest of a handed-off rollback: the failing step's own compensate if
|
|
978
|
+
# it had not finished (a map or compose adopts its settled state), the
|
|
979
|
+
# steps still on the undo stack, then the outcome the inline rollback
|
|
980
|
+
# gives (I-7).
|
|
981
|
+
def finish_rollback(state)
|
|
982
|
+
@compensation_manager.restore_rollback_failures(state["failures"])
|
|
983
|
+
return finish_undo if state["trigger"] == "undo"
|
|
984
|
+
|
|
985
|
+
failure = ContextSerializer.deserialize_value(state["failure"])
|
|
986
|
+
step_config = state["step"] && @reactor_class.steps[state["step"].to_sym]
|
|
987
|
+
if step_config && !state["compensated"]
|
|
988
|
+
@compensation_manager.handle_step_failure(step_config, failure.error, {})
|
|
989
|
+
else
|
|
990
|
+
@compensation_manager.rollback_completed_steps
|
|
991
|
+
if state["compensation_error"]
|
|
992
|
+
raise Error::CompensationError.new(state["compensation_error"], step: step_config&.name, context: @context)
|
|
993
|
+
end
|
|
994
|
+
end
|
|
995
|
+
@context.current_step = failure.step_name&.to_sym
|
|
996
|
+
@result_handler.handed_off_failure(failure)
|
|
997
|
+
rescue Error::CompensationError => e
|
|
998
|
+
@result_handler.handle_execution_error(e)
|
|
999
|
+
end
|
|
1000
|
+
|
|
1001
|
+
def finish_undo
|
|
1002
|
+
@compensation_manager.rollback_completed_steps
|
|
1003
|
+
@context.cancelled = true
|
|
1004
|
+
@context.cancellation_reason = "Undo triggered"
|
|
1005
|
+
@context.status = "cancelled"
|
|
1006
|
+
nil
|
|
1007
|
+
end
|
|
1008
|
+
|
|
1009
|
+
def clear_saved_rollback_failures
|
|
1010
|
+
return unless @context.rollback
|
|
1011
|
+
|
|
1012
|
+
@context.rollback.delete("failures")
|
|
1013
|
+
@context.rollback = nil if @context.rollback.empty?
|
|
1014
|
+
end
|
|
1015
|
+
|
|
802
1016
|
def handle_interrupt(interrupt_result)
|
|
803
1017
|
save_context
|
|
804
1018
|
|
|
@@ -3,10 +3,14 @@
|
|
|
3
3
|
module RubyReactor
|
|
4
4
|
module Map
|
|
5
5
|
class Collector
|
|
6
|
-
|
|
6
|
+
# Seconds a collector waits for another collector of the same map. One
|
|
7
|
+
# that found the map unsettled releases within milliseconds, and may be
|
|
8
|
+
# the only trigger left for a failure it just deferred (R-04); a holder
|
|
9
|
+
# busy longer is the one applying the map's result.
|
|
10
|
+
COLLECT_LOCK_WAIT = 2
|
|
7
11
|
|
|
8
12
|
def self.perform(arguments)
|
|
9
|
-
arguments =
|
|
13
|
+
arguments = Helpers.normalize_arguments(arguments)
|
|
10
14
|
map_id = arguments[:map_id]
|
|
11
15
|
|
|
12
16
|
# Serialize concurrent collector deliveries for the SAME map (eager queue +
|
|
@@ -30,7 +34,7 @@ module RubyReactor
|
|
|
30
34
|
lock = RubyReactor::Lock.new(
|
|
31
35
|
"map_collect:#{map_id}",
|
|
32
36
|
owner: SecureRandom.uuid, ttl: RubyReactor.configuration.context_lock_ttl,
|
|
33
|
-
wait:
|
|
37
|
+
wait: COLLECT_LOCK_WAIT, auto_extend: true
|
|
34
38
|
)
|
|
35
39
|
lock.acquire
|
|
36
40
|
lock
|
|
@@ -42,101 +46,39 @@ module RubyReactor
|
|
|
42
46
|
defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
|
|
43
47
|
end
|
|
44
48
|
|
|
49
|
+
# Once every index has a result slot (a value, `_error`, `_halt` or
|
|
50
|
+
# `_skipped`), signal the map's OWNER run — the top-level context —
|
|
51
|
+
# once, and write nothing else (009 R-03, I-1). The owner's Worker
|
|
52
|
+
# resumes, and `MapStep#run` adopts the settled outcome at any
|
|
53
|
+
# composition depth. An atomic map's failure is applied the same way
|
|
54
|
+
# only once every index has settled, so its compensate sees every
|
|
55
|
+
# element that completed, including ones still in flight when it
|
|
56
|
+
# failed. Until then the last element to settle, or the map sweeper,
|
|
57
|
+
# re-triggers this collector.
|
|
45
58
|
def self.perform_collection(arguments)
|
|
46
59
|
map_id = arguments[:map_id]
|
|
47
|
-
parent_context_id = arguments[:parent_context_id]
|
|
48
60
|
parent_reactor_class_name = arguments[:parent_reactor_class_name]
|
|
49
|
-
step_name = arguments[:step_name]
|
|
50
|
-
strict_ordering = arguments[:strict_ordering]
|
|
51
|
-
# timeout = arguments[:timeout]
|
|
52
|
-
|
|
53
61
|
storage = RubyReactor.configuration.storage_adapter
|
|
54
|
-
parent_context_data = storage.retrieve_context(parent_context_id, parent_reactor_class_name)
|
|
55
|
-
parent_context = RubyReactor::Context.deserialize_from_retry(parent_context_data)
|
|
56
|
-
|
|
57
|
-
# Idempotency: if the parent already recorded this map step's result, a
|
|
58
|
-
# prior collector already resumed it. Re-resuming would double-execute the
|
|
59
|
-
# steps after the map. Skip.
|
|
60
|
-
return if parent_context.intermediate_results.key?(step_name.to_sym)
|
|
61
62
|
|
|
62
|
-
# Check if all tasks are completed
|
|
63
63
|
metadata = storage.retrieve_map_metadata(map_id, parent_reactor_class_name)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
# Actually Collector currently assumes we only call it when we expect completion or check progress
|
|
70
|
-
# Since map_offset tracks dispatching progress and might exceed count due to batching reservation,
|
|
71
|
-
# we must strictly check against the total count of elements.
|
|
72
|
-
# Check for fail_fast failure FIRST
|
|
73
|
-
if (failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name))
|
|
74
|
-
handle_failure(failed_context_id, metadata, storage, parent_context, step_name)
|
|
75
|
-
return
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
return if results_count < total_count
|
|
79
|
-
|
|
80
|
-
# Retrieve results lazily
|
|
81
|
-
results = RubyReactor::Map::ResultEnumerator.new(
|
|
82
|
-
map_id,
|
|
83
|
-
parent_reactor_class_name,
|
|
84
|
-
strict_ordering: strict_ordering
|
|
85
|
-
)
|
|
86
|
-
|
|
87
|
-
# Apply collect block (or default collection)
|
|
88
|
-
step_config = parent_context.reactor_class.steps[step_name.to_sym]
|
|
89
|
-
|
|
90
|
-
begin
|
|
91
|
-
final_result = apply_collect_block(results, step_config)
|
|
92
|
-
|
|
93
|
-
if final_result.failure?
|
|
94
|
-
# Optionally log failure internally or just rely on context status update
|
|
95
|
-
end
|
|
96
|
-
rescue StandardError => e
|
|
97
|
-
final_result = RubyReactor::Failure(e)
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
# Resume parent execution
|
|
101
|
-
resume_parent_execution(parent_context, step_name, final_result, storage)
|
|
102
|
-
rescue StandardError => e
|
|
103
|
-
RubyReactor.configuration.logger.error("Map collector crashed: #{e.message}")
|
|
104
|
-
RubyReactor.configuration.logger.error(e.backtrace.join("\n")) if e.backtrace
|
|
105
|
-
raise e
|
|
106
|
-
end
|
|
64
|
+
return unless metadata
|
|
65
|
+
# Judged against the element count, not map_offset (which batching
|
|
66
|
+
# reservation can push past it).
|
|
67
|
+
return if storage.count_map_results(map_id, parent_reactor_class_name) < metadata["count"].to_i
|
|
68
|
+
return unless storage.claim_map_owner_signal(map_id, parent_reactor_class_name)
|
|
107
69
|
|
|
108
|
-
|
|
109
|
-
collect_block = step_config.arguments[:collect_block][:source].value if step_config.arguments[:collect_block]
|
|
110
|
-
# TODO: Check allow_partial_failure option
|
|
111
|
-
|
|
112
|
-
if collect_block
|
|
113
|
-
begin
|
|
114
|
-
# Pass Enumerator to collect block
|
|
115
|
-
collected = collect_block.call(results)
|
|
116
|
-
RubyReactor::Success(collected)
|
|
117
|
-
rescue StandardError => e
|
|
118
|
-
RubyReactor.configuration.logger.error("Map collect block raised: #{e.message}")
|
|
119
|
-
RubyReactor.configuration.logger.error(e.backtrace.join("\n")) if e.backtrace
|
|
120
|
-
RubyReactor::Failure(e)
|
|
121
|
-
end
|
|
122
|
-
else
|
|
123
|
-
# Default behavior: Return Success(Enumerator).
|
|
124
|
-
# Logic for checking failures is deferred to the consumer of the enumerator.
|
|
125
|
-
RubyReactor::Success(results)
|
|
126
|
-
end
|
|
70
|
+
signal_owner(metadata, arguments)
|
|
127
71
|
end
|
|
128
72
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
result = reason.is_a?(RubyReactor::Failure) ? reason : RubyReactor::Failure(reason)
|
|
139
|
-
resume_parent_execution(parent_context, step_name, result, storage)
|
|
73
|
+
# Metadata written before the upgrade names no owner (S-6): the parent is
|
|
74
|
+
# the owner of a root-level map.
|
|
75
|
+
def self.signal_owner(metadata, arguments)
|
|
76
|
+
owner_id = metadata["owner_context_id"] || metadata["parent_context_id"] || arguments[:parent_context_id]
|
|
77
|
+
owner_class = metadata["owner_reactor_class_name"] || metadata["parent_reactor_class_name"]
|
|
78
|
+
RubyReactor.configuration.logger.info(
|
|
79
|
+
"event=ruby_reactor.map.settled map_id=#{arguments[:map_id].inspect} owner=#{owner_id.inspect}"
|
|
80
|
+
)
|
|
81
|
+
RubyReactor.configuration.async_router.perform_async(owner_id, owner_class)
|
|
140
82
|
end
|
|
141
83
|
end
|
|
142
84
|
end
|