ruby_reactor 0.8.4 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/CHANGELOG.md +196 -0
- data/CLAUDE.md +1 -1
- data/README.md +47 -11
- data/lib/ruby_reactor/dsl/async_macros.rb +30 -1
- data/lib/ruby_reactor/dsl/async_reactor_builder.rb +12 -6
- data/lib/ruby_reactor/dsl/compose_builder.rb +12 -6
- data/lib/ruby_reactor/dsl/interrupt_builder.rb +1 -3
- data/lib/ruby_reactor/dsl/map_builder.rb +0 -2
- data/lib/ruby_reactor/dsl/step_builder.rb +91 -19
- data/lib/ruby_reactor/error/argument_resolution_error.rb +19 -0
- data/lib/ruby_reactor/error/rescuable.rb +28 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +30 -26
- data/lib/ruby_reactor/executor/result_handler.rb +18 -16
- data/lib/ruby_reactor/executor/step_coordination.rb +11 -8
- data/lib/ruby_reactor/executor/step_executor.rb +59 -49
- data/lib/ruby_reactor/executor.rb +38 -4
- data/lib/ruby_reactor/map/collector.rb +21 -11
- data/lib/ruby_reactor/map/dispatcher.rb +29 -3
- data/lib/ruby_reactor/map/element_executor.rb +9 -3
- data/lib/ruby_reactor/map/helpers.rb +32 -2
- data/lib/ruby_reactor/map/result_enumerator.rb +18 -12
- data/lib/ruby_reactor/reactor.rb +24 -0
- data/lib/ruby_reactor/rspec/matchers.rb +19 -3
- data/lib/ruby_reactor/step/compose_step.rb +7 -1
- data/lib/ruby_reactor/step/map_step.rb +109 -4
- data/lib/ruby_reactor/step.rb +7 -0
- data/lib/ruby_reactor/step_worker.rb +46 -22
- data/lib/ruby_reactor/storage/adapter.rb +4 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +9 -0
- data/lib/ruby_reactor/storage/redis_reactor_scan.rb +1 -1
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +1 -1
- data/lib/ruby_reactor/web/public/assets/{index-CeZU-ESu.js → index-CQbgHtd0.js} +10 -10
- data/lib/ruby_reactor/web/public/index.html +1 -1
- data/lib/ruby_reactor/worker.rb +3 -1
- data/lib/ruby_reactor.rb +17 -6
- data/specs/007-execution-flow-analysis/analysis/README.md +147 -0
- data/specs/007-execution-flow-analysis/analysis/execution-order.md +359 -0
- data/specs/007-execution-flow-analysis/analysis/findings-and-options.md +502 -0
- data/specs/007-execution-flow-analysis/analysis/invariants.md +109 -0
- data/specs/007-execution-flow-analysis/checklists/requirements.md +39 -0
- data/specs/007-execution-flow-analysis/contracts/report-structure.md +71 -0
- data/specs/007-execution-flow-analysis/data-model.md +83 -0
- data/specs/007-execution-flow-analysis/evidence/harness.rb +229 -0
- data/specs/007-execution-flow-analysis/evidence/output.txt +333 -0
- data/specs/007-execution-flow-analysis/evidence/probes/01_plain.rb +122 -0
- data/specs/007-execution-flow-analysis/evidence/probes/02_compose.rb +182 -0
- data/specs/007-execution-flow-analysis/evidence/probes/03_map.rb +232 -0
- data/specs/007-execution-flow-analysis/evidence/probes/04_async.rb +132 -0
- data/specs/007-execution-flow-analysis/evidence/probes/05_background.rb +58 -0
- data/specs/007-execution-flow-analysis/evidence/probes/06_coordination.rb +158 -0
- data/specs/007-execution-flow-analysis/evidence/probes/07_interrupts_manual.rb +185 -0
- data/specs/007-execution-flow-analysis/evidence/run.rb +15 -0
- data/specs/007-execution-flow-analysis/plan.md +127 -0
- data/specs/007-execution-flow-analysis/quickstart.md +51 -0
- data/specs/007-execution-flow-analysis/research.md +202 -0
- data/specs/007-execution-flow-analysis/spec.md +270 -0
- data/specs/007-execution-flow-analysis/tasks.md +257 -0
- data/specs/008-rollback-reliability/checklists/requirements.md +43 -0
- data/specs/008-rollback-reliability/contracts/api-surface.md +126 -0
- data/specs/008-rollback-reliability/contracts/rollback-semantics.md +76 -0
- data/specs/008-rollback-reliability/data-model.md +139 -0
- data/specs/008-rollback-reliability/plan.md +233 -0
- data/specs/008-rollback-reliability/quickstart.md +105 -0
- data/specs/008-rollback-reliability/research.md +653 -0
- data/specs/008-rollback-reliability/spec.md +561 -0
- data/specs/008-rollback-reliability/tasks.md +1110 -0
- data/specs/future_improvements.md +48 -0
- metadata +35 -2
|
@@ -80,12 +80,18 @@ module RubyReactor
|
|
|
80
80
|
# it, not this one. (`async_reactor` still resolves here: its resolved
|
|
81
81
|
# values are the child's INPUTS, needed at dispatch.)
|
|
82
82
|
deferred_body = step_config.async_dispatch == :step || handoff_at?(step_config, :before)
|
|
83
|
-
resolved_arguments =
|
|
83
|
+
resolved_arguments, resolution_error = resolve_before_start(step_config, deferred_body)
|
|
84
84
|
|
|
85
85
|
@middlewares.on(:start_step, step_config.name, resolved_arguments, @context)
|
|
86
86
|
completed = false
|
|
87
87
|
begin
|
|
88
|
-
|
|
88
|
+
# A step whose arguments could not be resolved fails like any step
|
|
89
|
+
# (`:failed_step`, attribution) but never started: the result
|
|
90
|
+
# handler undoes the completed steps and does not compensate it.
|
|
91
|
+
result = if resolution_error
|
|
92
|
+
@result_handler.handle_step_result(step_config,
|
|
93
|
+
pre_body_failure(step_config, resolution_error), {})
|
|
94
|
+
elsif step_config.interrupt?
|
|
89
95
|
handle_interrupt_step(step_config)
|
|
90
96
|
elsif step_config.async_dispatch == :step
|
|
91
97
|
dispatch_async_step(step_config)
|
|
@@ -113,6 +119,7 @@ module RubyReactor
|
|
|
113
119
|
unless completed
|
|
114
120
|
event = e.is_a?(Error::ExecutionParked) ? :snooze_step : :failed_step
|
|
115
121
|
@middlewares.on(event, step_config.name, e, @context)
|
|
122
|
+
track_interrupted_construct(step_config, resolved_arguments, e)
|
|
116
123
|
end
|
|
117
124
|
raise
|
|
118
125
|
end
|
|
@@ -120,6 +127,28 @@ module RubyReactor
|
|
|
120
127
|
|
|
121
128
|
private
|
|
122
129
|
|
|
130
|
+
# An interruption leaves a caller-process run `aborted` for a manual
|
|
131
|
+
# undo, which replays the undo stack. A compose or map cut short already
|
|
132
|
+
# holds completed child work that only its own undo reverts, so it joins
|
|
133
|
+
# the stack. A worker run is redelivered instead, and re-runs the step.
|
|
134
|
+
def track_interrupted_construct(step_config, arguments, error)
|
|
135
|
+
return if @context.inline_async_execution || Error::Rescuable === error # rubocop:disable Style/CaseEquality
|
|
136
|
+
return unless step_config.undoes_partial_run?
|
|
137
|
+
|
|
138
|
+
@compensation_manager.add_to_undo_stack({ step: step_config, arguments: arguments,
|
|
139
|
+
result: RubyReactor.Success(nil) })
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# `[arguments, nil]`, or `[{}, ArgumentResolutionError]` for the caller
|
|
143
|
+
# to fail the step with once its `:start_step` has fired.
|
|
144
|
+
def resolve_before_start(step_config, deferred_body)
|
|
145
|
+
return [{}, nil] if deferred_body
|
|
146
|
+
|
|
147
|
+
[step_config.resolve_arguments(@context), nil]
|
|
148
|
+
rescue Error::ArgumentResolutionError => e
|
|
149
|
+
[{}, e]
|
|
150
|
+
end
|
|
151
|
+
|
|
123
152
|
# The reactor's single hand-off point, `{ mode: :after|:before, step: }`.
|
|
124
153
|
# Nil for a reactor that never declares `background`.
|
|
125
154
|
def background_handoff
|
|
@@ -130,23 +159,20 @@ module RubyReactor
|
|
|
130
159
|
end
|
|
131
160
|
|
|
132
161
|
# Hand-off is keyed to REACHING the named step, not to where the
|
|
133
|
-
# declaration sits in the class body.
|
|
134
|
-
# must not run never triggers it — the hand-off only ever relocates work
|
|
135
|
-
# that is actually going to happen. Inside the worker the whole thing is
|
|
162
|
+
# declaration sits in the class body. Inside the worker the whole thing is
|
|
136
163
|
# suppressed (`inline_async_execution`) so it cannot re-trigger.
|
|
137
164
|
def handoff_at?(step_config, mode)
|
|
138
165
|
point = background_handoff
|
|
139
|
-
|
|
140
|
-
return false if @context.inline_async_execution
|
|
141
|
-
|
|
142
|
-
step_config.should_run?(@context)
|
|
166
|
+
point && point[:mode] == mode && point[:step] == step_config.name && !@context.inline_async_execution
|
|
143
167
|
end
|
|
144
168
|
|
|
145
|
-
# Post-execution trigger for `after:`. Only a
|
|
146
|
-
#
|
|
147
|
-
#
|
|
169
|
+
# Post-execution trigger for `after:`. Only a result the run continues
|
|
170
|
+
# from hands off: a Success, or a Skipped, which is only an
|
|
171
|
+
# instrumentation mark and never changes execution. A Halt stops the run,
|
|
172
|
+
# and a Failure, interrupt, queued retry or already-async result each own
|
|
173
|
+
# the flow instead.
|
|
148
174
|
def handoff_after?(step_config, result)
|
|
149
|
-
return false unless result.is_a?(RubyReactor::Success) && !result.is_a?(RubyReactor::
|
|
175
|
+
return false unless result.is_a?(RubyReactor::Success) && !result.is_a?(RubyReactor::Halt)
|
|
150
176
|
|
|
151
177
|
handoff_at?(step_config, :after)
|
|
152
178
|
end
|
|
@@ -175,7 +201,7 @@ module RubyReactor
|
|
|
175
201
|
end
|
|
176
202
|
|
|
177
203
|
def execute_step_with_retry(step_config, resolved_arguments = nil)
|
|
178
|
-
resolved_arguments ||= resolve_arguments(
|
|
204
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
179
205
|
result = @retry_manager.execute_with_retry(step_config, @reactor_class) do
|
|
180
206
|
safe_execute_step_sync(step_config, resolved_arguments)
|
|
181
207
|
end
|
|
@@ -188,7 +214,7 @@ module RubyReactor
|
|
|
188
214
|
end
|
|
189
215
|
|
|
190
216
|
def safe_execute_step_sync(step_config, resolved_arguments = nil)
|
|
191
|
-
resolved_arguments ||= resolve_arguments(
|
|
217
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
192
218
|
execute_step_sync_without_result_handling(step_config, resolved_arguments)
|
|
193
219
|
rescue Error::InputValidationError => e
|
|
194
220
|
# Validation failures are not retryable and must surface as a structured
|
|
@@ -216,10 +242,12 @@ module RubyReactor
|
|
|
216
242
|
rescue Error::ExecutionParked
|
|
217
243
|
@context.retry_context.decrement_attempt_for_step(step_config.name)
|
|
218
244
|
raise
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
245
|
+
# Arguments raised: the body never started. Not retried (the same
|
|
246
|
+
# inputs fail the same way), never compensated.
|
|
247
|
+
rescue Error::ArgumentResolutionError => e
|
|
248
|
+
pre_body_failure(step_config, e)
|
|
249
|
+
# Any other exception, standard or not, is this step's failure (008 R-16).
|
|
250
|
+
rescue Error::Rescuable => e
|
|
223
251
|
RubyReactor::Failure(
|
|
224
252
|
e,
|
|
225
253
|
step_name: step_config.name,
|
|
@@ -230,6 +258,16 @@ module RubyReactor
|
|
|
230
258
|
)
|
|
231
259
|
end
|
|
232
260
|
|
|
261
|
+
def pre_body_failure(step_config, error)
|
|
262
|
+
RubyReactor::Failure(error, step_name: step_config.name, reactor_name: @reactor_class.name,
|
|
263
|
+
inputs: @context.inputs, redact_inputs: redact_inputs, step_arguments: {},
|
|
264
|
+
retryable: false, exception_class: error.exception_class)
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def redact_inputs
|
|
268
|
+
@reactor_class.inputs.select { |_, config| config[:redact] }.keys
|
|
269
|
+
end
|
|
270
|
+
|
|
233
271
|
# Contention (US3). Synchronously there is no queue to park into: the
|
|
234
272
|
# step fails. In a worker the execution parks at this step — by raising
|
|
235
273
|
# `Error::StepContentionPark`, which every executor on the stack lets
|
|
@@ -315,14 +353,8 @@ module RubyReactor
|
|
|
315
353
|
|
|
316
354
|
def execute_step_sync(step_config, resolved_arguments = nil)
|
|
317
355
|
@context.with_step(step_config.name) do
|
|
318
|
-
# Check conditions and guards
|
|
319
|
-
unless step_config.should_run?(@context)
|
|
320
|
-
@dependency_graph.complete_step(step_config.name)
|
|
321
|
-
return RubyReactor.Success(nil)
|
|
322
|
-
end
|
|
323
|
-
|
|
324
356
|
# Resolve arguments
|
|
325
|
-
resolved_arguments ||= resolve_arguments(
|
|
357
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
326
358
|
|
|
327
359
|
# Validate arguments if validator is defined
|
|
328
360
|
validate_step_arguments(step_config, resolved_arguments)
|
|
@@ -338,14 +370,8 @@ module RubyReactor
|
|
|
338
370
|
# Execute step without handling the result (used during retries)
|
|
339
371
|
def execute_step_sync_without_result_handling(step_config, resolved_arguments = nil)
|
|
340
372
|
@context.with_step(step_config.name) do
|
|
341
|
-
# Check conditions and guards
|
|
342
|
-
unless step_config.should_run?(@context)
|
|
343
|
-
@dependency_graph.complete_step(step_config.name)
|
|
344
|
-
return RubyReactor.Success(nil)
|
|
345
|
-
end
|
|
346
|
-
|
|
347
373
|
# Resolve arguments
|
|
348
|
-
resolved_arguments ||= resolve_arguments(
|
|
374
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
349
375
|
|
|
350
376
|
yield resolved_arguments if block_given?
|
|
351
377
|
|
|
@@ -435,22 +461,6 @@ module RubyReactor
|
|
|
435
461
|
raise error
|
|
436
462
|
end
|
|
437
463
|
|
|
438
|
-
def resolve_arguments(step_config)
|
|
439
|
-
resolved = {}
|
|
440
|
-
|
|
441
|
-
step_config.arguments.each do |arg_name, arg_config|
|
|
442
|
-
source = arg_config[:source]
|
|
443
|
-
transform = arg_config[:transform]
|
|
444
|
-
|
|
445
|
-
value = source.resolve(@context)
|
|
446
|
-
value = transform.call(value) if transform
|
|
447
|
-
|
|
448
|
-
resolved[arg_name] = value
|
|
449
|
-
end
|
|
450
|
-
|
|
451
|
-
resolved
|
|
452
|
-
end
|
|
453
|
-
|
|
454
464
|
def run_step_implementation(step_config, arguments)
|
|
455
465
|
contract = step_config.input_contract
|
|
456
466
|
@context.append_execution_trace(
|
|
@@ -152,11 +152,14 @@ module RubyReactor
|
|
|
152
152
|
park_held_primitives! if @context.inline_async_execution
|
|
153
153
|
@contention_snooze = true
|
|
154
154
|
raise
|
|
155
|
-
rescue
|
|
156
|
-
@result = @result_handler.handle_execution_error(e)
|
|
155
|
+
rescue Error::Rescuable => e
|
|
156
|
+
@result = aborting_on_interruption { @result_handler.handle_execution_error(e) }
|
|
157
157
|
update_context_status(@result)
|
|
158
158
|
completed = true
|
|
159
159
|
@result
|
|
160
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
161
|
+
mark_aborted
|
|
162
|
+
raise
|
|
160
163
|
ensure
|
|
161
164
|
release_locks unless @parked
|
|
162
165
|
leave_ordered_lock_scope
|
|
@@ -239,6 +242,7 @@ module RubyReactor
|
|
|
239
242
|
@context.admit!
|
|
240
243
|
prepare_for_resume
|
|
241
244
|
save_context
|
|
245
|
+
@past_gates = true
|
|
242
246
|
|
|
243
247
|
@result = if @context.current_step
|
|
244
248
|
execute_current_step_and_continue
|
|
@@ -269,11 +273,14 @@ module RubyReactor
|
|
|
269
273
|
park_held_primitives!
|
|
270
274
|
@contention_snooze = true
|
|
271
275
|
raise e
|
|
272
|
-
rescue
|
|
273
|
-
handle_resume_error(e)
|
|
276
|
+
rescue Error::Rescuable => e
|
|
277
|
+
aborting_on_interruption { handle_resume_error(e) }
|
|
274
278
|
update_context_status(@result)
|
|
275
279
|
completed = true
|
|
276
280
|
@result
|
|
281
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
282
|
+
mark_aborted
|
|
283
|
+
raise
|
|
277
284
|
ensure
|
|
278
285
|
release_locks unless @parked
|
|
279
286
|
@acquired_context_lock&.release
|
|
@@ -288,6 +295,33 @@ module RubyReactor
|
|
|
288
295
|
@compensation_manager.rollback_completed_steps
|
|
289
296
|
end
|
|
290
297
|
|
|
298
|
+
# True once `resume_execution` is past its reactor-level gates (context
|
|
299
|
+
# lock, lock, semaphore, period): from there on it may have run steps.
|
|
300
|
+
def past_gates?
|
|
301
|
+
@past_gates == true
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
# Reached only by an interruption — a signal, an exit, out of memory, an
|
|
305
|
+
# enclosing timeout (008 R-08, R-16); every other exception was rescued as
|
|
306
|
+
# `Error::Rescuable` and rolled back. Running user rollback code now is
|
|
307
|
+
# unsafe, so none runs: a run in the caller's process is recorded
|
|
308
|
+
# `aborted`, with the undo entries not yet undone kept, for a manual
|
|
309
|
+
# `Reactor#undo`; the `ensure` persists it, best effort. A worker run stays
|
|
310
|
+
# `running` and its job is redelivered.
|
|
311
|
+
def mark_aborted
|
|
312
|
+
@context.status = :aborted unless @context.inline_async_execution
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
# A rollback run from a `rescue Error::Rescuable` body is outside the
|
|
316
|
+
# sibling `rescue Exception`, which never sees what that body raises: an
|
|
317
|
+
# interruption there must mark the run here.
|
|
318
|
+
def aborting_on_interruption
|
|
319
|
+
yield
|
|
320
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
321
|
+
mark_aborted unless Error::Rescuable === e # rubocop:disable Style/CaseEquality
|
|
322
|
+
raise
|
|
323
|
+
end
|
|
324
|
+
|
|
291
325
|
def undo_stack
|
|
292
326
|
@compensation_manager.undo_stack
|
|
293
327
|
end
|
|
@@ -5,6 +5,12 @@ module RubyReactor
|
|
|
5
5
|
class Collector
|
|
6
6
|
extend Helpers
|
|
7
7
|
|
|
8
|
+
# Seconds a collector waits for another collector of the same map. One
|
|
9
|
+
# that found the map unsettled releases within milliseconds, and may be
|
|
10
|
+
# the only trigger left for a failure it just deferred (R-04); a holder
|
|
11
|
+
# busy longer is the one applying the map's result.
|
|
12
|
+
COLLECT_LOCK_WAIT = 2
|
|
13
|
+
|
|
8
14
|
def self.perform(arguments)
|
|
9
15
|
arguments = arguments.transform_keys(&:to_sym)
|
|
10
16
|
map_id = arguments[:map_id]
|
|
@@ -30,7 +36,7 @@ module RubyReactor
|
|
|
30
36
|
lock = RubyReactor::Lock.new(
|
|
31
37
|
"map_collect:#{map_id}",
|
|
32
38
|
owner: SecureRandom.uuid, ttl: RubyReactor.configuration.context_lock_ttl,
|
|
33
|
-
wait:
|
|
39
|
+
wait: COLLECT_LOCK_WAIT, auto_extend: true
|
|
34
40
|
)
|
|
35
41
|
lock.acquire
|
|
36
42
|
lock
|
|
@@ -56,8 +62,9 @@ module RubyReactor
|
|
|
56
62
|
|
|
57
63
|
# Idempotency: if the parent already recorded this map step's result, a
|
|
58
64
|
# prior collector already resumed it. Re-resuming would double-execute the
|
|
59
|
-
# steps after the map. Skip.
|
|
60
|
-
|
|
65
|
+
# steps after the map. Skip. A parent already finished (a prior
|
|
66
|
+
# collector applied this map's failure) must not be rolled back twice.
|
|
67
|
+
return if parent_context.intermediate_results.key?(step_name.to_sym) || parent_context.finished?
|
|
61
68
|
|
|
62
69
|
# Check if all tasks are completed
|
|
63
70
|
metadata = storage.retrieve_map_metadata(map_id, parent_reactor_class_name)
|
|
@@ -65,18 +72,21 @@ module RubyReactor
|
|
|
65
72
|
|
|
66
73
|
results_count = storage.count_map_results(map_id, parent_reactor_class_name)
|
|
67
74
|
|
|
68
|
-
#
|
|
69
|
-
#
|
|
70
|
-
#
|
|
71
|
-
#
|
|
72
|
-
#
|
|
75
|
+
# Completion is judged against the total count of elements, not
|
|
76
|
+
# map_offset (which batching reservation can push past it).
|
|
77
|
+
#
|
|
78
|
+
# A fail-fast failure is applied only once every index has settled
|
|
79
|
+
# (a result, `_error`, `_halt` or `_skipped` slot), so the map's
|
|
80
|
+
# compensate sees every element that completed — including ones still
|
|
81
|
+
# in flight when the failure happened (R-04). Until then the last
|
|
82
|
+
# element to settle, or the map sweeper, re-triggers this collector.
|
|
83
|
+
return if results_count < total_count
|
|
84
|
+
|
|
73
85
|
if (failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name))
|
|
74
86
|
handle_failure(failed_context_id, metadata, storage, parent_context, step_name)
|
|
75
87
|
return
|
|
76
88
|
end
|
|
77
89
|
|
|
78
|
-
return if results_count < total_count
|
|
79
|
-
|
|
80
90
|
# Retrieve results lazily
|
|
81
91
|
results = RubyReactor::Map::ResultEnumerator.new(
|
|
82
92
|
map_id,
|
|
@@ -114,7 +124,7 @@ module RubyReactor
|
|
|
114
124
|
# Pass Enumerator to collect block
|
|
115
125
|
collected = collect_block.call(results)
|
|
116
126
|
RubyReactor::Success(collected)
|
|
117
|
-
rescue
|
|
127
|
+
rescue RubyReactor::Error::Rescuable => e
|
|
118
128
|
RubyReactor.configuration.logger.error("Map collect block raised: #{e.message}")
|
|
119
129
|
RubyReactor.configuration.logger.error(e.backtrace.join("\n")) if e.backtrace
|
|
120
130
|
RubyReactor::Failure(e)
|
|
@@ -65,9 +65,8 @@ module RubyReactor
|
|
|
65
65
|
reactor_class_name = arguments[:parent_reactor_class_name]
|
|
66
66
|
|
|
67
67
|
# Fail Fast Check
|
|
68
|
-
if arguments[:fail_fast]
|
|
69
|
-
|
|
70
|
-
return if failed_context_id
|
|
68
|
+
if arguments[:fail_fast] && storage.retrieve_map_failed_context_id(map_id, reactor_class_name)
|
|
69
|
+
return settle_undispatched(arguments, storage)
|
|
71
70
|
end
|
|
72
71
|
|
|
73
72
|
batch_size = arguments[:batch_size] || source.size # Default to all if no batch_size (async=true only)
|
|
@@ -104,6 +103,33 @@ module RubyReactor
|
|
|
104
103
|
end
|
|
105
104
|
end
|
|
106
105
|
|
|
106
|
+
# A fail-fast map stopped dispatching: claim every index not dispatched
|
|
107
|
+
# yet (one offset bump, so a later dispatcher claims none), settle each
|
|
108
|
+
# with a `_skipped` slot, count them down, and trigger the collector if
|
|
109
|
+
# that settled the map. Without this those indices never settle, so the
|
|
110
|
+
# failure would never be applied (R-04) and the map sweeper would keep
|
|
111
|
+
# re-dispatching them.
|
|
112
|
+
def self.settle_undispatched(arguments, storage)
|
|
113
|
+
map_id = arguments[:map_id]
|
|
114
|
+
reactor_class_name = arguments[:parent_reactor_class_name]
|
|
115
|
+
total = storage.retrieve_map_metadata(map_id, reactor_class_name)&.fetch("count", 0).to_i
|
|
116
|
+
new_offset = storage.increment_map_offset(map_id, total, reactor_class_name)
|
|
117
|
+
claimed = (new_offset - total)...[new_offset, total].min
|
|
118
|
+
return if claimed.none?
|
|
119
|
+
|
|
120
|
+
claimed.each do |index|
|
|
121
|
+
storage.store_map_result(map_id, index, { "_skipped" => true }, reactor_class_name,
|
|
122
|
+
strict_ordering: arguments[:strict_ordering])
|
|
123
|
+
end
|
|
124
|
+
return unless storage.decrement_map_counter_by(map_id, claimed.size, reactor_class_name) <= 0
|
|
125
|
+
|
|
126
|
+
RubyReactor.configuration.async_router.perform_map_collection_async(
|
|
127
|
+
parent_context_id: arguments[:parent_context_id], map_id: map_id,
|
|
128
|
+
parent_reactor_class_name: reactor_class_name, step_name: arguments[:step_name].to_s,
|
|
129
|
+
strict_ordering: arguments[:strict_ordering], timeout: 3600
|
|
130
|
+
)
|
|
131
|
+
end
|
|
132
|
+
|
|
107
133
|
# Re-dispatch a SPECIFIC index whose result slot is missing (Phase 5c, used
|
|
108
134
|
# by the map sweeper). Index-driven rather than offset-driven: resolve the
|
|
109
135
|
# source from the stored parent context and pick source[index]. Idempotent
|
|
@@ -55,11 +55,13 @@ module RubyReactor
|
|
|
55
55
|
context.inline_async_execution = true
|
|
56
56
|
|
|
57
57
|
storage = RubyReactor.configuration.storage_adapter
|
|
58
|
+
return if check_fail_fast?(arguments, storage)
|
|
59
|
+
|
|
60
|
+
# Indexed only once it runs: a skipped element saves no context, and
|
|
61
|
+
# the map's rollback reports an indexed element without one as expired.
|
|
58
62
|
storage.store_map_element_context_id(arguments[:map_id], context.context_id,
|
|
59
63
|
arguments[:parent_reactor_class_name])
|
|
60
64
|
|
|
61
|
-
return if check_fail_fast?(arguments, storage)
|
|
62
|
-
|
|
63
65
|
executor = Executor.new(context.reactor_class, {}, context)
|
|
64
66
|
begin
|
|
65
67
|
arguments[:serialized_context] ? executor.resume_execution : executor.execute
|
|
@@ -151,7 +153,11 @@ module RubyReactor
|
|
|
151
153
|
failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name)
|
|
152
154
|
return false unless failed_context_id
|
|
153
155
|
|
|
154
|
-
# Skip execution
|
|
156
|
+
# Skip execution, but settle the index: the collector applies a
|
|
157
|
+
# fail-fast failure only once every index has a result slot (R-04), and
|
|
158
|
+
# the map sweeper would otherwise keep re-dispatching this one.
|
|
159
|
+
storage.store_map_result(map_id, arguments[:index], { "_skipped" => true }, parent_reactor_class_name,
|
|
160
|
+
strict_ordering: arguments[:strict_ordering])
|
|
155
161
|
finalize_execution(arguments, storage)
|
|
156
162
|
true
|
|
157
163
|
end
|
|
@@ -41,7 +41,7 @@ module RubyReactor
|
|
|
41
41
|
begin
|
|
42
42
|
collected = collect_block.call(results)
|
|
43
43
|
RubyReactor::Success(collected)
|
|
44
|
-
rescue
|
|
44
|
+
rescue RubyReactor::Error::Rescuable => e
|
|
45
45
|
RubyReactor::Failure(e)
|
|
46
46
|
end
|
|
47
47
|
else
|
|
@@ -83,7 +83,20 @@ module RubyReactor
|
|
|
83
83
|
parent_context)
|
|
84
84
|
failure_response = nil
|
|
85
85
|
begin
|
|
86
|
-
|
|
86
|
+
# The map failed in its element jobs, so no executor compensated it.
|
|
87
|
+
# Fail it exactly as the inline path does: adopt the failing
|
|
88
|
+
# element's rollback failures, compensate the map (its completed
|
|
89
|
+
# elements; every index has settled, R-04), undo the completed
|
|
90
|
+
# steps, and fail with `CompensationError` if the map's rollback
|
|
91
|
+
# was incomplete.
|
|
92
|
+
manager = executor.compensation_manager
|
|
93
|
+
manager.rollback_failures.concat(final_result.rollback_failures)
|
|
94
|
+
failure_response = begin
|
|
95
|
+
manager.handle_step_failure(parent_context.reactor_class.steps[step_name_sym], final_result.error, {})
|
|
96
|
+
executor.result_handler.handle_execution_error(error)
|
|
97
|
+
rescue RubyReactor::Error::CompensationError => e
|
|
98
|
+
executor.result_handler.handle_execution_error(e)
|
|
99
|
+
end
|
|
87
100
|
# Manually update context status since we're not running executor loop
|
|
88
101
|
executor.send(:update_context_status, failure_response)
|
|
89
102
|
ensure
|
|
@@ -95,6 +108,12 @@ module RubyReactor
|
|
|
95
108
|
store_parent(parent_context, storage)
|
|
96
109
|
else
|
|
97
110
|
parent_context.set_result(step_name_sym, final_result.value)
|
|
111
|
+
# A completed fan-out map is tracked for undo like any step (R-03).
|
|
112
|
+
# The record stays empty: `MapStep#undo` finds its elements through
|
|
113
|
+
# the map's element index, so the parent does not grow per element.
|
|
114
|
+
parent_context.undo_stack << { step: parent_context.reactor_class.steps[step_name_sym], arguments: {},
|
|
115
|
+
result: RubyReactor.Success(nil) }
|
|
116
|
+
record_elements_started(parent_context, step_name, storage)
|
|
98
117
|
|
|
99
118
|
# Manually update execution trace to reflect completion
|
|
100
119
|
# This is necessary because resume_execution continues from the NEXT step
|
|
@@ -149,6 +168,17 @@ module RubyReactor
|
|
|
149
168
|
)
|
|
150
169
|
end
|
|
151
170
|
|
|
171
|
+
# Every index of a completed map ran: record the count on the map's
|
|
172
|
+
# reference in the parent, which outlives the element index, so a late
|
|
173
|
+
# rollback names each element whose context expired.
|
|
174
|
+
def record_elements_started(parent_context, step_name, storage)
|
|
175
|
+
ref = Utils::FetchIndifferent.call(parent_context.composed_contexts, step_name)
|
|
176
|
+
return unless ref
|
|
177
|
+
|
|
178
|
+
map_id = Utils::FetchIndifferent.call(ref, :map_id)
|
|
179
|
+
ref[:started] = storage.retrieve_map_metadata(map_id, parent_context.reactor_class.name)&.fetch("count", 0).to_i
|
|
180
|
+
end
|
|
181
|
+
|
|
152
182
|
def store_parent(parent_context, storage)
|
|
153
183
|
storage.store_context(
|
|
154
184
|
parent_context.context_id,
|
|
@@ -22,7 +22,8 @@ module RubyReactor
|
|
|
22
22
|
|
|
23
23
|
if @strict_ordering
|
|
24
24
|
count.times do |i|
|
|
25
|
-
|
|
25
|
+
raw = raw_at(i)
|
|
26
|
+
yield raw && wrap_result(raw) unless skipped?(raw)
|
|
26
27
|
end
|
|
27
28
|
else
|
|
28
29
|
offset = 0
|
|
@@ -37,7 +38,7 @@ module RubyReactor
|
|
|
37
38
|
|
|
38
39
|
break if results.empty?
|
|
39
40
|
|
|
40
|
-
results.each { |result| yield wrap_result(result) }
|
|
41
|
+
results.each { |result| yield wrap_result(result) unless skipped?(result) }
|
|
41
42
|
|
|
42
43
|
offset += results.size
|
|
43
44
|
break if results.size < @batch_size
|
|
@@ -59,21 +60,15 @@ module RubyReactor
|
|
|
59
60
|
!empty?
|
|
60
61
|
end
|
|
61
62
|
|
|
63
|
+
# nil for an index a fail-fast map skipped (never started).
|
|
62
64
|
def [](index)
|
|
63
65
|
index += count if index.negative?
|
|
64
66
|
return nil if index.negative? || index >= count
|
|
65
67
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
@reactor_class_name,
|
|
69
|
-
offset: index,
|
|
70
|
-
limit: 1,
|
|
71
|
-
strict_ordering: @strict_ordering
|
|
72
|
-
)
|
|
68
|
+
raw = raw_at(index)
|
|
69
|
+
return nil if raw.nil? || skipped?(raw)
|
|
73
70
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
wrap_result(results.first)
|
|
71
|
+
wrap_result(raw)
|
|
77
72
|
end
|
|
78
73
|
|
|
79
74
|
def first
|
|
@@ -94,6 +89,17 @@ module RubyReactor
|
|
|
94
89
|
|
|
95
90
|
private
|
|
96
91
|
|
|
92
|
+
def raw_at(index)
|
|
93
|
+
@storage.retrieve_map_results_batch(@map_id, @reactor_class_name, offset: index, limit: 1,
|
|
94
|
+
strict_ordering: @strict_ordering).first
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# A `_skipped` slot settles an index a fail-fast map never ran (R-04).
|
|
98
|
+
# Such a map is never collected as a success, so no consumer sees one.
|
|
99
|
+
def skipped?(result)
|
|
100
|
+
result.is_a?(Hash) && result.key?("_skipped")
|
|
101
|
+
end
|
|
102
|
+
|
|
97
103
|
def wrap_result(result)
|
|
98
104
|
if result.is_a?(Hash) && result.key?("_error")
|
|
99
105
|
# `backtrace: []` is load-bearing: without it Failure falls back to
|
data/lib/ruby_reactor/reactor.rb
CHANGED
|
@@ -153,6 +153,14 @@ module RubyReactor
|
|
|
153
153
|
"Cannot resume: reactor has been cancelled (Reason: #{@context.cancellation_reason})"
|
|
154
154
|
end
|
|
155
155
|
|
|
156
|
+
# Only a reactor paused at an interrupt takes a resume (008 FR-032): one
|
|
157
|
+
# that is executing or rolling back (`running`), finished, or `aborted`
|
|
158
|
+
# must not be run forward from its stored state.
|
|
159
|
+
unless @context.status.to_s == "paused"
|
|
160
|
+
raise Error::ValidationError,
|
|
161
|
+
"Cannot resume: the reactor is #{@context.status}, not paused at an interrupt"
|
|
162
|
+
end
|
|
163
|
+
|
|
156
164
|
validate_continue_step!(step_name)
|
|
157
165
|
|
|
158
166
|
if (failure = validate_continue_payload(payload, step_name))
|
|
@@ -170,6 +178,11 @@ module RubyReactor
|
|
|
170
178
|
return @result = enqueue_background_resume
|
|
171
179
|
end
|
|
172
180
|
|
|
181
|
+
# Claim the resume before running it, so a `continue` that arrives while
|
|
182
|
+
# this one executes or rolls back reads `running` and fails (FR-032).
|
|
183
|
+
@context.status = :running
|
|
184
|
+
save_context
|
|
185
|
+
|
|
173
186
|
# Resume execution
|
|
174
187
|
executor = Executor.new(self.class, {}, @context)
|
|
175
188
|
@result = executor.resume_execution
|
|
@@ -179,6 +192,12 @@ module RubyReactor
|
|
|
179
192
|
@execution_trace = executor.execution_trace
|
|
180
193
|
|
|
181
194
|
@result
|
|
195
|
+
rescue Lock::AcquisitionError, Semaphore::AcquisitionError => e
|
|
196
|
+
# Contended at the resume's own gates: nothing ran, so the run is still
|
|
197
|
+
# paused and the caller may retry, as with `Reactor.run`. A context-lock
|
|
198
|
+
# contention means a live resume holds the run: leave it alone.
|
|
199
|
+
reopen_paused unless executor&.past_gates? || e.is_a?(Lock::ContextLockContention)
|
|
200
|
+
raise
|
|
182
201
|
rescue Error::InputValidationError => e
|
|
183
202
|
# This might catch other validations, but here we specifically want payload validation.
|
|
184
203
|
# The block above handles payload validation explicitly.
|
|
@@ -211,6 +230,11 @@ module RubyReactor
|
|
|
211
230
|
RubyReactor::Configuration.instance
|
|
212
231
|
end
|
|
213
232
|
|
|
233
|
+
def reopen_paused
|
|
234
|
+
@context.status = :paused
|
|
235
|
+
save_context
|
|
236
|
+
end
|
|
237
|
+
|
|
214
238
|
def validate_steps!
|
|
215
239
|
return unless self.class.steps.empty?
|
|
216
240
|
|
|
@@ -261,6 +261,15 @@ module RubyReactor
|
|
|
261
261
|
RubyReactor.configuration.storage_adapter
|
|
262
262
|
end
|
|
263
263
|
|
|
264
|
+
def self.rate_limit_total(key_base, period, since)
|
|
265
|
+
return coordination_adapter.rate_limit_count(key_base, period) unless since
|
|
266
|
+
|
|
267
|
+
seconds = RubyReactor::Period.period_seconds(period)
|
|
268
|
+
((since.to_i / seconds)..(Time.now.to_i / seconds)).sum do |bucket|
|
|
269
|
+
coordination_adapter.rate_limit_count(key_base, period, now: bucket * seconds)
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
|
|
264
273
|
# Distinguishes `RubyReactor::Halt` (a clean halt) from a plain
|
|
265
274
|
# `Success`. Works on any object with a `halted?` predicate.
|
|
266
275
|
#
|
|
@@ -419,22 +428,29 @@ module RubyReactor
|
|
|
419
428
|
end
|
|
420
429
|
|
|
421
430
|
# Asserts the current rate-limit counter for a (key_base, period) pair.
|
|
422
|
-
# Use `.for(period_unit)` to specify which window.
|
|
431
|
+
# Use `.for(period_unit)` to specify which window. Add `.since(time)` to
|
|
432
|
+
# sum every bucket from `time` to now instead — buckets are fixed
|
|
433
|
+
# windows, so a run that straddles a boundary splits its count.
|
|
423
434
|
#
|
|
424
435
|
# expect("stripe:42").to have_rate_limit_count(3).for(:second)
|
|
436
|
+
# expect("stripe:42").to have_rate_limit_count(1).for(:minute).since(started_at)
|
|
425
437
|
::RSpec::Matchers.define :have_rate_limit_count do |expected|
|
|
426
438
|
match do |key_base|
|
|
427
439
|
raise ArgumentError, "have_rate_limit_count requires .for(period)" unless @period
|
|
428
440
|
|
|
429
|
-
Matchers.
|
|
441
|
+
Matchers.rate_limit_total(key_base, @period, @since) == expected
|
|
430
442
|
end
|
|
431
443
|
|
|
432
444
|
chain :for do |period|
|
|
433
445
|
@period = period
|
|
434
446
|
end
|
|
435
447
|
|
|
448
|
+
chain :since do |time|
|
|
449
|
+
@since = time
|
|
450
|
+
end
|
|
451
|
+
|
|
436
452
|
failure_message do |key_base|
|
|
437
|
-
actual = Matchers.
|
|
453
|
+
actual = Matchers.rate_limit_total(key_base, @period, @since)
|
|
438
454
|
"expected rate-limit '#{key_base}' (#{@period}) count to be #{expected}, got #{actual}"
|
|
439
455
|
end
|
|
440
456
|
end
|
|
@@ -34,7 +34,10 @@ module RubyReactor
|
|
|
34
34
|
return RubyReactor.Success() unless composed_data && composed_data[:context]
|
|
35
35
|
|
|
36
36
|
child_context = composed_data[:context]
|
|
37
|
-
|
|
37
|
+
# The child's own class: it survives serialization, while a Class in a
|
|
38
|
+
# stored undo record's arguments comes back as its name (a compose
|
|
39
|
+
# inside a map element is rolled back from its stored row).
|
|
40
|
+
executor = RubyReactor::Executor.new(child_context.reactor_class, {}, child_context)
|
|
38
41
|
executor.undo_all
|
|
39
42
|
executor.save_context
|
|
40
43
|
|
|
@@ -46,6 +49,9 @@ module RubyReactor
|
|
|
46
49
|
|
|
47
50
|
alias undo compensate
|
|
48
51
|
|
|
52
|
+
# An interrupted run is undone too: undo replays only the child's completed steps.
|
|
53
|
+
def self.undoes_partial_run? = true
|
|
54
|
+
|
|
49
55
|
private
|
|
50
56
|
|
|
51
57
|
def build_composed_inputs(mappings)
|