ruby_reactor 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/CHANGELOG.md +208 -0
- data/CLAUDE.md +1 -1
- data/README.md +140 -102
- data/lib/ruby_reactor/dsl/async_macros.rb +30 -1
- data/lib/ruby_reactor/dsl/async_reactor_builder.rb +10 -7
- data/lib/ruby_reactor/dsl/compose_builder.rb +10 -11
- data/lib/ruby_reactor/dsl/interrupt_builder.rb +1 -3
- data/lib/ruby_reactor/dsl/map_builder.rb +0 -2
- data/lib/ruby_reactor/dsl/reactor.rb +8 -11
- data/lib/ruby_reactor/dsl/retryable.rb +45 -0
- data/lib/ruby_reactor/dsl/step_builder.rb +131 -33
- data/lib/ruby_reactor/error/argument_resolution_error.rb +19 -0
- data/lib/ruby_reactor/error/rescuable.rb +28 -0
- data/lib/ruby_reactor/error/undeclared_input_error.rb +13 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +30 -26
- data/lib/ruby_reactor/executor/result_handler.rb +21 -18
- data/lib/ruby_reactor/executor/retry_manager.rb +11 -12
- data/lib/ruby_reactor/executor/step_coordination.rb +11 -8
- data/lib/ruby_reactor/executor/step_executor.rb +59 -49
- data/lib/ruby_reactor/executor.rb +38 -4
- data/lib/ruby_reactor/map/collector.rb +21 -11
- data/lib/ruby_reactor/map/dispatcher.rb +29 -3
- data/lib/ruby_reactor/map/element_executor.rb +9 -3
- data/lib/ruby_reactor/map/helpers.rb +32 -2
- data/lib/ruby_reactor/map/result_enumerator.rb +18 -12
- data/lib/ruby_reactor/reactor.rb +24 -0
- data/lib/ruby_reactor/rspec/matchers.rb +19 -3
- data/lib/ruby_reactor/rspec/test_subject.rb +3 -5
- data/lib/ruby_reactor/step/async_reactor_step.rb +6 -2
- data/lib/ruby_reactor/step/compose_step.rb +14 -4
- data/lib/ruby_reactor/step/input_contract.rb +5 -0
- data/lib/ruby_reactor/step/inputs.rb +57 -0
- data/lib/ruby_reactor/step/map_step.rb +140 -25
- data/lib/ruby_reactor/step.rb +20 -7
- data/lib/ruby_reactor/step_worker.rb +46 -22
- data/lib/ruby_reactor/storage/adapter.rb +4 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +9 -0
- data/lib/ruby_reactor/storage/redis_reactor_scan.rb +1 -1
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +1 -1
- data/lib/ruby_reactor/web/public/assets/{index-CeZU-ESu.js → index-CQbgHtd0.js} +10 -10
- data/lib/ruby_reactor/web/public/index.html +1 -1
- data/lib/ruby_reactor/worker.rb +3 -1
- data/lib/ruby_reactor.rb +20 -7
- data/specs/006-step-retry-declarations/checklists/requirements.md +41 -0
- data/specs/006-step-retry-declarations/contracts/dsl-surface.md +89 -0
- data/specs/006-step-retry-declarations/data-model.md +58 -0
- data/specs/006-step-retry-declarations/plan.md +187 -0
- data/specs/006-step-retry-declarations/quickstart.md +80 -0
- data/specs/006-step-retry-declarations/research.md +194 -0
- data/specs/006-step-retry-declarations/spec.md +453 -0
- data/specs/006-step-retry-declarations/tasks.md +382 -0
- data/specs/007-execution-flow-analysis/analysis/README.md +147 -0
- data/specs/007-execution-flow-analysis/analysis/execution-order.md +359 -0
- data/specs/007-execution-flow-analysis/analysis/findings-and-options.md +502 -0
- data/specs/007-execution-flow-analysis/analysis/invariants.md +109 -0
- data/specs/007-execution-flow-analysis/checklists/requirements.md +39 -0
- data/specs/007-execution-flow-analysis/contracts/report-structure.md +71 -0
- data/specs/007-execution-flow-analysis/data-model.md +83 -0
- data/specs/007-execution-flow-analysis/evidence/harness.rb +229 -0
- data/specs/007-execution-flow-analysis/evidence/output.txt +333 -0
- data/specs/007-execution-flow-analysis/evidence/probes/01_plain.rb +122 -0
- data/specs/007-execution-flow-analysis/evidence/probes/02_compose.rb +182 -0
- data/specs/007-execution-flow-analysis/evidence/probes/03_map.rb +232 -0
- data/specs/007-execution-flow-analysis/evidence/probes/04_async.rb +132 -0
- data/specs/007-execution-flow-analysis/evidence/probes/05_background.rb +58 -0
- data/specs/007-execution-flow-analysis/evidence/probes/06_coordination.rb +158 -0
- data/specs/007-execution-flow-analysis/evidence/probes/07_interrupts_manual.rb +185 -0
- data/specs/007-execution-flow-analysis/evidence/run.rb +15 -0
- data/specs/007-execution-flow-analysis/plan.md +127 -0
- data/specs/007-execution-flow-analysis/quickstart.md +51 -0
- data/specs/007-execution-flow-analysis/research.md +202 -0
- data/specs/007-execution-flow-analysis/spec.md +270 -0
- data/specs/007-execution-flow-analysis/tasks.md +257 -0
- data/specs/008-rollback-reliability/checklists/requirements.md +43 -0
- data/specs/008-rollback-reliability/contracts/api-surface.md +126 -0
- data/specs/008-rollback-reliability/contracts/rollback-semantics.md +76 -0
- data/specs/008-rollback-reliability/data-model.md +139 -0
- data/specs/008-rollback-reliability/plan.md +233 -0
- data/specs/008-rollback-reliability/quickstart.md +105 -0
- data/specs/008-rollback-reliability/research.md +653 -0
- data/specs/008-rollback-reliability/spec.md +561 -0
- data/specs/008-rollback-reliability/tasks.md +1110 -0
- data/specs/future_improvements.md +48 -0
- data/specs/specs-inputs-by-method-md-piped-wigderson.md +77 -0
- metadata +47 -2
|
@@ -62,12 +62,20 @@ module RubyReactor
|
|
|
62
62
|
@compensation_manager.rollback_completed_steps
|
|
63
63
|
build_validation_failure(error)
|
|
64
64
|
when Error::Base
|
|
65
|
-
# Other errors need rollback
|
|
65
|
+
# Other errors need rollback. A `CompensationError` names the step
|
|
66
|
+
# whose compensation failed.
|
|
66
67
|
@compensation_manager.rollback_completed_steps
|
|
67
|
-
RubyReactor.Failure("Execution error: #{error.message}", exception_class: error.class.name
|
|
68
|
+
RubyReactor.Failure("Execution error: #{error.message}", exception_class: error.class.name,
|
|
69
|
+
step_name: error.step || @context.current_step,
|
|
70
|
+
reactor_name: @context.reactor_class&.name)
|
|
68
71
|
else
|
|
69
|
-
#
|
|
70
|
-
|
|
72
|
+
# Any other StandardError after completed work (a checkpoint write,
|
|
73
|
+
# a hook) still leaves a partial saga: roll it back like any failure
|
|
74
|
+
# (Constitution II, 008 R-07) and name the step that was executing.
|
|
75
|
+
@compensation_manager.rollback_completed_steps
|
|
76
|
+
RubyReactor.Failure("Execution failed: #{error.message}", exception_class: error.class.name,
|
|
77
|
+
step_name: @context.current_step,
|
|
78
|
+
reactor_name: @context.reactor_class&.name)
|
|
71
79
|
end
|
|
72
80
|
end
|
|
73
81
|
|
|
@@ -108,9 +116,9 @@ module RubyReactor
|
|
|
108
116
|
result
|
|
109
117
|
end
|
|
110
118
|
|
|
111
|
-
# A step returned `RubyReactor.Skipped(...)
|
|
112
|
-
#
|
|
113
|
-
#
|
|
119
|
+
# A step returned `RubyReactor.Skipped(...)`: the run continues exactly as
|
|
120
|
+
# for a Success (value, hand-off, period mark), but the step is NOT pushed
|
|
121
|
+
# for undo — it had nothing to do, so there is nothing to revert (008 R-19).
|
|
114
122
|
def handle_skipped(step_config, result, resolved_arguments)
|
|
115
123
|
validate_step_output(step_config, result.value, resolved_arguments)
|
|
116
124
|
@step_results[step_config.name] = result
|
|
@@ -130,11 +138,7 @@ module RubyReactor
|
|
|
130
138
|
def handle_success(step_config, result, resolved_arguments)
|
|
131
139
|
validate_step_output(step_config, result.value, resolved_arguments)
|
|
132
140
|
@step_results[step_config.name] = result
|
|
133
|
-
|
|
134
|
-
# work with their own compensation flows — the parent rolling back must
|
|
135
|
-
# not "undo" a dispatch whose unit runs (and may still succeed)
|
|
136
|
-
# elsewhere. They never enter the parent's undo stack.
|
|
137
|
-
unless async_unit?(step_config)
|
|
141
|
+
if step_config.rollback_tracked?
|
|
138
142
|
@compensation_manager.add_to_undo_stack({ step: step_config, arguments: resolved_arguments,
|
|
139
143
|
result: result })
|
|
140
144
|
end
|
|
@@ -142,10 +146,6 @@ module RubyReactor
|
|
|
142
146
|
@dependency_graph.complete_step(step_config.name)
|
|
143
147
|
end
|
|
144
148
|
|
|
145
|
-
def async_unit?(step_config)
|
|
146
|
-
step_config.respond_to?(:async_dispatch?) && step_config.async_dispatch?
|
|
147
|
-
end
|
|
148
|
-
|
|
149
149
|
# A composed child's Failure carries the child's own rollback failures;
|
|
150
150
|
# fold them in BEFORE this level rolls back, so they come first.
|
|
151
151
|
def adopt_rollback_failures(result)
|
|
@@ -188,13 +188,14 @@ module RubyReactor
|
|
|
188
188
|
raise error
|
|
189
189
|
end
|
|
190
190
|
|
|
191
|
+
# Wrapped first, so a returned `Step::Inputs` is validated and stored as its Hash.
|
|
191
192
|
def handle_unknown_result(step_config, result, resolved_arguments)
|
|
192
|
-
validate_step_output(step_config, result, resolved_arguments)
|
|
193
193
|
success_result = RubyReactor.Success(result)
|
|
194
|
+
validate_step_output(step_config, success_result.value, resolved_arguments)
|
|
194
195
|
@step_results[step_config.name] = success_result
|
|
195
196
|
@compensation_manager.add_to_undo_stack({ step: step_config, arguments: resolved_arguments,
|
|
196
197
|
result: success_result })
|
|
197
|
-
@context.set_result(step_config.name,
|
|
198
|
+
@context.set_result(step_config.name, success_result.value)
|
|
198
199
|
@dependency_graph.complete_step(step_config.name)
|
|
199
200
|
end
|
|
200
201
|
|
|
@@ -253,6 +254,8 @@ module RubyReactor
|
|
|
253
254
|
def resolve_exception_class(original_error, error)
|
|
254
255
|
# A step's own contention is reported by its cause (Lock::AcquisitionError, ...).
|
|
255
256
|
original_error = original_error.original if original_error.is_a?(StepCoordination::Contended)
|
|
257
|
+
# Argument failures report their cause's class (008 R-06).
|
|
258
|
+
return original_error.exception_class if original_error.is_a?(Error::ArgumentResolutionError)
|
|
256
259
|
return original_error.class.name if original_error
|
|
257
260
|
|
|
258
261
|
error.respond_to?(:exception_class) ? error.exception_class : nil
|
|
@@ -24,19 +24,18 @@ module RubyReactor
|
|
|
24
24
|
step_config.retry_config[:max_attempts])
|
|
25
25
|
end
|
|
26
26
|
|
|
27
|
-
def calculate_backoff_delay(step_config
|
|
27
|
+
def calculate_backoff_delay(step_config)
|
|
28
28
|
attempt_number = @context.retry_context.attempts_for_step(step_config.name)
|
|
29
|
-
|
|
30
|
-
base_delay = step_config.retry_config[:base_delay] || reactor_class.retry_defaults[:base_delay]
|
|
29
|
+
retry_config = step_config.retry_config
|
|
31
30
|
|
|
32
|
-
delay = RetryContext.calculate_backoff_delay(attempt_number,
|
|
31
|
+
delay = RetryContext.calculate_backoff_delay(attempt_number, retry_config[:backoff], retry_config[:base_delay])
|
|
33
32
|
@context.retry_context.next_retry_at = Time.now + delay
|
|
34
33
|
delay
|
|
35
34
|
end
|
|
36
35
|
|
|
37
|
-
def requeue_job_for_step_retry(step_config
|
|
36
|
+
def requeue_job_for_step_retry(step_config)
|
|
38
37
|
@context.current_step = step_config.name
|
|
39
|
-
delay = calculate_backoff_delay(step_config
|
|
38
|
+
delay = calculate_backoff_delay(step_config)
|
|
40
39
|
|
|
41
40
|
requeue_job(step_config, delay)
|
|
42
41
|
end
|
|
@@ -147,14 +146,14 @@ module RubyReactor
|
|
|
147
146
|
|
|
148
147
|
# Always try async retry if configured
|
|
149
148
|
if is_async
|
|
150
|
-
handle_async_retry(step_config
|
|
149
|
+
handle_async_retry(step_config)
|
|
151
150
|
else
|
|
152
|
-
handle_sync_retry(step_config
|
|
151
|
+
handle_sync_retry(step_config)
|
|
153
152
|
end
|
|
154
153
|
end
|
|
155
154
|
|
|
156
|
-
def handle_async_retry(step_config
|
|
157
|
-
requeue_result = requeue_job_for_step_retry(step_config
|
|
155
|
+
def handle_async_retry(step_config)
|
|
156
|
+
requeue_result = requeue_job_for_step_retry(step_config)
|
|
158
157
|
|
|
159
158
|
# If it returned an DispatchResult, we are truly async.
|
|
160
159
|
# Otherwise, it ran inline and we should return the result of that execution.
|
|
@@ -169,8 +168,8 @@ module RubyReactor
|
|
|
169
168
|
end
|
|
170
169
|
end
|
|
171
170
|
|
|
172
|
-
def handle_sync_retry(step_config
|
|
173
|
-
delay = calculate_backoff_delay(step_config
|
|
171
|
+
def handle_sync_retry(step_config)
|
|
172
|
+
delay = calculate_backoff_delay(step_config)
|
|
174
173
|
sleep(delay)
|
|
175
174
|
nil # continue loop
|
|
176
175
|
end
|
|
@@ -99,7 +99,7 @@ module RubyReactor
|
|
|
99
99
|
key
|
|
100
100
|
rescue KeyError
|
|
101
101
|
raise
|
|
102
|
-
rescue
|
|
102
|
+
rescue RubyReactor::Error::Rescuable => e
|
|
103
103
|
raise KeyError.new(
|
|
104
104
|
"#{step_name}: coordination key proc raised #{e.class}: #{e.message}",
|
|
105
105
|
step: step_name, original_error: e
|
|
@@ -188,7 +188,7 @@ module RubyReactor
|
|
|
188
188
|
# counter and marker must survive the park.
|
|
189
189
|
rescue Contended, Error::ExecutionParked
|
|
190
190
|
raise
|
|
191
|
-
rescue
|
|
191
|
+
rescue Error::Rescuable
|
|
192
192
|
clear_contention_state
|
|
193
193
|
raise
|
|
194
194
|
end
|
|
@@ -498,7 +498,7 @@ module RubyReactor
|
|
|
498
498
|
rescue Error::ExecutionParked
|
|
499
499
|
outcome = :parked
|
|
500
500
|
raise
|
|
501
|
-
rescue
|
|
501
|
+
rescue Error::Rescuable
|
|
502
502
|
outcome = :failed
|
|
503
503
|
raise
|
|
504
504
|
ensure
|
|
@@ -513,8 +513,8 @@ module RubyReactor
|
|
|
513
513
|
# step again and must keep its place: the stash survives, so the next
|
|
514
514
|
# attempt re-reads the SAME nonce. The heartbeat is stopped across the
|
|
515
515
|
# gap; `poison_pill_timeout` bounds it.
|
|
516
|
-
# - :abandoned — an
|
|
517
|
-
# `
|
|
516
|
+
# - :abandoned — an interruption (`Sidekiq::Shutdown`, `NoMemoryError`,
|
|
517
|
+
# ...; `Error::Rescuable`, 008 R-16). Not advanced: `Sidekiq::Shutdown` pushes the
|
|
518
518
|
# job back to run again, which must keep this place, and `failed: true`
|
|
519
519
|
# would poison successors for work that may yet complete. With the
|
|
520
520
|
# heartbeat stopped, the poison pill releases the position within
|
|
@@ -644,7 +644,7 @@ module RubyReactor
|
|
|
644
644
|
return skipped_for_period if storage_adapter.period_seen?(key)
|
|
645
645
|
|
|
646
646
|
result = yield
|
|
647
|
-
if
|
|
647
|
+
if continuing_success?(result) && !chain_failed?(result)
|
|
648
648
|
storage_adapter.period_mark(key, RubyReactor::Period.ttl_seconds(config[:every]))
|
|
649
649
|
end
|
|
650
650
|
result
|
|
@@ -654,8 +654,11 @@ module RubyReactor
|
|
|
654
654
|
RubyReactor.Skipped(nil, reason: :period, step_name: step_name)
|
|
655
655
|
end
|
|
656
656
|
|
|
657
|
-
|
|
658
|
-
|
|
657
|
+
# A body's `Skipped` marks the bucket like a Success: it is only an
|
|
658
|
+
# instrumentation mark. The library's own skips (period, ordered lock)
|
|
659
|
+
# are returned by the gates around this one, never through it.
|
|
660
|
+
def continuing_success?(result)
|
|
661
|
+
result.is_a?(RubyReactor::Success) && !result.is_a?(RubyReactor::Halt)
|
|
659
662
|
end
|
|
660
663
|
|
|
661
664
|
def period_key(config)
|
|
@@ -80,12 +80,18 @@ module RubyReactor
|
|
|
80
80
|
# it, not this one. (`async_reactor` still resolves here: its resolved
|
|
81
81
|
# values are the child's INPUTS, needed at dispatch.)
|
|
82
82
|
deferred_body = step_config.async_dispatch == :step || handoff_at?(step_config, :before)
|
|
83
|
-
resolved_arguments =
|
|
83
|
+
resolved_arguments, resolution_error = resolve_before_start(step_config, deferred_body)
|
|
84
84
|
|
|
85
85
|
@middlewares.on(:start_step, step_config.name, resolved_arguments, @context)
|
|
86
86
|
completed = false
|
|
87
87
|
begin
|
|
88
|
-
|
|
88
|
+
# A step whose arguments could not be resolved fails like any step
|
|
89
|
+
# (`:failed_step`, attribution) but never started: the result
|
|
90
|
+
# handler undoes the completed steps and does not compensate it.
|
|
91
|
+
result = if resolution_error
|
|
92
|
+
@result_handler.handle_step_result(step_config,
|
|
93
|
+
pre_body_failure(step_config, resolution_error), {})
|
|
94
|
+
elsif step_config.interrupt?
|
|
89
95
|
handle_interrupt_step(step_config)
|
|
90
96
|
elsif step_config.async_dispatch == :step
|
|
91
97
|
dispatch_async_step(step_config)
|
|
@@ -113,6 +119,7 @@ module RubyReactor
|
|
|
113
119
|
unless completed
|
|
114
120
|
event = e.is_a?(Error::ExecutionParked) ? :snooze_step : :failed_step
|
|
115
121
|
@middlewares.on(event, step_config.name, e, @context)
|
|
122
|
+
track_interrupted_construct(step_config, resolved_arguments, e)
|
|
116
123
|
end
|
|
117
124
|
raise
|
|
118
125
|
end
|
|
@@ -120,6 +127,28 @@ module RubyReactor
|
|
|
120
127
|
|
|
121
128
|
private
|
|
122
129
|
|
|
130
|
+
# An interruption leaves a caller-process run `aborted` for a manual
|
|
131
|
+
# undo, which replays the undo stack. A compose or map cut short already
|
|
132
|
+
# holds completed child work that only its own undo reverts, so it joins
|
|
133
|
+
# the stack. A worker run is redelivered instead, and re-runs the step.
|
|
134
|
+
def track_interrupted_construct(step_config, arguments, error)
|
|
135
|
+
return if @context.inline_async_execution || Error::Rescuable === error # rubocop:disable Style/CaseEquality
|
|
136
|
+
return unless step_config.undoes_partial_run?
|
|
137
|
+
|
|
138
|
+
@compensation_manager.add_to_undo_stack({ step: step_config, arguments: arguments,
|
|
139
|
+
result: RubyReactor.Success(nil) })
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# `[arguments, nil]`, or `[{}, ArgumentResolutionError]` for the caller
|
|
143
|
+
# to fail the step with once its `:start_step` has fired.
|
|
144
|
+
def resolve_before_start(step_config, deferred_body)
|
|
145
|
+
return [{}, nil] if deferred_body
|
|
146
|
+
|
|
147
|
+
[step_config.resolve_arguments(@context), nil]
|
|
148
|
+
rescue Error::ArgumentResolutionError => e
|
|
149
|
+
[{}, e]
|
|
150
|
+
end
|
|
151
|
+
|
|
123
152
|
# The reactor's single hand-off point, `{ mode: :after|:before, step: }`.
|
|
124
153
|
# Nil for a reactor that never declares `background`.
|
|
125
154
|
def background_handoff
|
|
@@ -130,23 +159,20 @@ module RubyReactor
|
|
|
130
159
|
end
|
|
131
160
|
|
|
132
161
|
# Hand-off is keyed to REACHING the named step, not to where the
|
|
133
|
-
# declaration sits in the class body.
|
|
134
|
-
# must not run never triggers it — the hand-off only ever relocates work
|
|
135
|
-
# that is actually going to happen. Inside the worker the whole thing is
|
|
162
|
+
# declaration sits in the class body. Inside the worker the whole thing is
|
|
136
163
|
# suppressed (`inline_async_execution`) so it cannot re-trigger.
|
|
137
164
|
def handoff_at?(step_config, mode)
|
|
138
165
|
point = background_handoff
|
|
139
|
-
|
|
140
|
-
return false if @context.inline_async_execution
|
|
141
|
-
|
|
142
|
-
step_config.should_run?(@context)
|
|
166
|
+
point && point[:mode] == mode && point[:step] == step_config.name && !@context.inline_async_execution
|
|
143
167
|
end
|
|
144
168
|
|
|
145
|
-
# Post-execution trigger for `after:`. Only a
|
|
146
|
-
#
|
|
147
|
-
#
|
|
169
|
+
# Post-execution trigger for `after:`. Only a result the run continues
|
|
170
|
+
# from hands off: a Success, or a Skipped, which is only an
|
|
171
|
+
# instrumentation mark and never changes execution. A Halt stops the run,
|
|
172
|
+
# and a Failure, interrupt, queued retry or already-async result each own
|
|
173
|
+
# the flow instead.
|
|
148
174
|
def handoff_after?(step_config, result)
|
|
149
|
-
return false unless result.is_a?(RubyReactor::Success) && !result.is_a?(RubyReactor::
|
|
175
|
+
return false unless result.is_a?(RubyReactor::Success) && !result.is_a?(RubyReactor::Halt)
|
|
150
176
|
|
|
151
177
|
handoff_at?(step_config, :after)
|
|
152
178
|
end
|
|
@@ -175,7 +201,7 @@ module RubyReactor
|
|
|
175
201
|
end
|
|
176
202
|
|
|
177
203
|
def execute_step_with_retry(step_config, resolved_arguments = nil)
|
|
178
|
-
resolved_arguments ||= resolve_arguments(
|
|
204
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
179
205
|
result = @retry_manager.execute_with_retry(step_config, @reactor_class) do
|
|
180
206
|
safe_execute_step_sync(step_config, resolved_arguments)
|
|
181
207
|
end
|
|
@@ -188,7 +214,7 @@ module RubyReactor
|
|
|
188
214
|
end
|
|
189
215
|
|
|
190
216
|
def safe_execute_step_sync(step_config, resolved_arguments = nil)
|
|
191
|
-
resolved_arguments ||= resolve_arguments(
|
|
217
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
192
218
|
execute_step_sync_without_result_handling(step_config, resolved_arguments)
|
|
193
219
|
rescue Error::InputValidationError => e
|
|
194
220
|
# Validation failures are not retryable and must surface as a structured
|
|
@@ -216,10 +242,12 @@ module RubyReactor
|
|
|
216
242
|
rescue Error::ExecutionParked
|
|
217
243
|
@context.retry_context.decrement_attempt_for_step(step_config.name)
|
|
218
244
|
raise
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
245
|
+
# Arguments raised: the body never started. Not retried (the same
|
|
246
|
+
# inputs fail the same way), never compensated.
|
|
247
|
+
rescue Error::ArgumentResolutionError => e
|
|
248
|
+
pre_body_failure(step_config, e)
|
|
249
|
+
# Any other exception, standard or not, is this step's failure (008 R-16).
|
|
250
|
+
rescue Error::Rescuable => e
|
|
223
251
|
RubyReactor::Failure(
|
|
224
252
|
e,
|
|
225
253
|
step_name: step_config.name,
|
|
@@ -230,6 +258,16 @@ module RubyReactor
|
|
|
230
258
|
)
|
|
231
259
|
end
|
|
232
260
|
|
|
261
|
+
def pre_body_failure(step_config, error)
|
|
262
|
+
RubyReactor::Failure(error, step_name: step_config.name, reactor_name: @reactor_class.name,
|
|
263
|
+
inputs: @context.inputs, redact_inputs: redact_inputs, step_arguments: {},
|
|
264
|
+
retryable: false, exception_class: error.exception_class)
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def redact_inputs
|
|
268
|
+
@reactor_class.inputs.select { |_, config| config[:redact] }.keys
|
|
269
|
+
end
|
|
270
|
+
|
|
233
271
|
# Contention (US3). Synchronously there is no queue to park into: the
|
|
234
272
|
# step fails. In a worker the execution parks at this step — by raising
|
|
235
273
|
# `Error::StepContentionPark`, which every executor on the stack lets
|
|
@@ -315,14 +353,8 @@ module RubyReactor
|
|
|
315
353
|
|
|
316
354
|
def execute_step_sync(step_config, resolved_arguments = nil)
|
|
317
355
|
@context.with_step(step_config.name) do
|
|
318
|
-
# Check conditions and guards
|
|
319
|
-
unless step_config.should_run?(@context)
|
|
320
|
-
@dependency_graph.complete_step(step_config.name)
|
|
321
|
-
return RubyReactor.Success(nil)
|
|
322
|
-
end
|
|
323
|
-
|
|
324
356
|
# Resolve arguments
|
|
325
|
-
resolved_arguments ||= resolve_arguments(
|
|
357
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
326
358
|
|
|
327
359
|
# Validate arguments if validator is defined
|
|
328
360
|
validate_step_arguments(step_config, resolved_arguments)
|
|
@@ -338,14 +370,8 @@ module RubyReactor
|
|
|
338
370
|
# Execute step without handling the result (used during retries)
|
|
339
371
|
def execute_step_sync_without_result_handling(step_config, resolved_arguments = nil)
|
|
340
372
|
@context.with_step(step_config.name) do
|
|
341
|
-
# Check conditions and guards
|
|
342
|
-
unless step_config.should_run?(@context)
|
|
343
|
-
@dependency_graph.complete_step(step_config.name)
|
|
344
|
-
return RubyReactor.Success(nil)
|
|
345
|
-
end
|
|
346
|
-
|
|
347
373
|
# Resolve arguments
|
|
348
|
-
resolved_arguments ||= resolve_arguments(
|
|
374
|
+
resolved_arguments ||= step_config.resolve_arguments(@context)
|
|
349
375
|
|
|
350
376
|
yield resolved_arguments if block_given?
|
|
351
377
|
|
|
@@ -435,22 +461,6 @@ module RubyReactor
|
|
|
435
461
|
raise error
|
|
436
462
|
end
|
|
437
463
|
|
|
438
|
-
def resolve_arguments(step_config)
|
|
439
|
-
resolved = {}
|
|
440
|
-
|
|
441
|
-
step_config.arguments.each do |arg_name, arg_config|
|
|
442
|
-
source = arg_config[:source]
|
|
443
|
-
transform = arg_config[:transform]
|
|
444
|
-
|
|
445
|
-
value = source.resolve(@context)
|
|
446
|
-
value = transform.call(value) if transform
|
|
447
|
-
|
|
448
|
-
resolved[arg_name] = value
|
|
449
|
-
end
|
|
450
|
-
|
|
451
|
-
resolved
|
|
452
|
-
end
|
|
453
|
-
|
|
454
464
|
def run_step_implementation(step_config, arguments)
|
|
455
465
|
contract = step_config.input_contract
|
|
456
466
|
@context.append_execution_trace(
|
|
@@ -152,11 +152,14 @@ module RubyReactor
|
|
|
152
152
|
park_held_primitives! if @context.inline_async_execution
|
|
153
153
|
@contention_snooze = true
|
|
154
154
|
raise
|
|
155
|
-
rescue
|
|
156
|
-
@result = @result_handler.handle_execution_error(e)
|
|
155
|
+
rescue Error::Rescuable => e
|
|
156
|
+
@result = aborting_on_interruption { @result_handler.handle_execution_error(e) }
|
|
157
157
|
update_context_status(@result)
|
|
158
158
|
completed = true
|
|
159
159
|
@result
|
|
160
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
161
|
+
mark_aborted
|
|
162
|
+
raise
|
|
160
163
|
ensure
|
|
161
164
|
release_locks unless @parked
|
|
162
165
|
leave_ordered_lock_scope
|
|
@@ -239,6 +242,7 @@ module RubyReactor
|
|
|
239
242
|
@context.admit!
|
|
240
243
|
prepare_for_resume
|
|
241
244
|
save_context
|
|
245
|
+
@past_gates = true
|
|
242
246
|
|
|
243
247
|
@result = if @context.current_step
|
|
244
248
|
execute_current_step_and_continue
|
|
@@ -269,11 +273,14 @@ module RubyReactor
|
|
|
269
273
|
park_held_primitives!
|
|
270
274
|
@contention_snooze = true
|
|
271
275
|
raise e
|
|
272
|
-
rescue
|
|
273
|
-
handle_resume_error(e)
|
|
276
|
+
rescue Error::Rescuable => e
|
|
277
|
+
aborting_on_interruption { handle_resume_error(e) }
|
|
274
278
|
update_context_status(@result)
|
|
275
279
|
completed = true
|
|
276
280
|
@result
|
|
281
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
282
|
+
mark_aborted
|
|
283
|
+
raise
|
|
277
284
|
ensure
|
|
278
285
|
release_locks unless @parked
|
|
279
286
|
@acquired_context_lock&.release
|
|
@@ -288,6 +295,33 @@ module RubyReactor
|
|
|
288
295
|
@compensation_manager.rollback_completed_steps
|
|
289
296
|
end
|
|
290
297
|
|
|
298
|
+
# True once `resume_execution` is past its reactor-level gates (context
|
|
299
|
+
# lock, lock, semaphore, period): from there on it may have run steps.
|
|
300
|
+
def past_gates?
|
|
301
|
+
@past_gates == true
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
# Reached only by an interruption — a signal, an exit, out of memory, an
|
|
305
|
+
# enclosing timeout (008 R-08, R-16); every other exception was rescued as
|
|
306
|
+
# `Error::Rescuable` and rolled back. Running user rollback code now is
|
|
307
|
+
# unsafe, so none runs: a run in the caller's process is recorded
|
|
308
|
+
# `aborted`, with the undo entries not yet undone kept, for a manual
|
|
309
|
+
# `Reactor#undo`; the `ensure` persists it, best effort. A worker run stays
|
|
310
|
+
# `running` and its job is redelivered.
|
|
311
|
+
def mark_aborted
|
|
312
|
+
@context.status = :aborted unless @context.inline_async_execution
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
# A rollback run from a `rescue Error::Rescuable` body is outside the
|
|
316
|
+
# sibling `rescue Exception`, which never sees what that body raises: an
|
|
317
|
+
# interruption there must mark the run here.
|
|
318
|
+
def aborting_on_interruption
|
|
319
|
+
yield
|
|
320
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
321
|
+
mark_aborted unless Error::Rescuable === e # rubocop:disable Style/CaseEquality
|
|
322
|
+
raise
|
|
323
|
+
end
|
|
324
|
+
|
|
291
325
|
def undo_stack
|
|
292
326
|
@compensation_manager.undo_stack
|
|
293
327
|
end
|
|
@@ -5,6 +5,12 @@ module RubyReactor
|
|
|
5
5
|
class Collector
|
|
6
6
|
extend Helpers
|
|
7
7
|
|
|
8
|
+
# Seconds a collector waits for another collector of the same map. One
|
|
9
|
+
# that found the map unsettled releases within milliseconds, and may be
|
|
10
|
+
# the only trigger left for a failure it just deferred (R-04); a holder
|
|
11
|
+
# busy longer is the one applying the map's result.
|
|
12
|
+
COLLECT_LOCK_WAIT = 2
|
|
13
|
+
|
|
8
14
|
def self.perform(arguments)
|
|
9
15
|
arguments = arguments.transform_keys(&:to_sym)
|
|
10
16
|
map_id = arguments[:map_id]
|
|
@@ -30,7 +36,7 @@ module RubyReactor
|
|
|
30
36
|
lock = RubyReactor::Lock.new(
|
|
31
37
|
"map_collect:#{map_id}",
|
|
32
38
|
owner: SecureRandom.uuid, ttl: RubyReactor.configuration.context_lock_ttl,
|
|
33
|
-
wait:
|
|
39
|
+
wait: COLLECT_LOCK_WAIT, auto_extend: true
|
|
34
40
|
)
|
|
35
41
|
lock.acquire
|
|
36
42
|
lock
|
|
@@ -56,8 +62,9 @@ module RubyReactor
|
|
|
56
62
|
|
|
57
63
|
# Idempotency: if the parent already recorded this map step's result, a
|
|
58
64
|
# prior collector already resumed it. Re-resuming would double-execute the
|
|
59
|
-
# steps after the map. Skip.
|
|
60
|
-
|
|
65
|
+
# steps after the map. Skip. A parent already finished (a prior
|
|
66
|
+
# collector applied this map's failure) must not be rolled back twice.
|
|
67
|
+
return if parent_context.intermediate_results.key?(step_name.to_sym) || parent_context.finished?
|
|
61
68
|
|
|
62
69
|
# Check if all tasks are completed
|
|
63
70
|
metadata = storage.retrieve_map_metadata(map_id, parent_reactor_class_name)
|
|
@@ -65,18 +72,21 @@ module RubyReactor
|
|
|
65
72
|
|
|
66
73
|
results_count = storage.count_map_results(map_id, parent_reactor_class_name)
|
|
67
74
|
|
|
68
|
-
#
|
|
69
|
-
#
|
|
70
|
-
#
|
|
71
|
-
#
|
|
72
|
-
#
|
|
75
|
+
# Completion is judged against the total count of elements, not
|
|
76
|
+
# map_offset (which batching reservation can push past it).
|
|
77
|
+
#
|
|
78
|
+
# A fail-fast failure is applied only once every index has settled
|
|
79
|
+
# (a result, `_error`, `_halt` or `_skipped` slot), so the map's
|
|
80
|
+
# compensate sees every element that completed — including ones still
|
|
81
|
+
# in flight when the failure happened (R-04). Until then the last
|
|
82
|
+
# element to settle, or the map sweeper, re-triggers this collector.
|
|
83
|
+
return if results_count < total_count
|
|
84
|
+
|
|
73
85
|
if (failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name))
|
|
74
86
|
handle_failure(failed_context_id, metadata, storage, parent_context, step_name)
|
|
75
87
|
return
|
|
76
88
|
end
|
|
77
89
|
|
|
78
|
-
return if results_count < total_count
|
|
79
|
-
|
|
80
90
|
# Retrieve results lazily
|
|
81
91
|
results = RubyReactor::Map::ResultEnumerator.new(
|
|
82
92
|
map_id,
|
|
@@ -114,7 +124,7 @@ module RubyReactor
|
|
|
114
124
|
# Pass Enumerator to collect block
|
|
115
125
|
collected = collect_block.call(results)
|
|
116
126
|
RubyReactor::Success(collected)
|
|
117
|
-
rescue
|
|
127
|
+
rescue RubyReactor::Error::Rescuable => e
|
|
118
128
|
RubyReactor.configuration.logger.error("Map collect block raised: #{e.message}")
|
|
119
129
|
RubyReactor.configuration.logger.error(e.backtrace.join("\n")) if e.backtrace
|
|
120
130
|
RubyReactor::Failure(e)
|
|
@@ -65,9 +65,8 @@ module RubyReactor
|
|
|
65
65
|
reactor_class_name = arguments[:parent_reactor_class_name]
|
|
66
66
|
|
|
67
67
|
# Fail Fast Check
|
|
68
|
-
if arguments[:fail_fast]
|
|
69
|
-
|
|
70
|
-
return if failed_context_id
|
|
68
|
+
if arguments[:fail_fast] && storage.retrieve_map_failed_context_id(map_id, reactor_class_name)
|
|
69
|
+
return settle_undispatched(arguments, storage)
|
|
71
70
|
end
|
|
72
71
|
|
|
73
72
|
batch_size = arguments[:batch_size] || source.size # Default to all if no batch_size (async=true only)
|
|
@@ -104,6 +103,33 @@ module RubyReactor
|
|
|
104
103
|
end
|
|
105
104
|
end
|
|
106
105
|
|
|
106
|
+
# A fail-fast map stopped dispatching: claim every index not dispatched
|
|
107
|
+
# yet (one offset bump, so a later dispatcher claims none), settle each
|
|
108
|
+
# with a `_skipped` slot, count them down, and trigger the collector if
|
|
109
|
+
# that settled the map. Without this those indices never settle, so the
|
|
110
|
+
# failure would never be applied (R-04) and the map sweeper would keep
|
|
111
|
+
# re-dispatching them.
|
|
112
|
+
def self.settle_undispatched(arguments, storage)
|
|
113
|
+
map_id = arguments[:map_id]
|
|
114
|
+
reactor_class_name = arguments[:parent_reactor_class_name]
|
|
115
|
+
total = storage.retrieve_map_metadata(map_id, reactor_class_name)&.fetch("count", 0).to_i
|
|
116
|
+
new_offset = storage.increment_map_offset(map_id, total, reactor_class_name)
|
|
117
|
+
claimed = (new_offset - total)...[new_offset, total].min
|
|
118
|
+
return if claimed.none?
|
|
119
|
+
|
|
120
|
+
claimed.each do |index|
|
|
121
|
+
storage.store_map_result(map_id, index, { "_skipped" => true }, reactor_class_name,
|
|
122
|
+
strict_ordering: arguments[:strict_ordering])
|
|
123
|
+
end
|
|
124
|
+
return unless storage.decrement_map_counter_by(map_id, claimed.size, reactor_class_name) <= 0
|
|
125
|
+
|
|
126
|
+
RubyReactor.configuration.async_router.perform_map_collection_async(
|
|
127
|
+
parent_context_id: arguments[:parent_context_id], map_id: map_id,
|
|
128
|
+
parent_reactor_class_name: reactor_class_name, step_name: arguments[:step_name].to_s,
|
|
129
|
+
strict_ordering: arguments[:strict_ordering], timeout: 3600
|
|
130
|
+
)
|
|
131
|
+
end
|
|
132
|
+
|
|
107
133
|
# Re-dispatch a SPECIFIC index whose result slot is missing (Phase 5c, used
|
|
108
134
|
# by the map sweeper). Index-driven rather than offset-driven: resolve the
|
|
109
135
|
# source from the stored parent context and pick source[index]. Idempotent
|
|
@@ -55,11 +55,13 @@ module RubyReactor
|
|
|
55
55
|
context.inline_async_execution = true
|
|
56
56
|
|
|
57
57
|
storage = RubyReactor.configuration.storage_adapter
|
|
58
|
+
return if check_fail_fast?(arguments, storage)
|
|
59
|
+
|
|
60
|
+
# Indexed only once it runs: a skipped element saves no context, and
|
|
61
|
+
# the map's rollback reports an indexed element without one as expired.
|
|
58
62
|
storage.store_map_element_context_id(arguments[:map_id], context.context_id,
|
|
59
63
|
arguments[:parent_reactor_class_name])
|
|
60
64
|
|
|
61
|
-
return if check_fail_fast?(arguments, storage)
|
|
62
|
-
|
|
63
65
|
executor = Executor.new(context.reactor_class, {}, context)
|
|
64
66
|
begin
|
|
65
67
|
arguments[:serialized_context] ? executor.resume_execution : executor.execute
|
|
@@ -151,7 +153,11 @@ module RubyReactor
|
|
|
151
153
|
failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name)
|
|
152
154
|
return false unless failed_context_id
|
|
153
155
|
|
|
154
|
-
# Skip execution
|
|
156
|
+
# Skip execution, but settle the index: the collector applies a
|
|
157
|
+
# fail-fast failure only once every index has a result slot (R-04), and
|
|
158
|
+
# the map sweeper would otherwise keep re-dispatching this one.
|
|
159
|
+
storage.store_map_result(map_id, arguments[:index], { "_skipped" => true }, parent_reactor_class_name,
|
|
160
|
+
strict_ordering: arguments[:strict_ordering])
|
|
155
161
|
finalize_execution(arguments, storage)
|
|
156
162
|
true
|
|
157
163
|
end
|