ruby_reactor 0.8.5 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/.specify/memory/constitution.md +17 -14
- data/CHANGELOG.md +58 -0
- data/CLAUDE.md +1 -1
- data/README.md +23 -8
- data/lib/ruby_reactor/adapters/active_job/map_element_rollback_worker.rb +20 -0
- data/lib/ruby_reactor/adapters/active_job/router.rb +32 -4
- data/lib/ruby_reactor/adapters/sidekiq/map_element_rollback_worker.rb +16 -0
- data/lib/ruby_reactor/adapters/sidekiq/router.rb +32 -4
- data/lib/ruby_reactor/context.rb +14 -1
- data/lib/ruby_reactor/dsl/definition_warnings.rb +30 -0
- data/lib/ruby_reactor/dsl/map_builder.rb +27 -7
- data/lib/ruby_reactor/dsl/step_builder.rb +8 -21
- data/lib/ruby_reactor/error/rescuable.rb +6 -1
- data/lib/ruby_reactor/error/rollback_handed_off.rb +27 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +33 -9
- data/lib/ruby_reactor/executor/result_handler.rb +72 -52
- data/lib/ruby_reactor/executor/retry_manager.rb +4 -3
- data/lib/ruby_reactor/executor/step_executor.rb +3 -1
- data/lib/ruby_reactor/executor.rb +186 -6
- data/lib/ruby_reactor/map/collector.rb +25 -93
- data/lib/ruby_reactor/map/dispatcher.rb +38 -7
- data/lib/ruby_reactor/map/element_executor.rb +14 -10
- data/lib/ruby_reactor/map/element_rollback.rb +158 -0
- data/lib/ruby_reactor/map/helpers.rb +11 -155
- data/lib/ruby_reactor/map/step_rollback.rb +136 -0
- data/lib/ruby_reactor/map/sweeper.rb +66 -8
- data/lib/ruby_reactor/map.rb +17 -0
- data/lib/ruby_reactor/max_retries_exhausted_failure.rb +4 -2
- data/lib/ruby_reactor/reactor.rb +41 -2
- data/lib/ruby_reactor/rspec/active_job_helpers.rb +13 -2
- data/lib/ruby_reactor/rspec/matchers.rb +40 -21
- data/lib/ruby_reactor/rspec/sidekiq_helpers.rb +7 -3
- data/lib/ruby_reactor/rspec/test_subject.rb +13 -3
- data/lib/ruby_reactor/rspec.rb +3 -6
- data/lib/ruby_reactor/step/compose_step.rb +3 -0
- data/lib/ruby_reactor/step/map_step.rb +117 -114
- data/lib/ruby_reactor/storage/adapter.rb +14 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +18 -2
- data/lib/ruby_reactor/storage/redis_map_rollback.rb +153 -0
- data/lib/ruby_reactor/storage/redis_reactor_scan.rb +1 -1
- data/lib/ruby_reactor/sweeper.rb +3 -1
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +11 -1
- data/lib/ruby_reactor/web/public/assets/{index-CQbgHtd0.js → index-C6ejszCQ.js} +12 -12
- data/lib/ruby_reactor/web/public/assets/index-DM5darhy.css +1 -0
- data/lib/ruby_reactor/web/public/index.html +2 -2
- data/lib/ruby_reactor/worker.rb +4 -2
- data/specs/007-execution-flow-analysis/evidence/harness.rb +2 -0
- data/specs/007-execution-flow-analysis/evidence/probes/03_map.rb +4 -1
- data/specs/009-distributed-map-undo/checklists/requirements.md +53 -0
- data/specs/009-distributed-map-undo/contracts/api-surface.md +125 -0
- data/specs/009-distributed-map-undo/contracts/rollback-protocol.md +157 -0
- data/specs/009-distributed-map-undo/data-model.md +151 -0
- data/specs/009-distributed-map-undo/plan.md +323 -0
- data/specs/009-distributed-map-undo/pr-description.md +78 -0
- data/specs/009-distributed-map-undo/quickstart.md +154 -0
- data/specs/009-distributed-map-undo/research.md +585 -0
- data/specs/009-distributed-map-undo/spec.md +360 -0
- data/specs/009-distributed-map-undo/tasks.md +1060 -0
- data/specs/future_improvements.md +18 -30
- metadata +21 -3
- data/lib/ruby_reactor/web/public/assets/index-BQvIWPdx.css +0 -1
|
@@ -12,11 +12,16 @@ module RubyReactor
|
|
|
12
12
|
# raises — rescuing that would stop the caller's timeout from ever firing.
|
|
13
13
|
# Running rollback code during any of them is unsafe; the executor marks
|
|
14
14
|
# the run `aborted` for a manual undo instead.
|
|
15
|
+
#
|
|
16
|
+
# Nor does it match `RollbackHandedOff`, a control signal (009 R-04): it
|
|
17
|
+
# must pass the `rescue Error::Rescuable` sites between `MapStep#undo` and
|
|
18
|
+
# the executor (`CompensationManager#undo_step`, `StepExecutor`), which
|
|
19
|
+
# would otherwise record it as an undo failure.
|
|
15
20
|
module Rescuable
|
|
16
21
|
INTERRUPTIONS = [SignalException, SystemExit, NoMemoryError].freeze
|
|
17
22
|
|
|
18
23
|
def self.===(exception)
|
|
19
|
-
exception.is_a?(Exception) && !interruption?(exception)
|
|
24
|
+
exception.is_a?(Exception) && !interruption?(exception) && !exception.is_a?(RollbackHandedOff)
|
|
20
25
|
end
|
|
21
26
|
|
|
22
27
|
def self.interruption?(exception)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
module Error
|
|
5
|
+
# A control signal, never a failure (009 R-04): a fan-out map started its
|
|
6
|
+
# distributed rollback and must wait for its element jobs. Raised from
|
|
7
|
+
# `MapStep#compensate` / `#undo`, it unwinds the rollback without popping
|
|
8
|
+
# the map's undo entry (the resume cursor), passes every `rescue
|
|
9
|
+
# Error::Rescuable` (see `Rescuable.===`), and is rescued only by
|
|
10
|
+
# `Executor` and `Reactor#undo`, which record where the rollback stands,
|
|
11
|
+
# save, and hand the run off as `rolling_back`.
|
|
12
|
+
#
|
|
13
|
+
# `failure` carries the Failure the run will end with, minus its rollback
|
|
14
|
+
# failures, set by the `ResultHandler` it passes through and consumed by
|
|
15
|
+
# the executor of that level.
|
|
16
|
+
class RollbackHandedOff < Base
|
|
17
|
+
attr_reader :map_id, :reactor_class_name
|
|
18
|
+
attr_accessor :failure
|
|
19
|
+
|
|
20
|
+
def initialize(map_id:, reactor_class_name: nil, message: "rollback handed off at map #{map_id}")
|
|
21
|
+
super(message)
|
|
22
|
+
@map_id = map_id
|
|
23
|
+
@reactor_class_name = reactor_class_name
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -30,11 +30,23 @@ module RubyReactor
|
|
|
30
30
|
# (005 FR-004). `ResultHandler` attaches it to the final Failure.
|
|
31
31
|
attr_reader :undo_trace, :rollback_failures
|
|
32
32
|
|
|
33
|
+
# The step failure being rolled back: `{ step:, error:, compensated:,
|
|
34
|
+
# compensation_error: }`. A rollback that hands off (009 R-04) records it,
|
|
35
|
+
# so the resume knows whether that step's own compensate still runs.
|
|
36
|
+
attr_reader :pending
|
|
37
|
+
|
|
38
|
+
# Rollback failures collected before a hand-off, saved serialized on
|
|
39
|
+
# the context's `rollback` and restored on resume.
|
|
40
|
+
def restore_rollback_failures(list)
|
|
41
|
+
@rollback_failures.replace(Array(ContextSerializer.deserialize_value(list)))
|
|
42
|
+
end
|
|
43
|
+
|
|
33
44
|
def add_to_undo_stack(step_info)
|
|
34
45
|
@context.undo_stack << step_info
|
|
35
46
|
end
|
|
36
47
|
|
|
37
48
|
def handle_step_failure(step_config, error, arguments)
|
|
49
|
+
@pending = { step: step_config.name, error: error, compensated: false }
|
|
38
50
|
# A step whose OWN coordination acquisition failed (contention, or a
|
|
39
51
|
# bad key proc) never ran its body — "no step compensates, the
|
|
40
52
|
# contended step's work has not been attempted" (US3-1/T018), which
|
|
@@ -44,29 +56,38 @@ module RubyReactor
|
|
|
44
56
|
# turning a plain contention failure into a confusing
|
|
45
57
|
# CompensationError. Prior steps still roll back normally.
|
|
46
58
|
if step_never_started?(error)
|
|
59
|
+
@pending[:compensated] = true
|
|
47
60
|
rollback_completed_steps
|
|
48
|
-
return RubyReactor.Failure(
|
|
61
|
+
return RubyReactor.Failure(step_failure_message(step_config, error))
|
|
49
62
|
end
|
|
50
63
|
|
|
51
64
|
# Try compensation
|
|
52
65
|
compensation_result = compensate_step(step_config, error, arguments)
|
|
66
|
+
@pending[:compensated] = true
|
|
67
|
+
@pending[:compensation_error] = compensation_error_message(step_config, compensation_result)
|
|
53
68
|
case compensation_result
|
|
54
69
|
when RubyReactor::Success
|
|
55
70
|
# Compensation succeeded, continue with rollback
|
|
56
71
|
rollback_completed_steps
|
|
57
|
-
RubyReactor.Failure(
|
|
72
|
+
RubyReactor.Failure(step_failure_message(step_config, error))
|
|
58
73
|
when RubyReactor::Failure
|
|
59
74
|
# Compensation failed, this is more serious
|
|
60
75
|
rollback_completed_steps
|
|
61
|
-
raise Error::CompensationError.new(
|
|
62
|
-
|
|
63
|
-
step: step_config.name,
|
|
64
|
-
context: @context,
|
|
65
|
-
original_error: error
|
|
66
|
-
)
|
|
76
|
+
raise Error::CompensationError.new(@pending[:compensation_error], step: step_config.name, context: @context,
|
|
77
|
+
original_error: error)
|
|
67
78
|
end
|
|
68
79
|
end
|
|
69
80
|
|
|
81
|
+
def step_failure_message(step_config, error)
|
|
82
|
+
"Step '#{step_config.name}' failed: #{error}"
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def compensation_error_message(step_config, compensation_result)
|
|
86
|
+
return unless compensation_result.is_a?(RubyReactor::Failure)
|
|
87
|
+
|
|
88
|
+
"Compensation for step '#{step_config.name}' failed: #{compensation_result.error}"
|
|
89
|
+
end
|
|
90
|
+
|
|
70
91
|
# A unit that compensates itself outside the executor loop (StepWorker's
|
|
71
92
|
# `async_step`): the same coordination re-take, trace, middleware events
|
|
72
93
|
# and `rollback_failures` as a step compensated here.
|
|
@@ -77,7 +98,9 @@ module RubyReactor
|
|
|
77
98
|
# Newest first. Each entry leaves the stack only once its undo returned,
|
|
78
99
|
# so an interruption mid-rollback leaves exactly the entries still to
|
|
79
100
|
# undo — the interrupted one included — for a manual undo (008 R-16).
|
|
80
|
-
|
|
101
|
+
# `after_pop` runs after each pop: an element rollback job saves there,
|
|
102
|
+
# so a redelivery resumes after the last undone entry (009 R-07).
|
|
103
|
+
def rollback_completed_steps(&after_pop)
|
|
81
104
|
until undo_stack.empty?
|
|
82
105
|
step_info = undo_stack.last
|
|
83
106
|
result = @context.with_step(step_info[:step].name) do
|
|
@@ -86,6 +109,7 @@ module RubyReactor
|
|
|
86
109
|
undo_stack.pop
|
|
87
110
|
@undo_trace << { type: :undo, step: step_info[:step].name, result: result,
|
|
88
111
|
arguments: step_info[:arguments] }
|
|
112
|
+
after_pop&.call
|
|
89
113
|
end
|
|
90
114
|
end
|
|
91
115
|
|
|
@@ -35,9 +35,13 @@ module RubyReactor
|
|
|
35
35
|
# Every reactor-level Failure that follows a rollback is built here, so
|
|
36
36
|
# this is the one place `rollback_failures` is attached (005 R-08).
|
|
37
37
|
def handle_execution_error(error)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
with_rollback_failures(build_execution_failure(error))
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# The end of a rollback that handed off (009 R-04): the Failure recorded
|
|
42
|
+
# at the hand-off, with every rollback failure collected since.
|
|
43
|
+
def handed_off_failure(failure)
|
|
44
|
+
with_rollback_failures(failure)
|
|
41
45
|
end
|
|
42
46
|
|
|
43
47
|
def final_result(reactor_class)
|
|
@@ -51,34 +55,62 @@ module RubyReactor
|
|
|
51
55
|
|
|
52
56
|
private
|
|
53
57
|
|
|
58
|
+
def with_rollback_failures(failure)
|
|
59
|
+
failure.rollback_failures.concat(@compensation_manager.rollback_failures) if failure.is_a?(RubyReactor::Failure)
|
|
60
|
+
failure
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Every error rolls back the completed steps first (Constitution II), then
|
|
64
|
+
# becomes the run's Failure.
|
|
54
65
|
def build_execution_failure(error)
|
|
66
|
+
if error.is_a?(Error::StepFailureError)
|
|
67
|
+
current_context = error.context || @context
|
|
68
|
+
current_context.current_step = error.step
|
|
69
|
+
store_failed_map_context(current_context) if current_context.map_metadata
|
|
70
|
+
end
|
|
71
|
+
handing_off_as(error) { @compensation_manager.rollback_completed_steps }
|
|
72
|
+
failure_for(error)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# A rollback that hands off mid-way (009 R-04) carries the Failure this
|
|
76
|
+
# run will end with, minus its rollback failures, to the executor that
|
|
77
|
+
# records the hand-off. `error` is what the run fails with once the
|
|
78
|
+
# rollback is done (a lambda when it is costly or not built yet).
|
|
79
|
+
def handing_off_as(error)
|
|
80
|
+
yield
|
|
81
|
+
rescue Error::RollbackHandedOff => e
|
|
82
|
+
e.failure ||= failure_for(error.respond_to?(:call) ? error.call : error)
|
|
83
|
+
raise
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def failure_for(error)
|
|
55
87
|
case error
|
|
56
88
|
when Error::StepFailureError
|
|
57
|
-
|
|
89
|
+
create_failure_from_error(error, redacted_input_names(error.context))
|
|
58
90
|
when Error::InputValidationError
|
|
59
91
|
# Unified validation failure shape (inputs, step args, step output).
|
|
60
|
-
# Roll back any completed steps so saga semantics hold for mid-reactor
|
|
61
|
-
# validation failures (a no-op for input validation at reactor start).
|
|
62
|
-
@compensation_manager.rollback_completed_steps
|
|
63
92
|
build_validation_failure(error)
|
|
64
93
|
when Error::Base
|
|
65
|
-
#
|
|
66
|
-
# whose compensation failed.
|
|
67
|
-
@compensation_manager.rollback_completed_steps
|
|
94
|
+
# A `CompensationError` names the step whose compensation failed.
|
|
68
95
|
RubyReactor.Failure("Execution error: #{error.message}", exception_class: error.class.name,
|
|
69
96
|
step_name: error.step || @context.current_step,
|
|
70
97
|
reactor_name: @context.reactor_class&.name)
|
|
71
98
|
else
|
|
72
99
|
# Any other StandardError after completed work (a checkpoint write,
|
|
73
|
-
# a hook) still leaves a partial saga
|
|
74
|
-
# (Constitution II, 008 R-07)
|
|
75
|
-
@compensation_manager.rollback_completed_steps
|
|
100
|
+
# a hook) still leaves a partial saga, rolled back like any failure
|
|
101
|
+
# (Constitution II, 008 R-07); name the step that was executing.
|
|
76
102
|
RubyReactor.Failure("Execution failed: #{error.message}", exception_class: error.class.name,
|
|
77
103
|
step_name: @context.current_step,
|
|
78
104
|
reactor_name: @context.reactor_class&.name)
|
|
79
105
|
end
|
|
80
106
|
end
|
|
81
107
|
|
|
108
|
+
def redacted_input_names(context)
|
|
109
|
+
return [] unless context&.reactor_class
|
|
110
|
+
|
|
111
|
+
context.reactor_class.inputs.select { |_, config| config[:redact] }.keys
|
|
112
|
+
end
|
|
113
|
+
|
|
82
114
|
# Failure for a validation error (reactor inputs, step arguments, or
|
|
83
115
|
# step output), carrying both the structured field errors and the step/
|
|
84
116
|
# reactor attribution stamped at the raise site (nil step_name for
|
|
@@ -139,7 +171,8 @@ module RubyReactor
|
|
|
139
171
|
validate_step_output(step_config, result.value, resolved_arguments)
|
|
140
172
|
@step_results[step_config.name] = result
|
|
141
173
|
if step_config.rollback_tracked?
|
|
142
|
-
@compensation_manager.add_to_undo_stack({ step: step_config,
|
|
174
|
+
@compensation_manager.add_to_undo_stack({ step: step_config,
|
|
175
|
+
arguments: step_config.rollback_arguments(resolved_arguments),
|
|
143
176
|
result: result })
|
|
144
177
|
end
|
|
145
178
|
@context.set_result(step_config.name, result.value)
|
|
@@ -156,36 +189,38 @@ module RubyReactor
|
|
|
156
189
|
|
|
157
190
|
def handle_retries_exhausted(step_config, result, resolved_arguments)
|
|
158
191
|
adopt_rollback_failures(result)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
original_error: orig_err,
|
|
163
|
-
step_arguments: resolved_arguments,
|
|
164
|
-
validation_errors: result.validation_errors)
|
|
165
|
-
if result.respond_to?(:backtrace) && result.backtrace
|
|
166
|
-
error.set_backtrace(result.backtrace)
|
|
167
|
-
elsif orig_err
|
|
168
|
-
error.set_backtrace(orig_err.backtrace)
|
|
192
|
+
error = step_failure_error(step_config, result.error, result, resolved_arguments, cause: result.original_error)
|
|
193
|
+
handing_off_as(error) do
|
|
194
|
+
@compensation_manager.handle_step_failure(step_config, result.original_error, resolved_arguments)
|
|
169
195
|
end
|
|
170
196
|
raise error
|
|
171
197
|
end
|
|
172
198
|
|
|
173
199
|
def handle_failure(step_config, result, resolved_arguments)
|
|
174
200
|
adopt_rollback_failures(result)
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
201
|
+
message = @compensation_manager.step_failure_message(step_config, result.error)
|
|
202
|
+
failure_result = handing_off_as(-> { step_failure_error(step_config, message, result, resolved_arguments) }) do
|
|
203
|
+
@compensation_manager.handle_step_failure(step_config, result.error, resolved_arguments)
|
|
204
|
+
end
|
|
205
|
+
raise step_failure_error(step_config, failure_result.error, result, resolved_arguments)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# A step that propagates another unit's failure (an async_step reader, a
|
|
209
|
+
# map adopting a fan-out element's Failure) keeps its field errors and
|
|
210
|
+
# exception class on the reactor's failure.
|
|
211
|
+
def step_failure_error(step_config, message, result, resolved_arguments, cause: result.error)
|
|
212
|
+
orig_err = cause.is_a?(Exception) ? cause : nil
|
|
213
|
+
error = Error::StepFailureError.new(message, step: step_config.name, context: @context,
|
|
214
|
+
original_error: orig_err,
|
|
215
|
+
step_arguments: resolved_arguments,
|
|
216
|
+
exception_class: (result.exception_class unless orig_err),
|
|
217
|
+
validation_errors: result.validation_errors)
|
|
183
218
|
if result.respond_to?(:backtrace) && result.backtrace
|
|
184
219
|
error.set_backtrace(result.backtrace)
|
|
185
220
|
elsif orig_err
|
|
186
221
|
error.set_backtrace(orig_err.backtrace)
|
|
187
222
|
end
|
|
188
|
-
|
|
223
|
+
error
|
|
189
224
|
end
|
|
190
225
|
|
|
191
226
|
# Wrapped first, so a returned `Step::Inputs` is validated and stored as its Hash.
|
|
@@ -193,31 +228,16 @@ module RubyReactor
|
|
|
193
228
|
success_result = RubyReactor.Success(result)
|
|
194
229
|
validate_step_output(step_config, success_result.value, resolved_arguments)
|
|
195
230
|
@step_results[step_config.name] = success_result
|
|
196
|
-
@compensation_manager.add_to_undo_stack({ step: step_config,
|
|
231
|
+
@compensation_manager.add_to_undo_stack({ step: step_config,
|
|
232
|
+
arguments: step_config.rollback_arguments(resolved_arguments),
|
|
197
233
|
result: success_result })
|
|
198
234
|
@context.set_result(step_config.name, success_result.value)
|
|
199
235
|
@dependency_graph.complete_step(step_config.name)
|
|
200
236
|
end
|
|
201
237
|
|
|
202
|
-
def handle_step_failure_error(error)
|
|
203
|
-
current_context = error.context || @context
|
|
204
|
-
current_context.current_step = error.step
|
|
205
|
-
|
|
206
|
-
store_failed_map_context(current_context) if current_context.map_metadata
|
|
207
|
-
|
|
208
|
-
@compensation_manager.rollback_completed_steps
|
|
209
|
-
|
|
210
|
-
redact_inputs = []
|
|
211
|
-
if error.context&.reactor_class
|
|
212
|
-
redact_inputs = error.context.reactor_class.inputs.select { |_, config| config[:redact] }.keys
|
|
213
|
-
end
|
|
214
|
-
|
|
215
|
-
create_failure_from_error(error, redact_inputs)
|
|
216
|
-
end
|
|
217
|
-
|
|
218
238
|
def store_failed_map_context(context)
|
|
219
239
|
return unless context.map_metadata && context.map_metadata[:map_id]
|
|
220
|
-
return unless context.map_metadata[:
|
|
240
|
+
return unless Map::Helpers.normalize_arguments(context.map_metadata)[:atomic]
|
|
221
241
|
|
|
222
242
|
storage = RubyReactor.configuration.storage_adapter
|
|
223
243
|
storage.store_map_failed_context_id(
|
|
@@ -276,7 +296,7 @@ module RubyReactor
|
|
|
276
296
|
# compensation and roll back prior steps, so the side effect is not
|
|
277
297
|
# orphaned. Then surface the structured validation error (the later
|
|
278
298
|
# rollback in handle_execution_error is a no-op — stack already clear).
|
|
279
|
-
@compensation_manager.handle_step_failure(step_config, error, resolved_arguments)
|
|
299
|
+
handing_off_as(error) { @compensation_manager.handle_step_failure(step_config, error, resolved_arguments) }
|
|
280
300
|
raise error
|
|
281
301
|
end
|
|
282
302
|
|
|
@@ -61,7 +61,7 @@ module RubyReactor
|
|
|
61
61
|
serialized_context = ContextSerializer.serialize(context_to_serialize)
|
|
62
62
|
|
|
63
63
|
if @context.map_metadata
|
|
64
|
-
map_args = @context.map_metadata
|
|
64
|
+
map_args = Map::Helpers.normalize_arguments(@context.map_metadata)
|
|
65
65
|
configuration.async_router.perform_map_element_in(
|
|
66
66
|
delay,
|
|
67
67
|
map_id: map_args[:map_id],
|
|
@@ -75,7 +75,7 @@ module RubyReactor
|
|
|
75
75
|
step_name: map_args[:step_name],
|
|
76
76
|
batch_size: map_args[:batch_size],
|
|
77
77
|
serialized_context: serialized_context,
|
|
78
|
-
|
|
78
|
+
atomic: map_args[:atomic]
|
|
79
79
|
)
|
|
80
80
|
else
|
|
81
81
|
# Persist BEFORE enqueue — the job payload is identity-only (F2). The
|
|
@@ -193,7 +193,8 @@ module RubyReactor
|
|
|
193
193
|
reactor_name: reactor_class.name,
|
|
194
194
|
step_arguments: result.respond_to?(:step_arguments) ? result.step_arguments : {},
|
|
195
195
|
validation_errors: result.validation_errors,
|
|
196
|
-
rollback_failures: result.rollback_failures
|
|
196
|
+
rollback_failures: result.rollback_failures,
|
|
197
|
+
exception_class: result.exception_class
|
|
197
198
|
)
|
|
198
199
|
end
|
|
199
200
|
|
|
@@ -133,9 +133,11 @@ module RubyReactor
|
|
|
133
133
|
# the stack. A worker run is redelivered instead, and re-runs the step.
|
|
134
134
|
def track_interrupted_construct(step_config, arguments, error)
|
|
135
135
|
return if @context.inline_async_execution || Error::Rescuable === error # rubocop:disable Style/CaseEquality
|
|
136
|
+
return if error.is_a?(Error::RollbackHandedOff) # a rollback signal, not an interruption
|
|
136
137
|
return unless step_config.undoes_partial_run?
|
|
137
138
|
|
|
138
|
-
@compensation_manager.add_to_undo_stack({ step: step_config,
|
|
139
|
+
@compensation_manager.add_to_undo_stack({ step: step_config,
|
|
140
|
+
arguments: step_config.rollback_arguments(arguments),
|
|
139
141
|
result: RubyReactor.Success(nil) })
|
|
140
142
|
end
|
|
141
143
|
|
|
@@ -135,6 +135,10 @@ module RubyReactor
|
|
|
135
135
|
handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
|
|
136
136
|
completed = true
|
|
137
137
|
@result
|
|
138
|
+
rescue Error::RollbackHandedOff => e
|
|
139
|
+
record_rollback_handoff(e)
|
|
140
|
+
completed = true
|
|
141
|
+
@result
|
|
138
142
|
rescue RubyReactor::Lock::AcquisitionError,
|
|
139
143
|
RubyReactor::Semaphore::AcquisitionError,
|
|
140
144
|
RubyReactor::RateLimit::ExceededError,
|
|
@@ -153,7 +157,7 @@ module RubyReactor
|
|
|
153
157
|
@contention_snooze = true
|
|
154
158
|
raise
|
|
155
159
|
rescue Error::Rescuable => e
|
|
156
|
-
@result = aborting_on_interruption { @result_handler.handle_execution_error(e) }
|
|
160
|
+
@result = handing_off { aborting_on_interruption { @result_handler.handle_execution_error(e) } }
|
|
157
161
|
update_context_status(@result)
|
|
158
162
|
completed = true
|
|
159
163
|
@result
|
|
@@ -184,6 +188,9 @@ module RubyReactor
|
|
|
184
188
|
end
|
|
185
189
|
|
|
186
190
|
def resume_execution # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity
|
|
191
|
+
# A composed child re-entered while its own rollback is handed off.
|
|
192
|
+
return resume_rollback if @context.rolling_back?
|
|
193
|
+
|
|
187
194
|
middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
|
|
188
195
|
completed = false
|
|
189
196
|
|
|
@@ -256,6 +263,10 @@ module RubyReactor
|
|
|
256
263
|
handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
|
|
257
264
|
completed = true
|
|
258
265
|
@result
|
|
266
|
+
rescue Error::RollbackHandedOff => e
|
|
267
|
+
record_rollback_handoff(e)
|
|
268
|
+
completed = true
|
|
269
|
+
@result
|
|
259
270
|
rescue RubyReactor::Lock::AcquisitionError,
|
|
260
271
|
RubyReactor::Semaphore::AcquisitionError,
|
|
261
272
|
RubyReactor::RateLimit::ExceededError,
|
|
@@ -274,7 +285,7 @@ module RubyReactor
|
|
|
274
285
|
@contention_snooze = true
|
|
275
286
|
raise e
|
|
276
287
|
rescue Error::Rescuable => e
|
|
277
|
-
aborting_on_interruption { handle_resume_error(e) }
|
|
288
|
+
handing_off { aborting_on_interruption { handle_resume_error(e) } }
|
|
278
289
|
update_context_status(@result)
|
|
279
290
|
completed = true
|
|
280
291
|
@result
|
|
@@ -283,16 +294,100 @@ module RubyReactor
|
|
|
283
294
|
raise
|
|
284
295
|
ensure
|
|
285
296
|
release_locks unless @parked
|
|
297
|
+
# Saved while the context lock is still held (009 R-13): released first,
|
|
298
|
+
# a worker resuming this run could load the pre-save blob, and this save
|
|
299
|
+
# would then overwrite its progress.
|
|
300
|
+
save_context unless skip_context_persist?
|
|
286
301
|
@acquired_context_lock&.release
|
|
287
302
|
@acquired_context_lock = nil
|
|
288
303
|
leave_ordered_lock_scope
|
|
304
|
+
|
|
305
|
+
emit_lifecycle_completion(completed)
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Undoes every completed step (manual undo, a composed child's undo, a
|
|
309
|
+
# map element's rollback). A rollback that hands off here (009 R-04)
|
|
310
|
+
# leaves this level's rollback failures on its context, and the next
|
|
311
|
+
# `undo_all` of the same context picks them up, so none is lost though
|
|
312
|
+
# the executor is rebuilt on resume (G2).
|
|
313
|
+
def undo_all(&after_pop)
|
|
314
|
+
saved = @context.rollback && @context.rollback["failures"]
|
|
315
|
+
@compensation_manager.restore_rollback_failures(saved) if saved
|
|
316
|
+
@compensation_manager.rollback_completed_steps(&after_pop)
|
|
317
|
+
clear_saved_rollback_failures
|
|
318
|
+
rescue Error::RollbackHandedOff
|
|
319
|
+
(@context.rollback ||= {})["failures"] =
|
|
320
|
+
ContextSerializer.serialize_value(@compensation_manager.rollback_failures)
|
|
321
|
+
raise
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# Finishes a rollback that handed off at a fan-out map (009 R-04), once
|
|
325
|
+
# the map's element rollbacks have all reported: the Worker of a
|
|
326
|
+
# `rolling_back` run, or a composed child re-entered by its root. Under
|
|
327
|
+
# the run's context lock, as `resume_execution`.
|
|
328
|
+
def resume_rollback
|
|
329
|
+
state = @context.rollback || {}
|
|
330
|
+
# The hand-off came out of a running step (a composed child rolling
|
|
331
|
+
# back inside `ComposeStep#run`): resume forward, and that step
|
|
332
|
+
# finishes its child's rollback.
|
|
333
|
+
if state["trigger"] == "step" || state.empty?
|
|
334
|
+
@context.rollback = nil
|
|
335
|
+
@context.status = :running
|
|
336
|
+
return resume_execution
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
|
|
340
|
+
completed = false
|
|
341
|
+
enter_ordered_lock_scope
|
|
342
|
+
acquire_context_lock
|
|
343
|
+
@result = finish_rollback(state)
|
|
344
|
+
@context.rollback = nil
|
|
345
|
+
update_context_status(@result)
|
|
346
|
+
completed = true
|
|
347
|
+
@result
|
|
348
|
+
rescue Error::RollbackHandedOff => e
|
|
349
|
+
record_rollback_handoff(e)
|
|
350
|
+
completed = true
|
|
351
|
+
@result
|
|
352
|
+
rescue RubyReactor::Lock::AcquisitionError => e
|
|
353
|
+
@contention_snooze = true
|
|
354
|
+
raise composed_contention_park(e) || e
|
|
355
|
+
rescue Error::Rescuable => e
|
|
356
|
+
handing_off { aborting_on_interruption { handle_resume_error(e) } }
|
|
357
|
+
update_context_status(@result)
|
|
358
|
+
completed = true
|
|
359
|
+
@result
|
|
360
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
361
|
+
mark_aborted
|
|
362
|
+
raise
|
|
363
|
+
ensure
|
|
289
364
|
save_context unless skip_context_persist?
|
|
365
|
+
@acquired_context_lock&.release
|
|
366
|
+
@acquired_context_lock = nil
|
|
367
|
+
leave_ordered_lock_scope
|
|
290
368
|
|
|
291
369
|
emit_lifecycle_completion(completed)
|
|
292
370
|
end
|
|
293
371
|
|
|
294
|
-
|
|
295
|
-
|
|
372
|
+
# The rollback's other half (009 R-05): marks the hand-off saved, then
|
|
373
|
+
# enqueues the owner's Worker if the map rollback already settled and this
|
|
374
|
+
# side claims the signal. Whichever of this and the last element rollback
|
|
375
|
+
# job sees both second enqueues the owner, once. Called right after the
|
|
376
|
+
# `rolling_back` save, while the context lock is still held.
|
|
377
|
+
def hand_off_rollback!(handed_off)
|
|
378
|
+
storage = RubyReactor.configuration.storage_adapter
|
|
379
|
+
map_id = handed_off.map_id
|
|
380
|
+
map_class = handed_off.reactor_class_name
|
|
381
|
+
storage.mark_map_rollback_handed_off(map_id, map_class)
|
|
382
|
+
RubyReactor.configuration.logger.info(
|
|
383
|
+
"event=ruby_reactor.rollback.handed_off context_id=#{@context.context_id.inspect} map_id=#{map_id.inspect}"
|
|
384
|
+
)
|
|
385
|
+
meta = storage.retrieve_map_rollback_metadata(map_id, map_class)
|
|
386
|
+
return unless meta && storage.count_map_rollback_outcomes(map_id, map_class) >= meta["total"].to_i
|
|
387
|
+
return unless storage.claim_map_rollback_signal(map_id, map_class)
|
|
388
|
+
|
|
389
|
+
RubyReactor.configuration.async_router.perform_async(@context.context_id,
|
|
390
|
+
RubyReactor.reactor_storage_name(@reactor_class))
|
|
296
391
|
end
|
|
297
392
|
|
|
298
393
|
# True once `resume_execution` is past its reactor-level gates (context
|
|
@@ -318,10 +413,18 @@ module RubyReactor
|
|
|
318
413
|
def aborting_on_interruption
|
|
319
414
|
yield
|
|
320
415
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
321
|
-
mark_aborted unless Error::Rescuable === e # rubocop:disable Style/CaseEquality
|
|
416
|
+
mark_aborted unless Error::Rescuable === e || e.is_a?(Error::RollbackHandedOff) # rubocop:disable Style/CaseEquality
|
|
322
417
|
raise
|
|
323
418
|
end
|
|
324
419
|
|
|
420
|
+
# A rollback run from a `rescue Error::Rescuable` body (the error's own
|
|
421
|
+
# rollback) is outside the sibling `rescue Error::RollbackHandedOff`.
|
|
422
|
+
def handing_off
|
|
423
|
+
yield
|
|
424
|
+
rescue Error::RollbackHandedOff => e
|
|
425
|
+
record_rollback_handoff(e)
|
|
426
|
+
end
|
|
427
|
+
|
|
325
428
|
def undo_stack
|
|
326
429
|
@compensation_manager.undo_stack
|
|
327
430
|
end
|
|
@@ -768,7 +871,8 @@ module RubyReactor
|
|
|
768
871
|
|
|
769
872
|
case result
|
|
770
873
|
when RubyReactor::DispatchResult
|
|
771
|
-
|
|
874
|
+
# A rollback hand-off returns one too, and stays `rolling_back`.
|
|
875
|
+
@context.status = :running unless @context.rolling_back?
|
|
772
876
|
when RubyReactor::Halt
|
|
773
877
|
@context.status = :halted
|
|
774
878
|
when RubyReactor::Success
|
|
@@ -833,6 +937,82 @@ module RubyReactor
|
|
|
833
937
|
@result
|
|
834
938
|
end
|
|
835
939
|
|
|
940
|
+
# Where this level's rollback stands when it hands off (009 DM §1). A
|
|
941
|
+
# composed child keeps it on its own context, inside the root's blob, and
|
|
942
|
+
# re-raises; the top-level run saves it as `rolling_back` while it still
|
|
943
|
+
# holds the context lock (I-6), runs the handshake, and returns a
|
|
944
|
+
# `DispatchResult`, as a forward hand-off does.
|
|
945
|
+
def record_rollback_handoff(handed_off)
|
|
946
|
+
failure = handed_off.failure
|
|
947
|
+
handed_off.failure = nil # this level's Failure; the level above builds its own
|
|
948
|
+
@context.rollback = rollback_state(failure)
|
|
949
|
+
@context.status = :rolling_back
|
|
950
|
+
raise handed_off if @context.root_context
|
|
951
|
+
|
|
952
|
+
save_context
|
|
953
|
+
hand_off_rollback!(handed_off)
|
|
954
|
+
@result = RubyReactor::DispatchResult.new(job_id: "map_rollback:#{handed_off.map_id}",
|
|
955
|
+
intermediate_results: @context.intermediate_results,
|
|
956
|
+
execution_id: @context.context_id)
|
|
957
|
+
end
|
|
958
|
+
|
|
959
|
+
# `trigger`: `failure` (a step failure is rolling back), `undo` (set by
|
|
960
|
+
# `Reactor#undo`), or `step` (a running step's child handed off, and no
|
|
961
|
+
# rollback started at this level). A later hand-off of the same rollback
|
|
962
|
+
# keeps what an earlier one recorded.
|
|
963
|
+
def rollback_state(failure)
|
|
964
|
+
state = (@context.rollback || {}).dup
|
|
965
|
+
pending = @compensation_manager.pending
|
|
966
|
+
state["trigger"] ||= failure || pending ? "failure" : "step"
|
|
967
|
+
if pending
|
|
968
|
+
state["step"] = pending[:step].to_s
|
|
969
|
+
state["compensated"] = pending[:compensated]
|
|
970
|
+
state["compensation_error"] = pending[:compensation_error]
|
|
971
|
+
end
|
|
972
|
+
state["failure"] = ContextSerializer.serialize_value(failure) if failure
|
|
973
|
+
state["failures"] = ContextSerializer.serialize_value(@compensation_manager.rollback_failures)
|
|
974
|
+
state
|
|
975
|
+
end
|
|
976
|
+
|
|
977
|
+
# The rest of a handed-off rollback: the failing step's own compensate if
|
|
978
|
+
# it had not finished (a map or compose adopts its settled state), the
|
|
979
|
+
# steps still on the undo stack, then the outcome the inline rollback
|
|
980
|
+
# gives (I-7).
|
|
981
|
+
def finish_rollback(state)
|
|
982
|
+
@compensation_manager.restore_rollback_failures(state["failures"])
|
|
983
|
+
return finish_undo if state["trigger"] == "undo"
|
|
984
|
+
|
|
985
|
+
failure = ContextSerializer.deserialize_value(state["failure"])
|
|
986
|
+
step_config = state["step"] && @reactor_class.steps[state["step"].to_sym]
|
|
987
|
+
if step_config && !state["compensated"]
|
|
988
|
+
@compensation_manager.handle_step_failure(step_config, failure.error, {})
|
|
989
|
+
else
|
|
990
|
+
@compensation_manager.rollback_completed_steps
|
|
991
|
+
if state["compensation_error"]
|
|
992
|
+
raise Error::CompensationError.new(state["compensation_error"], step: step_config&.name, context: @context)
|
|
993
|
+
end
|
|
994
|
+
end
|
|
995
|
+
@context.current_step = failure.step_name&.to_sym
|
|
996
|
+
@result_handler.handed_off_failure(failure)
|
|
997
|
+
rescue Error::CompensationError => e
|
|
998
|
+
@result_handler.handle_execution_error(e)
|
|
999
|
+
end
|
|
1000
|
+
|
|
1001
|
+
def finish_undo
|
|
1002
|
+
@compensation_manager.rollback_completed_steps
|
|
1003
|
+
@context.cancelled = true
|
|
1004
|
+
@context.cancellation_reason = "Undo triggered"
|
|
1005
|
+
@context.status = "cancelled"
|
|
1006
|
+
nil
|
|
1007
|
+
end
|
|
1008
|
+
|
|
1009
|
+
def clear_saved_rollback_failures
|
|
1010
|
+
return unless @context.rollback
|
|
1011
|
+
|
|
1012
|
+
@context.rollback.delete("failures")
|
|
1013
|
+
@context.rollback = nil if @context.rollback.empty?
|
|
1014
|
+
end
|
|
1015
|
+
|
|
836
1016
|
def handle_interrupt(interrupt_result)
|
|
837
1017
|
save_context
|
|
838
1018
|
|