ruby_reactor 0.8.5 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/.specify/memory/constitution.md +17 -14
- data/CHANGELOG.md +58 -0
- data/CLAUDE.md +1 -1
- data/README.md +23 -8
- data/lib/ruby_reactor/adapters/active_job/map_element_rollback_worker.rb +20 -0
- data/lib/ruby_reactor/adapters/active_job/router.rb +32 -4
- data/lib/ruby_reactor/adapters/sidekiq/map_element_rollback_worker.rb +16 -0
- data/lib/ruby_reactor/adapters/sidekiq/router.rb +32 -4
- data/lib/ruby_reactor/context.rb +14 -1
- data/lib/ruby_reactor/dsl/definition_warnings.rb +30 -0
- data/lib/ruby_reactor/dsl/map_builder.rb +27 -7
- data/lib/ruby_reactor/dsl/step_builder.rb +8 -21
- data/lib/ruby_reactor/error/rescuable.rb +6 -1
- data/lib/ruby_reactor/error/rollback_handed_off.rb +27 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +33 -9
- data/lib/ruby_reactor/executor/result_handler.rb +72 -52
- data/lib/ruby_reactor/executor/retry_manager.rb +4 -3
- data/lib/ruby_reactor/executor/step_executor.rb +3 -1
- data/lib/ruby_reactor/executor.rb +186 -6
- data/lib/ruby_reactor/map/collector.rb +25 -93
- data/lib/ruby_reactor/map/dispatcher.rb +38 -7
- data/lib/ruby_reactor/map/element_executor.rb +14 -10
- data/lib/ruby_reactor/map/element_rollback.rb +158 -0
- data/lib/ruby_reactor/map/helpers.rb +11 -155
- data/lib/ruby_reactor/map/step_rollback.rb +136 -0
- data/lib/ruby_reactor/map/sweeper.rb +66 -8
- data/lib/ruby_reactor/map.rb +17 -0
- data/lib/ruby_reactor/max_retries_exhausted_failure.rb +4 -2
- data/lib/ruby_reactor/reactor.rb +41 -2
- data/lib/ruby_reactor/rspec/active_job_helpers.rb +13 -2
- data/lib/ruby_reactor/rspec/matchers.rb +40 -21
- data/lib/ruby_reactor/rspec/sidekiq_helpers.rb +7 -3
- data/lib/ruby_reactor/rspec/test_subject.rb +13 -3
- data/lib/ruby_reactor/rspec.rb +3 -6
- data/lib/ruby_reactor/step/compose_step.rb +3 -0
- data/lib/ruby_reactor/step/map_step.rb +117 -114
- data/lib/ruby_reactor/storage/adapter.rb +14 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +18 -2
- data/lib/ruby_reactor/storage/redis_map_rollback.rb +153 -0
- data/lib/ruby_reactor/storage/redis_reactor_scan.rb +1 -1
- data/lib/ruby_reactor/sweeper.rb +3 -1
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +11 -1
- data/lib/ruby_reactor/web/public/assets/{index-CQbgHtd0.js → index-C6ejszCQ.js} +12 -12
- data/lib/ruby_reactor/web/public/assets/index-DM5darhy.css +1 -0
- data/lib/ruby_reactor/web/public/index.html +2 -2
- data/lib/ruby_reactor/worker.rb +4 -2
- data/specs/007-execution-flow-analysis/evidence/harness.rb +2 -0
- data/specs/007-execution-flow-analysis/evidence/probes/03_map.rb +4 -1
- data/specs/009-distributed-map-undo/checklists/requirements.md +53 -0
- data/specs/009-distributed-map-undo/contracts/api-surface.md +125 -0
- data/specs/009-distributed-map-undo/contracts/rollback-protocol.md +157 -0
- data/specs/009-distributed-map-undo/data-model.md +151 -0
- data/specs/009-distributed-map-undo/plan.md +323 -0
- data/specs/009-distributed-map-undo/pr-description.md +78 -0
- data/specs/009-distributed-map-undo/quickstart.md +154 -0
- data/specs/009-distributed-map-undo/research.md +585 -0
- data/specs/009-distributed-map-undo/spec.md +360 -0
- data/specs/009-distributed-map-undo/tasks.md +1060 -0
- data/specs/future_improvements.md +18 -30
- metadata +21 -3
- data/lib/ruby_reactor/web/public/assets/index-BQvIWPdx.css +0 -1
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
module RubyReactor
|
|
4
4
|
module Map
|
|
5
5
|
class Collector
|
|
6
|
-
extend Helpers
|
|
7
|
-
|
|
8
6
|
# Seconds a collector waits for another collector of the same map. One
|
|
9
7
|
# that found the map unsettled releases within milliseconds, and may be
|
|
10
8
|
# the only trigger left for a failure it just deferred (R-04); a holder
|
|
@@ -12,7 +10,7 @@ module RubyReactor
|
|
|
12
10
|
COLLECT_LOCK_WAIT = 2
|
|
13
11
|
|
|
14
12
|
def self.perform(arguments)
|
|
15
|
-
arguments =
|
|
13
|
+
arguments = Helpers.normalize_arguments(arguments)
|
|
16
14
|
map_id = arguments[:map_id]
|
|
17
15
|
|
|
18
16
|
# Serialize concurrent collector deliveries for the SAME map (eager queue +
|
|
@@ -48,105 +46,39 @@ module RubyReactor
|
|
|
48
46
|
defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
|
|
49
47
|
end
|
|
50
48
|
|
|
49
|
+
# Once every index has a result slot (a value, `_error`, `_halt` or
|
|
50
|
+
# `_skipped`), signal the map's OWNER run — the top-level context —
|
|
51
|
+
# once, and write nothing else (009 R-03, I-1). The owner's Worker
|
|
52
|
+
# resumes, and `MapStep#run` adopts the settled outcome at any
|
|
53
|
+
# composition depth. An atomic map's failure is applied the same way
|
|
54
|
+
# only once every index has settled, so its compensate sees every
|
|
55
|
+
# element that completed, including ones still in flight when it
|
|
56
|
+
# failed. Until then the last element to settle, or the map sweeper,
|
|
57
|
+
# re-triggers this collector.
|
|
51
58
|
def self.perform_collection(arguments)
|
|
52
59
|
map_id = arguments[:map_id]
|
|
53
|
-
parent_context_id = arguments[:parent_context_id]
|
|
54
60
|
parent_reactor_class_name = arguments[:parent_reactor_class_name]
|
|
55
|
-
step_name = arguments[:step_name]
|
|
56
|
-
strict_ordering = arguments[:strict_ordering]
|
|
57
|
-
# timeout = arguments[:timeout]
|
|
58
|
-
|
|
59
61
|
storage = RubyReactor.configuration.storage_adapter
|
|
60
|
-
parent_context_data = storage.retrieve_context(parent_context_id, parent_reactor_class_name)
|
|
61
|
-
parent_context = RubyReactor::Context.deserialize_from_retry(parent_context_data)
|
|
62
|
-
|
|
63
|
-
# Idempotency: if the parent already recorded this map step's result, a
|
|
64
|
-
# prior collector already resumed it. Re-resuming would double-execute the
|
|
65
|
-
# steps after the map. Skip. A parent already finished (a prior
|
|
66
|
-
# collector applied this map's failure) must not be rolled back twice.
|
|
67
|
-
return if parent_context.intermediate_results.key?(step_name.to_sym) || parent_context.finished?
|
|
68
62
|
|
|
69
|
-
# Check if all tasks are completed
|
|
70
63
|
metadata = storage.retrieve_map_metadata(map_id, parent_reactor_class_name)
|
|
71
|
-
|
|
64
|
+
return unless metadata
|
|
65
|
+
# Judged against the element count, not map_offset (which batching
|
|
66
|
+
# reservation can push past it).
|
|
67
|
+
return if storage.count_map_results(map_id, parent_reactor_class_name) < metadata["count"].to_i
|
|
68
|
+
return unless storage.claim_map_owner_signal(map_id, parent_reactor_class_name)
|
|
72
69
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
# Completion is judged against the total count of elements, not
|
|
76
|
-
# map_offset (which batching reservation can push past it).
|
|
77
|
-
#
|
|
78
|
-
# A fail-fast failure is applied only once every index has settled
|
|
79
|
-
# (a result, `_error`, `_halt` or `_skipped` slot), so the map's
|
|
80
|
-
# compensate sees every element that completed — including ones still
|
|
81
|
-
# in flight when the failure happened (R-04). Until then the last
|
|
82
|
-
# element to settle, or the map sweeper, re-triggers this collector.
|
|
83
|
-
return if results_count < total_count
|
|
84
|
-
|
|
85
|
-
if (failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name))
|
|
86
|
-
handle_failure(failed_context_id, metadata, storage, parent_context, step_name)
|
|
87
|
-
return
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
# Retrieve results lazily
|
|
91
|
-
results = RubyReactor::Map::ResultEnumerator.new(
|
|
92
|
-
map_id,
|
|
93
|
-
parent_reactor_class_name,
|
|
94
|
-
strict_ordering: strict_ordering
|
|
95
|
-
)
|
|
96
|
-
|
|
97
|
-
# Apply collect block (or default collection)
|
|
98
|
-
step_config = parent_context.reactor_class.steps[step_name.to_sym]
|
|
99
|
-
|
|
100
|
-
begin
|
|
101
|
-
final_result = apply_collect_block(results, step_config)
|
|
102
|
-
|
|
103
|
-
if final_result.failure?
|
|
104
|
-
# Optionally log failure internally or just rely on context status update
|
|
105
|
-
end
|
|
106
|
-
rescue StandardError => e
|
|
107
|
-
final_result = RubyReactor::Failure(e)
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
# Resume parent execution
|
|
111
|
-
resume_parent_execution(parent_context, step_name, final_result, storage)
|
|
112
|
-
rescue StandardError => e
|
|
113
|
-
RubyReactor.configuration.logger.error("Map collector crashed: #{e.message}")
|
|
114
|
-
RubyReactor.configuration.logger.error(e.backtrace.join("\n")) if e.backtrace
|
|
115
|
-
raise e
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
def self.apply_collect_block(results, step_config)
|
|
119
|
-
collect_block = step_config.arguments[:collect_block][:source].value if step_config.arguments[:collect_block]
|
|
120
|
-
# TODO: Check allow_partial_failure option
|
|
121
|
-
|
|
122
|
-
if collect_block
|
|
123
|
-
begin
|
|
124
|
-
# Pass Enumerator to collect block
|
|
125
|
-
collected = collect_block.call(results)
|
|
126
|
-
RubyReactor::Success(collected)
|
|
127
|
-
rescue RubyReactor::Error::Rescuable => e
|
|
128
|
-
RubyReactor.configuration.logger.error("Map collect block raised: #{e.message}")
|
|
129
|
-
RubyReactor.configuration.logger.error(e.backtrace.join("\n")) if e.backtrace
|
|
130
|
-
RubyReactor::Failure(e)
|
|
131
|
-
end
|
|
132
|
-
else
|
|
133
|
-
# Default behavior: Return Success(Enumerator).
|
|
134
|
-
# Logic for checking failures is deferred to the consumer of the enumerator.
|
|
135
|
-
RubyReactor::Success(results)
|
|
136
|
-
end
|
|
70
|
+
signal_owner(metadata, arguments)
|
|
137
71
|
end
|
|
138
72
|
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
result = reason.is_a?(RubyReactor::Failure) ? reason : RubyReactor::Failure(reason)
|
|
149
|
-
resume_parent_execution(parent_context, step_name, result, storage)
|
|
73
|
+
# Metadata written before the upgrade names no owner (S-6): the parent is
|
|
74
|
+
# the owner of a root-level map.
|
|
75
|
+
def self.signal_owner(metadata, arguments)
|
|
76
|
+
owner_id = metadata["owner_context_id"] || metadata["parent_context_id"] || arguments[:parent_context_id]
|
|
77
|
+
owner_class = metadata["owner_reactor_class_name"] || metadata["parent_reactor_class_name"]
|
|
78
|
+
RubyReactor.configuration.logger.info(
|
|
79
|
+
"event=ruby_reactor.map.settled map_id=#{arguments[:map_id].inspect} owner=#{owner_id.inspect}"
|
|
80
|
+
)
|
|
81
|
+
RubyReactor.configuration.async_router.perform_async(owner_id, owner_class)
|
|
150
82
|
end
|
|
151
83
|
end
|
|
152
84
|
end
|
|
@@ -6,7 +6,7 @@ module RubyReactor
|
|
|
6
6
|
extend Helpers
|
|
7
7
|
|
|
8
8
|
def self.perform(arguments)
|
|
9
|
-
arguments =
|
|
9
|
+
arguments = Helpers.normalize_arguments(arguments)
|
|
10
10
|
parent_reactor_class_name = arguments[:parent_reactor_class_name]
|
|
11
11
|
|
|
12
12
|
storage = RubyReactor.configuration.storage_adapter
|
|
@@ -64,12 +64,12 @@ module RubyReactor
|
|
|
64
64
|
map_id = arguments[:map_id]
|
|
65
65
|
reactor_class_name = arguments[:parent_reactor_class_name]
|
|
66
66
|
|
|
67
|
-
#
|
|
68
|
-
if arguments[:
|
|
67
|
+
# An atomic map stops dispatching at its first failure.
|
|
68
|
+
if arguments[:atomic] && storage.retrieve_map_failed_context_id(map_id, reactor_class_name)
|
|
69
69
|
return settle_undispatched(arguments, storage)
|
|
70
70
|
end
|
|
71
71
|
|
|
72
|
-
batch_size = arguments[:batch_size] ||
|
|
72
|
+
batch_size = arguments[:batch_size] || RubyReactor::Map::DEFAULT_BATCH_SIZE
|
|
73
73
|
|
|
74
74
|
# Atomically reserve a batch
|
|
75
75
|
new_offset = storage.increment_map_offset(map_id, batch_size, reactor_class_name)
|
|
@@ -103,7 +103,7 @@ module RubyReactor
|
|
|
103
103
|
end
|
|
104
104
|
end
|
|
105
105
|
|
|
106
|
-
#
|
|
106
|
+
# An atomic map stopped dispatching: claim every index not dispatched
|
|
107
107
|
# yet (one offset bump, so a later dispatcher claims none), settle each
|
|
108
108
|
# with a `_skipped` slot, count them down, and trigger the collector if
|
|
109
109
|
# that settled the map. Without this those indices never settle, so the
|
|
@@ -146,7 +146,7 @@ module RubyReactor
|
|
|
146
146
|
strict_ordering: map_meta["strict_ordering"],
|
|
147
147
|
parent_context_id: map_meta["parent_context_id"],
|
|
148
148
|
parent_reactor_class_name: parent_class_name,
|
|
149
|
-
|
|
149
|
+
atomic: map_meta["atomic"],
|
|
150
150
|
batch_size: map_meta["batch_size"]
|
|
151
151
|
}
|
|
152
152
|
|
|
@@ -211,10 +211,41 @@ module RubyReactor
|
|
|
211
211
|
parent_reactor_class_name: context.reactor_class.name,
|
|
212
212
|
step_name: options[:step_name].to_s,
|
|
213
213
|
batch_size: arguments[:batch_size], # Passed to worker so it knows to trigger next batch?
|
|
214
|
-
|
|
214
|
+
atomic: arguments[:atomic]
|
|
215
215
|
)
|
|
216
216
|
end
|
|
217
217
|
|
|
218
|
+
# One throw of a map's distributed rollback (009 R-02, R-06): claim the
|
|
219
|
+
# next `batch_size` positions of the element-context index, counted from
|
|
220
|
+
# its tail, and enqueue one rollback job per position. The job at the
|
|
221
|
+
# last position of a throw claims the next one, as forward.
|
|
222
|
+
def self.dispatch_rollback_batch(map_id:, parent_reactor_class_name:)
|
|
223
|
+
storage = RubyReactor.configuration.storage_adapter
|
|
224
|
+
meta = storage.retrieve_map_rollback_metadata(map_id, parent_reactor_class_name)
|
|
225
|
+
return 0 unless meta
|
|
226
|
+
|
|
227
|
+
positions = storage.claim_map_rollback_positions(map_id, parent_reactor_class_name, meta["batch_size"].to_i)
|
|
228
|
+
return 0 if positions.none?
|
|
229
|
+
|
|
230
|
+
ids = storage.retrieve_map_element_context_ids_from_tail(map_id, parent_reactor_class_name, positions.first,
|
|
231
|
+
positions.size, total: meta["total"].to_i)
|
|
232
|
+
positions.each_with_index do |position, i|
|
|
233
|
+
queue_rollback_job(meta, map_id, parent_reactor_class_name, position, ids[i])
|
|
234
|
+
end
|
|
235
|
+
positions.size
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# rubocop:disable Metrics/ParameterLists
|
|
239
|
+
def self.queue_rollback_job(meta, map_id, parent_reactor_class_name, position, element_context_id, attempt: 0)
|
|
240
|
+
RubyReactor.configuration.async_router.perform_map_element_rollback_async(
|
|
241
|
+
map_id: map_id, position: position, element_context_id: element_context_id,
|
|
242
|
+
reactor_class_info: meta["reactor_class_info"], parent_reactor_class_name: parent_reactor_class_name,
|
|
243
|
+
step_name: meta["step_name"], batch_size: meta["batch_size"], owner_context_id: meta["owner_context_id"],
|
|
244
|
+
owner_reactor_class_name: meta["owner_reactor_class_name"], attempt: attempt
|
|
245
|
+
)
|
|
246
|
+
end
|
|
247
|
+
# rubocop:enable Metrics/ParameterLists
|
|
248
|
+
|
|
218
249
|
def self.resolve_reactor_class_info(arguments, context)
|
|
219
250
|
mapped_reactor_class = arguments[:mapped_reactor_class]
|
|
220
251
|
step_name = arguments[:step_name]
|
|
@@ -6,7 +6,7 @@ module RubyReactor
|
|
|
6
6
|
extend Helpers
|
|
7
7
|
|
|
8
8
|
def self.perform(arguments)
|
|
9
|
-
arguments =
|
|
9
|
+
arguments = Helpers.normalize_arguments(arguments)
|
|
10
10
|
|
|
11
11
|
# Per-element liveness lock (Phase 5b): its presence is the map sweeper's
|
|
12
12
|
# "element alive" signal, and it serializes duplicate deliveries so a
|
|
@@ -55,12 +55,16 @@ module RubyReactor
|
|
|
55
55
|
context.inline_async_execution = true
|
|
56
56
|
|
|
57
57
|
storage = RubyReactor.configuration.storage_adapter
|
|
58
|
-
return if
|
|
58
|
+
return if check_atomic?(arguments, storage)
|
|
59
59
|
|
|
60
60
|
# Indexed only once it runs: a skipped element saves no context, and
|
|
61
61
|
# the map's rollback reports an indexed element without one as expired.
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
# Only a fresh context: a parked or retried element re-enters with its
|
|
63
|
+
# own, already indexed (009 R-06).
|
|
64
|
+
unless arguments[:serialized_context]
|
|
65
|
+
storage.store_map_element_context_id(arguments[:map_id], context.context_id,
|
|
66
|
+
arguments[:parent_reactor_class_name])
|
|
67
|
+
end
|
|
64
68
|
|
|
65
69
|
executor = Executor.new(context.reactor_class, {}, context)
|
|
66
70
|
begin
|
|
@@ -95,7 +99,7 @@ module RubyReactor
|
|
|
95
99
|
strict_ordering: arguments[:strict_ordering], parent_context_id: arguments[:parent_context_id],
|
|
96
100
|
parent_reactor_class_name: arguments[:parent_reactor_class_name], step_name: arguments[:step_name],
|
|
97
101
|
batch_size: arguments[:batch_size], serialized_context: ContextSerializer.serialize(context),
|
|
98
|
-
|
|
102
|
+
atomic: arguments[:atomic]
|
|
99
103
|
)
|
|
100
104
|
end
|
|
101
105
|
|
|
@@ -144,8 +148,8 @@ module RubyReactor
|
|
|
144
148
|
end
|
|
145
149
|
# rubocop:enable Style/IdenticalConditionalBranches
|
|
146
150
|
|
|
147
|
-
def self.
|
|
148
|
-
return false unless arguments[:
|
|
151
|
+
def self.check_atomic?(arguments, storage)
|
|
152
|
+
return false unless arguments[:atomic]
|
|
149
153
|
|
|
150
154
|
map_id = arguments[:map_id]
|
|
151
155
|
parent_reactor_class_name = arguments[:parent_reactor_class_name]
|
|
@@ -154,7 +158,7 @@ module RubyReactor
|
|
|
154
158
|
return false unless failed_context_id
|
|
155
159
|
|
|
156
160
|
# Skip execution, but settle the index: the collector applies a
|
|
157
|
-
#
|
|
161
|
+
# atomic map's failure only once every index has a result slot (R-04), and
|
|
158
162
|
# the map sweeper would otherwise keep re-dispatching this one.
|
|
159
163
|
storage.store_map_result(map_id, arguments[:index], { "_skipped" => true }, parent_reactor_class_name,
|
|
160
164
|
strict_ordering: arguments[:strict_ordering])
|
|
@@ -183,12 +187,12 @@ module RubyReactor
|
|
|
183
187
|
# Store the whole serialized Failure, not just its message: step_name,
|
|
184
188
|
# backtrace, file_path and code_snippet are the only record of why this
|
|
185
189
|
# element failed once its context row expires, and the dashboard has
|
|
186
|
-
# nothing else to show for a non-
|
|
190
|
+
# nothing else to show for a non-atomic map.
|
|
187
191
|
storage.store_map_result(map_id, index,
|
|
188
192
|
{ _error: ContextSerializer.serialize_value(result) }, parent_class,
|
|
189
193
|
strict_ordering: arguments[:strict_ordering])
|
|
190
194
|
|
|
191
|
-
if arguments[:
|
|
195
|
+
if arguments[:atomic]
|
|
192
196
|
storage.store_map_failed_context_id(map_id, context.context_id, parent_class)
|
|
193
197
|
# FAST FAIL: Trigger Collector immediately to cancel/fail the map execution
|
|
194
198
|
RubyReactor.configuration.async_router.perform_map_collection_async(
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
module Map
|
|
5
|
+
# Rolls back ONE element of a map (009 R-07): the element's own undo
|
|
6
|
+
# stack, replayed under its liveness lock and saved after every undone
|
|
7
|
+
# entry, so a redelivery resumes after the last one (only the entry that
|
|
8
|
+
# was cut off may run again). The inline path calls `.call` in process; a
|
|
9
|
+
# fan-out map's `MapElementRollbackWorker` calls `.perform`.
|
|
10
|
+
class ElementRollback
|
|
11
|
+
extend Helpers
|
|
12
|
+
|
|
13
|
+
# The element context row is gone: its index went with it.
|
|
14
|
+
def self.unavailable(step_name)
|
|
15
|
+
{ "index" => nil, "outcome" => "context_unavailable",
|
|
16
|
+
"failures" => ContextSerializer.serialize_value([entry(step_name, nil, :context_unavailable,
|
|
17
|
+
"context expired")]) }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# `{ "index", "outcome", "failures" }` (DM §5), with `failures`
|
|
21
|
+
# serialized. Outcome `contended`, never stored, means the element's lock
|
|
22
|
+
# was still held after `wait`.
|
|
23
|
+
def self.call(map_id:, element_context_id:, element_class:, step_name:, wait: Step::MapStep::ELEMENT_LOCK_WAIT)
|
|
24
|
+
storage_name = RubyReactor.reactor_storage_name(element_class)
|
|
25
|
+
data = element_context_id && RubyReactor.configuration.storage_adapter.retrieve_context(element_context_id,
|
|
26
|
+
storage_name)
|
|
27
|
+
return unavailable(step_name) unless data
|
|
28
|
+
|
|
29
|
+
element = Context.deserialize_from_retry(data)
|
|
30
|
+
index = Utils::FetchIndifferent.call(element.map_metadata || {}, :index).to_i
|
|
31
|
+
lock = acquire_element_lock(map_id, index, wait)
|
|
32
|
+
return outcome(index, "contended") if lock == :held
|
|
33
|
+
|
|
34
|
+
begin
|
|
35
|
+
roll_back(element, element_class, index, step_name)
|
|
36
|
+
ensure
|
|
37
|
+
lock.release if lock.respond_to?(:release)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# A completed element, or an aborted one (an inline run interrupted in
|
|
42
|
+
# it, which kept the undo entries of the steps it completed), undoes;
|
|
43
|
+
# a failed one already rolled itself back, and a halted or superseded
|
|
44
|
+
# running one did nothing to undo.
|
|
45
|
+
def self.roll_back(element, element_class, index, step_name)
|
|
46
|
+
return outcome(index, "not_needed") unless %w[completed aborted].include?(element.status.to_s)
|
|
47
|
+
|
|
48
|
+
executor = Executor.new(element.reactor_class || element_class, {}, element)
|
|
49
|
+
executor.undo_all { executor.checkpoint! }
|
|
50
|
+
executor.checkpoint!
|
|
51
|
+
tag = { map_step: step_name.to_sym, element_index: index }
|
|
52
|
+
failures = executor.compensation_manager.rollback_failures.map { |failure| failure.merge(tag) }
|
|
53
|
+
outcome(index, failures.empty? ? "undone" : "failed", failures)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The job's body. A contended lock requeues the job, storing nothing,
|
|
57
|
+
# up to `lock_snooze_max_attempts` times before reporting the element
|
|
58
|
+
# in flight: two contexts of one index in the same batch then serialize
|
|
59
|
+
# instead of one reporting a false failure (R-07).
|
|
60
|
+
def self.perform(arguments)
|
|
61
|
+
args = Helpers.normalize_arguments(arguments)
|
|
62
|
+
map_id = args[:map_id]
|
|
63
|
+
map_class = args[:parent_reactor_class_name]
|
|
64
|
+
storage = RubyReactor.configuration.storage_adapter
|
|
65
|
+
return if storage.map_rollback_outcome_stored?(map_id, map_class, args[:position])
|
|
66
|
+
|
|
67
|
+
attempt = args[:attempt].to_i
|
|
68
|
+
result = call(map_id: map_id, element_context_id: args[:element_context_id],
|
|
69
|
+
element_class: resolve_reactor_class(args[:reactor_class_info]), step_name: args[:step_name],
|
|
70
|
+
wait: attempt.zero? ? Step::MapStep::ELEMENT_LOCK_WAIT : 0)
|
|
71
|
+
if result["outcome"] == "contended"
|
|
72
|
+
return requeue(args, attempt) unless snooze_exhausted?(attempt)
|
|
73
|
+
|
|
74
|
+
result = in_flight(result["index"], args[:step_name])
|
|
75
|
+
end
|
|
76
|
+
return unless storage.store_map_rollback_outcome(map_id, map_class, args[:position], result)
|
|
77
|
+
|
|
78
|
+
log_outcome(args, result)
|
|
79
|
+
Dispatcher.dispatch_rollback_batch(map_id: map_id, parent_reactor_class_name: map_class) if batch_end?(args)
|
|
80
|
+
signal_owner_if_settled(args)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def self.in_flight(index, step_name)
|
|
84
|
+
outcome(index, "element_in_flight",
|
|
85
|
+
[entry(step_name, index, :element_in_flight, "was still running at rollback time")])
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def self.snooze_exhausted?(attempt)
|
|
89
|
+
max = RubyReactor.configuration.lock_snooze_max_attempts
|
|
90
|
+
max != :infinity && attempt >= max
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def self.requeue(args, attempt)
|
|
94
|
+
config = RubyReactor.configuration
|
|
95
|
+
config.async_router.perform_map_element_rollback_in(
|
|
96
|
+
Worker.snooze_delay(config, nil), **args.slice(*ROLLBACK_JOB_ARGS), attempt: attempt + 1
|
|
97
|
+
)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# The position that ends a throw claims the next one (R-02).
|
|
101
|
+
def self.batch_end?(args)
|
|
102
|
+
batch_size = args[:batch_size].to_i
|
|
103
|
+
batch_size.positive? && ((args[:position].to_i + 1) % batch_size).zero?
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# The handshake's job side (R-05): only once the hand-off is saved, so
|
|
107
|
+
# the owner never resumes from a stale blob — and never in inline job
|
|
108
|
+
# mode, where the map settles before any hand-off exists.
|
|
109
|
+
def self.signal_owner_if_settled(args)
|
|
110
|
+
storage = RubyReactor.configuration.storage_adapter
|
|
111
|
+
map_id = args[:map_id]
|
|
112
|
+
map_class = args[:parent_reactor_class_name]
|
|
113
|
+
meta = storage.retrieve_map_rollback_metadata(map_id, map_class)
|
|
114
|
+
return unless meta && storage.count_map_rollback_outcomes(map_id, map_class) >= meta["total"].to_i
|
|
115
|
+
return unless storage.map_rollback_handed_off?(map_id, map_class)
|
|
116
|
+
return unless storage.claim_map_rollback_signal(map_id, map_class)
|
|
117
|
+
|
|
118
|
+
RubyReactor.configuration.async_router.perform_async(args[:owner_context_id], args[:owner_reactor_class_name])
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def self.log_outcome(args, result)
|
|
122
|
+
Map.log("ruby_reactor.map.rollback.element",
|
|
123
|
+
reactor: args[:owner_reactor_class_name], context_id: args[:owner_context_id],
|
|
124
|
+
map_step: args[:step_name], index: result["index"], outcome: result["outcome"],
|
|
125
|
+
failures: Array(result["failures"]).size)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def self.outcome(index, outcome, failures = [])
|
|
129
|
+
{ "index" => index, "outcome" => outcome, "failures" => ContextSerializer.serialize_value(failures) }
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# A rollback failure entry for the element as a whole (008 shape).
|
|
133
|
+
def self.entry(step_name, index, reason, message)
|
|
134
|
+
step = step_name.to_sym
|
|
135
|
+
{ step: step, kind: :undo, key: nil, reason: reason, map_step: step, element_index: index,
|
|
136
|
+
message: "map element #{index} #{message}" }
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def self.acquire_element_lock(map_id, index, wait)
|
|
140
|
+
return nil if ElementExecutor.inline_testing_mode?
|
|
141
|
+
|
|
142
|
+
lock = RubyReactor::Lock.new("map_element:#{map_id}:#{index}", owner: SecureRandom.uuid,
|
|
143
|
+
ttl: RubyReactor.configuration.context_lock_ttl,
|
|
144
|
+
wait: wait)
|
|
145
|
+
lock.acquire
|
|
146
|
+
lock
|
|
147
|
+
rescue RubyReactor::Lock::AcquisitionError
|
|
148
|
+
:held
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
ROLLBACK_JOB_ARGS = %i[map_id position element_context_id reactor_class_info parent_reactor_class_name step_name
|
|
152
|
+
batch_size owner_context_id owner_reactor_class_name].freeze
|
|
153
|
+
|
|
154
|
+
private_class_method :roll_back, :in_flight, :snooze_exhausted?, :requeue, :batch_end?,
|
|
155
|
+
:signal_owner_if_settled, :log_outcome, :acquire_element_lock
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
@@ -4,6 +4,16 @@ module RubyReactor
|
|
|
4
4
|
module Map
|
|
5
5
|
# Shared helper methods for Map executors
|
|
6
6
|
module Helpers
|
|
7
|
+
# Job payloads enqueued before 009 carry `fail_fast` where they now carry
|
|
8
|
+
# `atomic` (R-11, FR-023): the one place the old key is read. Removed at
|
|
9
|
+
# the next MAJOR.
|
|
10
|
+
def self.normalize_arguments(arguments)
|
|
11
|
+
normalized = arguments.to_h.transform_keys(&:to_sym)
|
|
12
|
+
normalized[:atomic] = normalized[:fail_fast] if !normalized.key?(:atomic) && normalized.key?(:fail_fast)
|
|
13
|
+
normalized.delete(:fail_fast)
|
|
14
|
+
normalized
|
|
15
|
+
end
|
|
16
|
+
|
|
7
17
|
# Resolves the reactor class from reactor_class_info
|
|
8
18
|
def resolve_reactor_class(info)
|
|
9
19
|
if info["type"] == "class"
|
|
@@ -20,6 +30,7 @@ module RubyReactor
|
|
|
20
30
|
raise "Unknown reactor class info: #{info}"
|
|
21
31
|
end
|
|
22
32
|
end
|
|
33
|
+
module_function :resolve_reactor_class
|
|
23
34
|
|
|
24
35
|
# Loads parent context from storage
|
|
25
36
|
def load_parent_context_from_storage(parent_context_id, reactor_class_name, storage)
|
|
@@ -31,161 +42,6 @@ module RubyReactor
|
|
|
31
42
|
def build_element_inputs(mappings, parent_context, element)
|
|
32
43
|
RubyReactor::Step::MapStep.build_mapped_inputs(mappings, parent_context, element)
|
|
33
44
|
end
|
|
34
|
-
|
|
35
|
-
# Applies collect block to results
|
|
36
|
-
def apply_collect_block(results, step_config)
|
|
37
|
-
collect_block = step_config.arguments[:collect_block][:source].value
|
|
38
|
-
|
|
39
|
-
if collect_block
|
|
40
|
-
# Pass all results (Success and Failure) to collect block
|
|
41
|
-
begin
|
|
42
|
-
collected = collect_block.call(results)
|
|
43
|
-
RubyReactor::Success(collected)
|
|
44
|
-
rescue RubyReactor::Error::Rescuable => e
|
|
45
|
-
RubyReactor::Failure(e)
|
|
46
|
-
end
|
|
47
|
-
else
|
|
48
|
-
# Default behavior: fail if any failure
|
|
49
|
-
first_failure = results.find(&:failure?)
|
|
50
|
-
first_failure || RubyReactor::Success(results.map(&:value))
|
|
51
|
-
end
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
# Resumes parent reactor execution after map completion
|
|
55
|
-
def resume_parent_execution(parent_context, step_name, final_result, storage) # rubocop:disable Metrics/MethodLength
|
|
56
|
-
executor = RubyReactor::Executor.new(parent_context.reactor_class, {}, parent_context)
|
|
57
|
-
step_name_sym = step_name.to_sym
|
|
58
|
-
|
|
59
|
-
if final_result.failure?
|
|
60
|
-
parent_context.current_step = step_name_sym
|
|
61
|
-
|
|
62
|
-
error = RubyReactor::Error::StepFailureError.new(
|
|
63
|
-
final_result.error,
|
|
64
|
-
step: step_name_sym,
|
|
65
|
-
context: parent_context,
|
|
66
|
-
original_error: final_result.error.is_a?(Exception) ? final_result.error : nil,
|
|
67
|
-
exception_class: final_result.respond_to?(:exception_class) ? final_result.exception_class : nil
|
|
68
|
-
)
|
|
69
|
-
|
|
70
|
-
# Pass backtrace if available
|
|
71
|
-
if final_result.respond_to?(:backtrace) && final_result.backtrace
|
|
72
|
-
error.set_backtrace(final_result.backtrace)
|
|
73
|
-
elsif final_result.error.respond_to?(:backtrace)
|
|
74
|
-
error.set_backtrace(final_result.error.backtrace)
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
# Bracket the rollback with reactor lifecycle events so that
|
|
78
|
-
# compensation/undo spans nest under a reactor span (and stay attached
|
|
79
|
-
# to the originating trace), mirroring the success path's
|
|
80
|
-
# resume_execution. Without this, rollback runs in the collector worker
|
|
81
|
-
# with no active reactor span and the undo/compensation spans orphan.
|
|
82
|
-
executor.middlewares.on(:start_reactor, parent_context.reactor_class.name, parent_context.inputs,
|
|
83
|
-
parent_context)
|
|
84
|
-
failure_response = nil
|
|
85
|
-
begin
|
|
86
|
-
# The map failed in its element jobs, so no executor compensated it.
|
|
87
|
-
# Fail it exactly as the inline path does: adopt the failing
|
|
88
|
-
# element's rollback failures, compensate the map (its completed
|
|
89
|
-
# elements; every index has settled, R-04), undo the completed
|
|
90
|
-
# steps, and fail with `CompensationError` if the map's rollback
|
|
91
|
-
# was incomplete.
|
|
92
|
-
manager = executor.compensation_manager
|
|
93
|
-
manager.rollback_failures.concat(final_result.rollback_failures)
|
|
94
|
-
failure_response = begin
|
|
95
|
-
manager.handle_step_failure(parent_context.reactor_class.steps[step_name_sym], final_result.error, {})
|
|
96
|
-
executor.result_handler.handle_execution_error(error)
|
|
97
|
-
rescue RubyReactor::Error::CompensationError => e
|
|
98
|
-
executor.result_handler.handle_execution_error(e)
|
|
99
|
-
end
|
|
100
|
-
# Manually update context status since we're not running executor loop
|
|
101
|
-
executor.send(:update_context_status, failure_response)
|
|
102
|
-
ensure
|
|
103
|
-
executor.middlewares.on(:failed_reactor, parent_context.reactor_class.name, failure_response,
|
|
104
|
-
parent_context)
|
|
105
|
-
end
|
|
106
|
-
# This branch runs no executor loop, so nothing else persists the
|
|
107
|
-
# failed status: store it here.
|
|
108
|
-
store_parent(parent_context, storage)
|
|
109
|
-
else
|
|
110
|
-
parent_context.set_result(step_name_sym, final_result.value)
|
|
111
|
-
# A completed fan-out map is tracked for undo like any step (R-03).
|
|
112
|
-
# The record stays empty: `MapStep#undo` finds its elements through
|
|
113
|
-
# the map's element index, so the parent does not grow per element.
|
|
114
|
-
parent_context.undo_stack << { step: parent_context.reactor_class.steps[step_name_sym], arguments: {},
|
|
115
|
-
result: RubyReactor.Success(nil) }
|
|
116
|
-
record_elements_started(parent_context, step_name, storage)
|
|
117
|
-
|
|
118
|
-
# Manually update execution trace to reflect completion
|
|
119
|
-
# This is necessary because resume_execution continues from the NEXT step
|
|
120
|
-
# and the async step (which returned DispatchResult) needs to be marked as done with actual value
|
|
121
|
-
parent_context.append_execution_trace(
|
|
122
|
-
{
|
|
123
|
-
type: :result,
|
|
124
|
-
step: step_name_sym,
|
|
125
|
-
timestamp: Time.now,
|
|
126
|
-
value: final_result.value,
|
|
127
|
-
status: :success
|
|
128
|
-
}
|
|
129
|
-
)
|
|
130
|
-
|
|
131
|
-
parent_context.current_step = nil
|
|
132
|
-
|
|
133
|
-
# Mark the context as a worker before resuming: resume_execution runs
|
|
134
|
-
# the collector's own worker (this call), not the reactor's normal
|
|
135
|
-
# caller. Without this, StepExecutor#handoff_at? (gated on
|
|
136
|
-
# `inline_async_execution`) re-evaluates any later `background`
|
|
137
|
-
# before:/after: cut point and enqueues a second hand-off, or runs an
|
|
138
|
-
# `after:` target here in the collector instead of the original
|
|
139
|
-
# dispatching worker.
|
|
140
|
-
parent_context.inline_async_execution = true
|
|
141
|
-
resume_parked_aware(executor, parent_context)
|
|
142
|
-
end
|
|
143
|
-
end
|
|
144
|
-
|
|
145
|
-
# `resume_execution` persists the parent itself — under the parent's
|
|
146
|
-
# context lock, and deliberately NOT when it lost that lock to a live
|
|
147
|
-
# duplicate or replayed an already-terminal run. Storing again here, after
|
|
148
|
-
# the lock is released, would overwrite whatever the lock's holder wrote
|
|
149
|
-
# (single writer), so the resume's own save is the only one.
|
|
150
|
-
#
|
|
151
|
-
# This collector is a worker running the parent's execution, so it is
|
|
152
|
-
# also a final handler for park signals (005 R-01), like `Worker` and
|
|
153
|
-
# `ElementExecutor`: the resume has already parked the parent's holds and
|
|
154
|
-
# saved; hand the rest back to the parent's own worker.
|
|
155
|
-
#
|
|
156
|
-
# The parent is loaded from its own blob, so it has no `root_context`:
|
|
157
|
-
# a map inside a composed child resumes and requeues that child as its
|
|
158
|
-
# own execution, never its root. ponytail: the root is never resumed
|
|
159
|
-
# after such a map (already so on main); see "Fan-out map inside a
|
|
160
|
-
# composed child" in specs/future_improvements.md.
|
|
161
|
-
def resume_parked_aware(executor, parent_context)
|
|
162
|
-
executor.resume_execution
|
|
163
|
-
rescue RubyReactor::Error::ExecutionParked => e
|
|
164
|
-
config = RubyReactor.configuration
|
|
165
|
-
config.async_router.perform_in(
|
|
166
|
-
RubyReactor::Worker.snooze_delay(config, e), parent_context.context_id,
|
|
167
|
-
RubyReactor.reactor_storage_name(parent_context.reactor_class)
|
|
168
|
-
)
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
# Every index of a completed map ran: record the count on the map's
|
|
172
|
-
# reference in the parent, which outlives the element index, so a late
|
|
173
|
-
# rollback names each element whose context expired.
|
|
174
|
-
def record_elements_started(parent_context, step_name, storage)
|
|
175
|
-
ref = Utils::FetchIndifferent.call(parent_context.composed_contexts, step_name)
|
|
176
|
-
return unless ref
|
|
177
|
-
|
|
178
|
-
map_id = Utils::FetchIndifferent.call(ref, :map_id)
|
|
179
|
-
ref[:started] = storage.retrieve_map_metadata(map_id, parent_context.reactor_class.name)&.fetch("count", 0).to_i
|
|
180
|
-
end
|
|
181
|
-
|
|
182
|
-
def store_parent(parent_context, storage)
|
|
183
|
-
storage.store_context(
|
|
184
|
-
parent_context.context_id,
|
|
185
|
-
ContextSerializer.serialize(parent_context),
|
|
186
|
-
RubyReactor.reactor_storage_name(parent_context.reactor_class)
|
|
187
|
-
)
|
|
188
|
-
end
|
|
189
45
|
end
|
|
190
46
|
end
|
|
191
47
|
end
|