ruby_reactor 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/skills/demo-app-e2e-verify/SKILL.md +226 -0
- data/.claude/skills/speckit-demo-tests/SKILL.md +144 -0
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/.specify/memory/constitution.md +79 -12
- data/.specify/templates/tasks-template.md +7 -0
- data/CHANGELOG.md +125 -1
- data/CLAUDE.md +5 -0
- data/README.md +155 -25
- data/lib/ruby_reactor/adapters/active_job/router.rb +21 -6
- data/lib/ruby_reactor/adapters/active_job/step_worker.rb +20 -0
- data/lib/ruby_reactor/adapters/active_job/worker.rb +6 -1
- data/lib/ruby_reactor/adapters/sidekiq/router.rb +21 -6
- data/lib/ruby_reactor/adapters/sidekiq/step_worker.rb +17 -0
- data/lib/ruby_reactor/adapters/sidekiq/worker.rb +5 -2
- data/lib/ruby_reactor/async_waiter.rb +113 -0
- data/lib/ruby_reactor/configuration.rb +30 -1
- data/lib/ruby_reactor/context.rb +28 -0
- data/lib/ruby_reactor/context_serializer.rb +15 -0
- data/lib/ruby_reactor/dsl/async_macros.rb +176 -0
- data/lib/ruby_reactor/dsl/async_reactor_builder.rb +66 -0
- data/lib/ruby_reactor/dsl/compose_builder.rb +23 -4
- data/lib/ruby_reactor/dsl/interrupt_builder.rb +3 -2
- data/lib/ruby_reactor/dsl/interrupt_step_config.rb +7 -1
- data/lib/ruby_reactor/dsl/lockable.rb +3 -3
- data/lib/ruby_reactor/dsl/map_builder.rb +1 -2
- data/lib/ruby_reactor/dsl/reactor.rb +29 -10
- data/lib/ruby_reactor/dsl/step_builder.rb +30 -11
- data/lib/ruby_reactor/dsl/template_helpers.rb +11 -3
- data/lib/ruby_reactor/error/async_result_pending.rb +21 -0
- data/lib/ruby_reactor/error/async_wait_timeout_error.rb +10 -0
- data/lib/ruby_reactor/error/deprecated_dsl_error.rb +11 -0
- data/lib/ruby_reactor/executor/async_step_dispatch.rb +110 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +58 -45
- data/lib/ruby_reactor/executor/ordered_lock_support.rb +11 -11
- data/lib/ruby_reactor/executor/result_handler.rb +47 -11
- data/lib/ruby_reactor/executor/retry_manager.rb +13 -6
- data/lib/ruby_reactor/executor/step_executor.rb +77 -24
- data/lib/ruby_reactor/executor.rb +179 -23
- data/lib/ruby_reactor/lock.rb +19 -0
- data/lib/ruby_reactor/map/element_executor.rb +13 -2
- data/lib/ruby_reactor/map/helpers.rb +10 -8
- data/lib/ruby_reactor/map/result_enumerator.rb +7 -1
- data/lib/ruby_reactor/map/result_summary.rb +63 -0
- data/lib/ruby_reactor/map/sweeper.rb +1 -1
- data/lib/ruby_reactor/open_telemetry.rb +8 -5
- data/lib/ruby_reactor/ordered_lock.rb +3 -3
- data/lib/ruby_reactor/reactor.rb +25 -2
- data/lib/ruby_reactor/rspec/matchers.rb +61 -11
- data/lib/ruby_reactor/rspec/sidekiq_helpers.rb +2 -1
- data/lib/ruby_reactor/rspec/step_executor_patch.rb +2 -2
- data/lib/ruby_reactor/rspec/test_subject.rb +74 -19
- data/lib/ruby_reactor/semaphore.rb +10 -0
- data/lib/ruby_reactor/step/async_reactor_step.rb +207 -0
- data/lib/ruby_reactor/step/compose_step.rb +1 -1
- data/lib/ruby_reactor/step/map_step.rb +6 -2
- data/lib/ruby_reactor/step.rb +10 -4
- data/lib/ruby_reactor/step_signals.rb +33 -0
- data/lib/ruby_reactor/step_sweeper.rb +72 -0
- data/lib/ruby_reactor/step_worker.rb +260 -0
- data/lib/ruby_reactor/storage/adapter.rb +26 -1
- data/lib/ruby_reactor/storage/redis_adapter.rb +4 -71
- data/lib/ruby_reactor/storage/redis_locking.rb +7 -0
- data/lib/ruby_reactor/storage/redis_pub_sub.rb +31 -0
- data/lib/ruby_reactor/storage/redis_reactor_scan.rb +116 -0
- data/lib/ruby_reactor/storage/redis_step_results.rb +49 -0
- data/lib/ruby_reactor/sweeper.rb +7 -1
- data/lib/ruby_reactor/sweeper_job.rb +1 -0
- data/lib/ruby_reactor/template/result.rb +151 -5
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +117 -22
- data/lib/ruby_reactor/web/public/assets/index-BQvIWPdx.css +1 -0
- data/lib/ruby_reactor/web/public/assets/index-Dw4KV4QY.js +22 -0
- data/lib/ruby_reactor/web/public/index.html +2 -2
- data/lib/ruby_reactor/worker.rb +53 -3
- data/lib/ruby_reactor.rb +80 -10
- data/specs/active_job.md +1 -1
- metadata +23 -3
- data/lib/ruby_reactor/web/public/assets/index-CCnNVQy5.css +0 -1
- data/lib/ruby_reactor/web/public/assets/index-D7IBZvos.js +0 -21
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
module Step
|
|
5
|
+
# The dispatching half of `async_reactor`: everything that happens in the
|
|
6
|
+
# PARENT's process. The child then runs as an ordinary independently
|
|
7
|
+
# dispatched reactor execution — no new enqueue primitive, no new storage
|
|
8
|
+
# primitive, and no entry in the parent's compensation graph.
|
|
9
|
+
#
|
|
10
|
+
# Dispatch reuses the full pre-enqueue sequence of a top-level async run
|
|
11
|
+
# rather than a raw `perform_async`, because `Reactor#run` does
|
|
12
|
+
# three load-bearing things a naive `Context.new` + enqueue would silently
|
|
13
|
+
# skip: validate the child's inputs (the worker's resume path never
|
|
14
|
+
# validates, so skipping here starts a child on garbage), assign the
|
|
15
|
+
# ordered-lock nonce at ENQUEUE time (so ordering matches caller order), and
|
|
16
|
+
# persist before enqueueing (F2).
|
|
17
|
+
class AsyncReactorStep
|
|
18
|
+
include RubyReactor::Step
|
|
19
|
+
|
|
20
|
+
class << self
|
|
21
|
+
def run(arguments, context)
|
|
22
|
+
child_class = arguments[:async_reactor_class]
|
|
23
|
+
child_inputs = build_child_inputs(arguments[:argument_mappings] || {}, context)
|
|
24
|
+
|
|
25
|
+
# A dispatch-time failure fails the DISPATCHING step, i.e. normal saga
|
|
26
|
+
# handling in the parent. That is deliberately outside the
|
|
27
|
+
# no-auto-compensation rule, which governs the child's own execution.
|
|
28
|
+
validation = validate_child_inputs(child_class, child_inputs)
|
|
29
|
+
return validation if validation
|
|
30
|
+
|
|
31
|
+
deadlock = detect_lock_deadlock(child_class, child_inputs, context)
|
|
32
|
+
return deadlock if deadlock
|
|
33
|
+
|
|
34
|
+
return run_inline(child_class, child_inputs, context) if run_inline?(context)
|
|
35
|
+
|
|
36
|
+
dispatch(child_class, child_inputs, context)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def build_child_inputs(mappings, context)
|
|
42
|
+
mappings.transform_values { |source| source.resolve(context) }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def validate_child_inputs(child_class, child_inputs)
|
|
46
|
+
return nil unless child_class.respond_to?(:validate_inputs)
|
|
47
|
+
|
|
48
|
+
result = child_class.validate_inputs(child_inputs)
|
|
49
|
+
return nil unless result.failure?
|
|
50
|
+
|
|
51
|
+
RubyReactor.Failure(
|
|
52
|
+
"async_reactor child #{child_class.name} rejected its inputs: #{result.error.message}",
|
|
53
|
+
validation_errors: (result.error.field_errors if result.error.respond_to?(:field_errors))
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Lock ownership is NOT shared across the async boundary: parent
|
|
58
|
+
# and child run concurrently, so giving the child the parent's owner
|
|
59
|
+
# would put both inside the critical section at once — mutual exclusion
|
|
60
|
+
# silently broken, which is worse than a stall. Instead the one
|
|
61
|
+
# GUARANTEED deadlock (the parent holds a key its child will wait on,
|
|
62
|
+
# while the parent may go on to wait for that child) is caught here,
|
|
63
|
+
# loudly, at dispatch. Ordinary cross-execution contention on the same
|
|
64
|
+
# key is unaffected and still snoozes normally.
|
|
65
|
+
def detect_lock_deadlock(child_class, child_inputs, context)
|
|
66
|
+
held = held_lock_keys(context)
|
|
67
|
+
return nil if held.empty?
|
|
68
|
+
|
|
69
|
+
collision = child_lock_keys(child_class, child_inputs).find { |key| held.include?(key) }
|
|
70
|
+
return nil unless collision
|
|
71
|
+
|
|
72
|
+
RubyReactor.Failure(deadlock_message(collision, child_class, context))
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def child_lock_keys(child_class, child_inputs)
|
|
76
|
+
keys = []
|
|
77
|
+
if child_class.respond_to?(:lock_config) && child_class.lock_config
|
|
78
|
+
keys << child_class.lock_config[:key_proc].call(child_inputs)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# A semaphore with a single slot has the identical circular-wait
|
|
82
|
+
# shape. Higher limits are contention, not guaranteed deadlock.
|
|
83
|
+
semaphore = child_class.respond_to?(:semaphore_config) ? child_class.semaphore_config : nil
|
|
84
|
+
keys << semaphore[:key_proc].call(child_inputs) if semaphore && semaphore[:limit] == 1
|
|
85
|
+
|
|
86
|
+
keys.compact
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def held_lock_keys(context)
|
|
90
|
+
root = context.root_context || context
|
|
91
|
+
Array(root.private_data[:held_lock_keys] || root.private_data["held_lock_keys"])
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def deadlock_message(key, child_class, context)
|
|
95
|
+
parent = context.reactor_class&.name || "the dispatching reactor"
|
|
96
|
+
<<~MSG.strip
|
|
97
|
+
async_reactor dispatch of #{child_class.name || "<anonymous>"} would deadlock: it declares the \
|
|
98
|
+
lock key '#{key}', which #{parent} currently holds and will not release until it finishes.
|
|
99
|
+
The child would snooze forever, and if #{parent} later reads this child's result it would wait \
|
|
100
|
+
for work that can never start. Lock ownership is never shared across the async boundary — the \
|
|
101
|
+
two run concurrently, so sharing it would break mutual exclusion outright.
|
|
102
|
+
Fix, in order of preference:
|
|
103
|
+
1. Use `compose` instead of `async_reactor` if the child belongs inside #{parent}'s critical \
|
|
104
|
+
section and its result is needed — waiting for it means the work is sequential anyway.
|
|
105
|
+
2. Narrow the lock keys, if parent and child actually protect different resources.
|
|
106
|
+
3. Restructure so the locked reactor never reads the child's result — fire-and-forget, and \
|
|
107
|
+
verify in the child itself or in a successor reactor outside the lock window.
|
|
108
|
+
MSG
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# `RSpec::TestSubject`'s `async: false` clears the dispatch marker to run
|
|
112
|
+
# the whole reactor in one process.
|
|
113
|
+
def run_inline?(context)
|
|
114
|
+
config = context.reactor_class&.steps&.[](context.current_step)
|
|
115
|
+
config.respond_to?(:async_dispatch?) && !config.async_dispatch?
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def run_inline(child_class, child_inputs, context)
|
|
119
|
+
result = child_class.run(child_inputs)
|
|
120
|
+
store_reference(context, execution_id: (result.execution_id if result.respond_to?(:execution_id)),
|
|
121
|
+
child_class: child_class)
|
|
122
|
+
# Success(nil) for the same reason dispatch returns it: `result(:name)`
|
|
123
|
+
# must route through the reference (which now points at an already
|
|
124
|
+
# terminal child) rather than through a recorded value.
|
|
125
|
+
RubyReactor.Success(nil)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def dispatch(child_class, child_inputs, context)
|
|
129
|
+
child_context = build_child_context(child_class, child_inputs, context)
|
|
130
|
+
assign_ordered_lock_nonce!(child_class, child_context)
|
|
131
|
+
|
|
132
|
+
# Persist BEFORE enqueue (F2) — the payload is identity-only.
|
|
133
|
+
child_context.status = :running
|
|
134
|
+
# Marks the child as owned by a job of its own, which is what makes it
|
|
135
|
+
# sweepable: a compose child carries the same parent_context_id but runs
|
|
136
|
+
# inline in the parent, so re-enqueueing one would double-run it.
|
|
137
|
+
child_context.private_data[:async_dispatched] = true
|
|
138
|
+
save(child_context, child_class)
|
|
139
|
+
|
|
140
|
+
# The reference is written synchronously, by the process that is about
|
|
141
|
+
# to keep running other steps, so there is no cross-process race on it
|
|
142
|
+
# (unlike the child's eventual result, which the child itself writes).
|
|
143
|
+
store_reference(context, execution_id: child_context.context_id, child_class: child_class)
|
|
144
|
+
log_dispatch(context, child_class, child_context)
|
|
145
|
+
|
|
146
|
+
RubyReactor.configuration.async_router.perform_async(
|
|
147
|
+
child_context.context_id, RubyReactor.reactor_storage_name(child_class)
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
RubyReactor.Success(nil)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def build_child_context(child_class, child_inputs, context)
|
|
154
|
+
child_context = RubyReactor::Context.new(child_inputs, child_class)
|
|
155
|
+
# Linked for traceability only — the child is NOT nested inside the
|
|
156
|
+
# parent's context tree, because it must survive the parent finishing.
|
|
157
|
+
child_context.parent_context_id = context.context_id
|
|
158
|
+
child_context
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def assign_ordered_lock_nonce!(child_class, child_context)
|
|
162
|
+
return unless child_class.respond_to?(:ordered_lock_config) && child_class.ordered_lock_config
|
|
163
|
+
|
|
164
|
+
config = child_class.ordered_lock_config
|
|
165
|
+
key = config[:key_proc].call(child_context.inputs)
|
|
166
|
+
nonce, epoch = RubyReactor::OrderedLock.assign(key, ttl: config[:ttl])
|
|
167
|
+
|
|
168
|
+
child_context.private_data[:ordered_lock] = {
|
|
169
|
+
key: key, nonce: nonce, epoch: epoch,
|
|
170
|
+
poison_pill_timeout: config[:poison_pill_timeout],
|
|
171
|
+
ttl: config[:ttl], strict: config.fetch(:strict, true)
|
|
172
|
+
}
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def store_reference(context, execution_id:, child_class:)
|
|
176
|
+
context.composed_contexts[context.current_step] = {
|
|
177
|
+
name: context.current_step,
|
|
178
|
+
type: :async_reactor_ref,
|
|
179
|
+
execution_id: execution_id,
|
|
180
|
+
reactor_class_name: RubyReactor.reactor_storage_name(child_class),
|
|
181
|
+
dispatched_at: Time.now
|
|
182
|
+
}
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def save(child_context, child_class)
|
|
186
|
+
RubyReactor.configuration.storage_adapter.store_context(
|
|
187
|
+
child_context.context_id,
|
|
188
|
+
RubyReactor::ContextSerializer.serialize(child_context),
|
|
189
|
+
RubyReactor.reactor_storage_name(child_class)
|
|
190
|
+
)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# The link between parent and child, machine-parseable. Matters
|
|
194
|
+
# most here — a fire-and-forget child's failure may have no other surface
|
|
195
|
+
# in the parent at all.
|
|
196
|
+
def log_dispatch(context, child_class, child_context)
|
|
197
|
+
RubyReactor.configuration.logger.info(
|
|
198
|
+
"event=\"ruby_reactor.async_reactor.dispatched\" " \
|
|
199
|
+
"reactor=#{context.reactor_class&.name.inspect} step=#{context.current_step.inspect} " \
|
|
200
|
+
"execution_id=#{context.context_id.inspect} child_reactor=#{child_class.name.inspect} " \
|
|
201
|
+
"child_execution_id=#{child_context.context_id.inspect}"
|
|
202
|
+
)
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
end
|
|
@@ -108,7 +108,7 @@ module RubyReactor
|
|
|
108
108
|
end
|
|
109
109
|
|
|
110
110
|
def handle_execution_result(result)
|
|
111
|
-
return result if result.is_a?(RubyReactor::
|
|
111
|
+
return result if result.is_a?(RubyReactor::DispatchResult) || result.is_a?(RubyReactor::RetryQueuedResult)
|
|
112
112
|
|
|
113
113
|
if result.success?
|
|
114
114
|
RubyReactor.Success(result.value)
|
|
@@ -64,7 +64,7 @@ module RubyReactor
|
|
|
64
64
|
|
|
65
65
|
def run_inline(arguments, context)
|
|
66
66
|
results = execute_inline_map(arguments, context)
|
|
67
|
-
return results if results.is_a?(RubyReactor::Failure)
|
|
67
|
+
return results if results.is_a?(RubyReactor::Failure) || results.is_a?(RubyReactor::Halt)
|
|
68
68
|
|
|
69
69
|
process_results(results, arguments[:collect_block], arguments[:fail_fast])
|
|
70
70
|
end
|
|
@@ -76,6 +76,10 @@ module RubyReactor
|
|
|
76
76
|
arguments[:source].each do |element|
|
|
77
77
|
result = execute_single_element(element, arguments, context)
|
|
78
78
|
|
|
79
|
+
# An element-level Halt propagates as a run halt: stop immediately
|
|
80
|
+
# rather than being collected as a (nil) value.
|
|
81
|
+
return result if result.is_a?(RubyReactor::Halt)
|
|
82
|
+
|
|
79
83
|
if fail_fast && result.failure?
|
|
80
84
|
return result # Stop immediately on first failure
|
|
81
85
|
end
|
|
@@ -168,7 +172,7 @@ module RubyReactor
|
|
|
168
172
|
element_reactor_class: arguments[:mapped_reactor_class].name
|
|
169
173
|
}
|
|
170
174
|
|
|
171
|
-
RubyReactor::
|
|
175
|
+
RubyReactor::DispatchResult.new(
|
|
172
176
|
job_id: job_id,
|
|
173
177
|
intermediate_results: context.intermediate_results,
|
|
174
178
|
execution_id: context.context_id
|
data/lib/ruby_reactor/step.rb
CHANGED
|
@@ -7,6 +7,8 @@ module RubyReactor
|
|
|
7
7
|
end
|
|
8
8
|
|
|
9
9
|
module ClassMethods
|
|
10
|
+
include RubyReactor::StepSignals
|
|
11
|
+
|
|
10
12
|
# rubocop:disable Naming/MethodName
|
|
11
13
|
def Success(value = nil)
|
|
12
14
|
RubyReactor::Success(value)
|
|
@@ -16,8 +18,12 @@ module RubyReactor
|
|
|
16
18
|
RubyReactor::Failure(error)
|
|
17
19
|
end
|
|
18
20
|
|
|
19
|
-
def
|
|
20
|
-
RubyReactor.
|
|
21
|
+
def Halt(reason: nil, **kwargs)
|
|
22
|
+
RubyReactor.Halt(reason: reason, **kwargs)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def Skipped(...)
|
|
26
|
+
RubyReactor.Skipped(...)
|
|
21
27
|
end
|
|
22
28
|
# rubocop:enable Naming/MethodName
|
|
23
29
|
|
|
@@ -26,11 +32,11 @@ module RubyReactor
|
|
|
26
32
|
end
|
|
27
33
|
|
|
28
34
|
def compensate(_reason, _arguments, _context)
|
|
29
|
-
RubyReactor.
|
|
35
|
+
RubyReactor.Skipped() # Default: nothing defined, rollback continues
|
|
30
36
|
end
|
|
31
37
|
|
|
32
38
|
def undo(_result, _arguments, _context)
|
|
33
|
-
RubyReactor.
|
|
39
|
+
RubyReactor.Skipped() # Default: nothing defined, rollback continues
|
|
34
40
|
end
|
|
35
41
|
end
|
|
36
42
|
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
# `success!` / `skip!` / `fail!` / `halt!` — end a step body immediately
|
|
5
|
+
# with the matching signal, from any call depth. Mixed into both authoring
|
|
6
|
+
# surfaces (class steps and inline `run` blocks) so the helpers behave
|
|
7
|
+
# identically in either style (contracts/step-helpers.md).
|
|
8
|
+
#
|
|
9
|
+
# Implemented as throw/catch rather than an exception: a `throw` passes
|
|
10
|
+
# straight through `rescue Exception` while `ensure` blocks still run
|
|
11
|
+
# (verified in research.md R2), so a step's own broad rescue cannot swallow
|
|
12
|
+
# the author's intended outcome. The catching `catch(StepSignals::TAG)` lives
|
|
13
|
+
# at each step-body invocation site (step_executor.rb, compensation_manager.rb).
|
|
14
|
+
module StepSignals
|
|
15
|
+
TAG = :ruby_reactor_step_signal
|
|
16
|
+
|
|
17
|
+
def success!(value = nil)
|
|
18
|
+
throw TAG, RubyReactor.Success(value)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def skip!(...)
|
|
22
|
+
throw TAG, RubyReactor.Skipped(...)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def fail!(error, **opts)
|
|
26
|
+
throw TAG, RubyReactor.Failure(error, **opts)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def halt!(reason: nil, **kwargs)
|
|
30
|
+
throw TAG, RubyReactor.Halt(reason: reason, **kwargs)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
# Re-dispatches `async_step` units whose job was lost.
|
|
5
|
+
#
|
|
6
|
+
# An `async_step` is the one dispatched unit with no context of its own, so
|
|
7
|
+
# neither Sweeper (top-level contexts) nor Map::Sweeper (map elements) can see
|
|
8
|
+
# it. All it leaves behind is a Step Result Record stuck at `dispatched`, and
|
|
9
|
+
# because that record is also the re-attach marker, `already_dispatched?`
|
|
10
|
+
# actively refuses to dispatch again — so without this sweep a lost unit strands
|
|
11
|
+
# its parent forever, re-parking on every recovery of the parent itself.
|
|
12
|
+
#
|
|
13
|
+
# Liveness is the `async_step:` lock StepWorker holds for the life of the unit,
|
|
14
|
+
# mirroring how Sweeper reads `async:` and Map::Sweeper reads `map_element:`. A
|
|
15
|
+
# duplicate that races a live worker loses that lock and drops itself, so a
|
|
16
|
+
# mis-judged unit re-runs its body at most once.
|
|
17
|
+
class StepSweeper
|
|
18
|
+
DEFAULT_LIMIT = 1000
|
|
19
|
+
|
|
20
|
+
# The re-dispatch arguments, which the record carries verbatim because its
|
|
21
|
+
# key names the reactor that owns the step rather than the root the worker
|
|
22
|
+
# must load. Records written before recovery existed lack them and are skipped.
|
|
23
|
+
DISPATCH_KEYS = %w[root_context_id reactor_class_name step_context_id step_name].freeze
|
|
24
|
+
|
|
25
|
+
def self.run_once(limit: DEFAULT_LIMIT)
|
|
26
|
+
new.run_once(limit: limit)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def initialize(storage: nil, async_router: nil, logger: nil)
|
|
30
|
+
@storage = storage || RubyReactor.configuration.storage_adapter
|
|
31
|
+
@async_router = async_router || RubyReactor.configuration.async_router
|
|
32
|
+
@logger = logger || RubyReactor.configuration.logger
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Scans stored Step Result Records and re-dispatches the dispatched-but-dead
|
|
36
|
+
# ones. Returns the number re-dispatched.
|
|
37
|
+
def run_once(limit: DEFAULT_LIMIT)
|
|
38
|
+
redispatched = 0
|
|
39
|
+
|
|
40
|
+
@storage.scan_step_results(count: limit).each do |record|
|
|
41
|
+
next unless record["status"] == "dispatched"
|
|
42
|
+
|
|
43
|
+
arguments = dispatch_arguments(record)
|
|
44
|
+
next unless arguments
|
|
45
|
+
next if live?(arguments)
|
|
46
|
+
|
|
47
|
+
@async_router.perform_step_async(**arguments)
|
|
48
|
+
redispatched += 1
|
|
49
|
+
rescue StandardError => e
|
|
50
|
+
# One bad record must not abort the whole sweep.
|
|
51
|
+
@logger.warn("RubyReactor::StepSweeper failed to re-dispatch #{record["step_name"]}: #{e.class}: #{e.message}")
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
redispatched
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
def dispatch_arguments(record)
|
|
60
|
+
values = record.values_at(*DISPATCH_KEYS)
|
|
61
|
+
return nil if values.any?(&:nil?)
|
|
62
|
+
|
|
63
|
+
DISPATCH_KEYS.map(&:to_sym).zip(values).to_h
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def live?(arguments)
|
|
67
|
+
@storage.lock_held?(
|
|
68
|
+
RubyReactor.async_step_lock_key(arguments[:step_context_id], arguments[:step_name])
|
|
69
|
+
)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
# The body of one dispatched `async_step`, shared by every queueing backend
|
|
5
|
+
# (`Adapters::Sidekiq::StepWorker`, `Adapters::ActiveJob::StepWorker`) exactly
|
|
6
|
+
# as `Map::ElementExecutor` is shared by the map element workers.
|
|
7
|
+
#
|
|
8
|
+
# It is deliberately NOT a reactor run: it loads the parent context, resolves
|
|
9
|
+
# just this one step's arguments against it, runs the step body, writes the
|
|
10
|
+
# durable Step Result Record, and publishes the completion signal. Ordering is
|
|
11
|
+
# load-bearing — the record is written BEFORE the signal, so a reader that
|
|
12
|
+
# misses the (at-most-once) signal still finds the answer on its next
|
|
13
|
+
# fallback re-check.
|
|
14
|
+
class StepWorker
|
|
15
|
+
class << self
|
|
16
|
+
def perform(arguments)
|
|
17
|
+
arguments = arguments.transform_keys(&:to_sym)
|
|
18
|
+
new(**slice_arguments(arguments)).perform
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def slice_arguments(arguments)
|
|
24
|
+
{
|
|
25
|
+
root_context_id: arguments[:root_context_id],
|
|
26
|
+
reactor_class_name: arguments[:reactor_class_name],
|
|
27
|
+
step_context_id: arguments[:step_context_id],
|
|
28
|
+
step_name: arguments[:step_name].to_sym
|
|
29
|
+
}
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def initialize(root_context_id:, reactor_class_name:, step_context_id:, step_name:)
|
|
34
|
+
@root_context_id = root_context_id
|
|
35
|
+
@reactor_class_name = reactor_class_name
|
|
36
|
+
@step_context_id = step_context_id || root_context_id
|
|
37
|
+
@step_name = step_name
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The lock is what makes a lost unit recoverable: the record alone cannot say
|
|
41
|
+
# whether a `dispatched` unit is mid-flight or gone, so StepSweeper reads this
|
|
42
|
+
# lock as the liveness signal. It also drops a duplicate delivery rather than
|
|
43
|
+
# running the body a second time.
|
|
44
|
+
def perform
|
|
45
|
+
lock = acquire_liveness_lock
|
|
46
|
+
return if lock == :contended
|
|
47
|
+
|
|
48
|
+
perform_unit
|
|
49
|
+
ensure
|
|
50
|
+
lock.release if lock.respond_to?(:release)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def perform_unit
|
|
56
|
+
context = load_step_context
|
|
57
|
+
return record_missing_parent unless context
|
|
58
|
+
|
|
59
|
+
step_config = context.reactor_class&.steps&.[](@step_name)
|
|
60
|
+
return record_missing_step unless step_config
|
|
61
|
+
|
|
62
|
+
complete(run_step(context, step_config), context)
|
|
63
|
+
rescue StandardError => e
|
|
64
|
+
# The unit's failure belongs in its record, where a reader can see it.
|
|
65
|
+
# Raising instead would hand the job to the backend's retry machinery to
|
|
66
|
+
# fail identically N more times while every reader waits out its timeout.
|
|
67
|
+
log(:error, "failed", error: "#{e.class}: #{e.message}")
|
|
68
|
+
complete(RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name), nil)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def acquire_liveness_lock
|
|
72
|
+
# Inline testing re-enters this frame synchronously, so the lock would
|
|
73
|
+
# self-contend; it only guards cross-process delivery, impossible inline.
|
|
74
|
+
return :inline if inline_testing_mode?
|
|
75
|
+
|
|
76
|
+
lock = RubyReactor::Lock.new(
|
|
77
|
+
RubyReactor.async_step_lock_key(@step_context_id, @step_name),
|
|
78
|
+
owner: SecureRandom.uuid, ttl: RubyReactor.configuration.context_lock_ttl,
|
|
79
|
+
wait: 0, auto_extend: true
|
|
80
|
+
)
|
|
81
|
+
lock.acquire
|
|
82
|
+
lock
|
|
83
|
+
rescue RubyReactor::Lock::AcquisitionError
|
|
84
|
+
log(:info, "duplicate_dropped")
|
|
85
|
+
:contended
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def inline_testing_mode?
|
|
89
|
+
defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def run_step(context, step_config)
|
|
93
|
+
arguments = resolve_arguments(step_config, context)
|
|
94
|
+
log(:info, "running")
|
|
95
|
+
|
|
96
|
+
attempt = 0
|
|
97
|
+
result = nil
|
|
98
|
+
|
|
99
|
+
loop do
|
|
100
|
+
attempt += 1
|
|
101
|
+
result = execute_step_body(step_config, arguments, context)
|
|
102
|
+
break unless retry?(step_config, result, attempt)
|
|
103
|
+
|
|
104
|
+
delay = backoff_delay(step_config, attempt)
|
|
105
|
+
log(:warn, "retrying", attempt: attempt, delay: delay)
|
|
106
|
+
sleep(delay)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
result
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def execute_step_body(step_config, arguments, context)
|
|
113
|
+
result =
|
|
114
|
+
if step_config.has_run_block?
|
|
115
|
+
args = arguments.empty? ? context.inputs : arguments
|
|
116
|
+
step_config.run_block.call(args, context)
|
|
117
|
+
elsif step_config.has_impl?
|
|
118
|
+
step_config.impl.run(arguments, context)
|
|
119
|
+
else
|
|
120
|
+
RubyReactor.Failure("Step '#{@step_name}' has no implementation")
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
normalize(result)
|
|
124
|
+
rescue StandardError => e
|
|
125
|
+
RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Mirrors `Executor::RetryManager#can_retry_step?` for the one path that
|
|
129
|
+
# never reaches it: an `async_step`'s body runs entirely inside this
|
|
130
|
+
# worker, so retries here must be attempted synchronously in-process
|
|
131
|
+
# rather than requeued as a new job.
|
|
132
|
+
def retry?(step_config, result, attempt)
|
|
133
|
+
return false unless result.is_a?(RubyReactor::Failure) && result.retryable?
|
|
134
|
+
|
|
135
|
+
step_config.retryable? && attempt < step_config.retry_config[:max_attempts]
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def backoff_delay(step_config, attempt)
|
|
139
|
+
RetryContext.calculate_backoff_delay(
|
|
140
|
+
attempt, step_config.retry_config[:backoff], step_config.retry_config[:base_delay]
|
|
141
|
+
)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def normalize(result)
|
|
145
|
+
return result if result.is_a?(RubyReactor::Success) || result.is_a?(RubyReactor::Failure)
|
|
146
|
+
|
|
147
|
+
RubyReactor.Success(result)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def resolve_arguments(step_config, context)
|
|
151
|
+
step_config.arguments.to_h do |arg_name, arg_config|
|
|
152
|
+
value = arg_config[:source].resolve(context)
|
|
153
|
+
value = arg_config[:transform].call(value) if arg_config[:transform]
|
|
154
|
+
[arg_name, value]
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Write first, publish second. The record is the answer; the signal only
|
|
159
|
+
# saves the reader a fallback interval.
|
|
160
|
+
def complete(result, context)
|
|
161
|
+
storage.store_step_result(
|
|
162
|
+
@step_context_id, @step_name,
|
|
163
|
+
{
|
|
164
|
+
"status" => "completed",
|
|
165
|
+
"success" => result.success?,
|
|
166
|
+
"result" => ContextSerializer.serialize_value(result.success? ? result.value : result.to_h),
|
|
167
|
+
"completed_at" => Time.now.iso8601
|
|
168
|
+
},
|
|
169
|
+
@reactor_class_name
|
|
170
|
+
)
|
|
171
|
+
log(result.success? ? :info : :warn, result.success? ? "completed" : "completed_with_failure")
|
|
172
|
+
storage.publish(RubyReactor.async_step_channel(@step_context_id, @step_name), "done")
|
|
173
|
+
result
|
|
174
|
+
ensure
|
|
175
|
+
# A step body may have mutated the sub-context; nothing else will persist
|
|
176
|
+
# it, and the dashboard reads the parent's blob.
|
|
177
|
+
save_root(context) if context
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def record_missing_parent
|
|
181
|
+
# The parent was swept or outlived its retention window, so this
|
|
182
|
+
# unit's arguments can never be resolved. A record saying so beats a
|
|
183
|
+
# reader waiting out the full timeout for an answer that will never come.
|
|
184
|
+
log(:error, "parent_context_missing")
|
|
185
|
+
complete(
|
|
186
|
+
RubyReactor.Failure(
|
|
187
|
+
"Parent context #{@step_context_id} for async_step :#{@step_name} is no longer in storage " \
|
|
188
|
+
"(swept, or dispatched longer ago than `context_ttl`). The step's arguments cannot be resolved."
|
|
189
|
+
),
|
|
190
|
+
nil
|
|
191
|
+
)
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def record_missing_step
|
|
195
|
+
log(:error, "step_not_found")
|
|
196
|
+
complete(
|
|
197
|
+
RubyReactor.Failure("async_step :#{@step_name} is not defined on #{@reactor_class_name}"),
|
|
198
|
+
nil
|
|
199
|
+
)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def load_step_context
|
|
203
|
+
data = storage.retrieve_context(@root_context_id, @reactor_class_name)
|
|
204
|
+
return nil unless data
|
|
205
|
+
|
|
206
|
+
root = ContextSerializer.deserialize_hash(data)
|
|
207
|
+
@root_context = root
|
|
208
|
+
found = find_context(root, @step_context_id)
|
|
209
|
+
# The step runs in its own job; nothing it reaches should hand off again.
|
|
210
|
+
found&.inline_async_execution = true
|
|
211
|
+
found
|
|
212
|
+
rescue RubyReactor::Error::DeserializationError, RubyReactor::Error::SchemaVersionError => e
|
|
213
|
+
log(:error, "parent_context_unreadable", error: "#{e.class}: #{e.message}")
|
|
214
|
+
nil
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def find_context(context, target_id)
|
|
218
|
+
return context if context.context_id == target_id
|
|
219
|
+
|
|
220
|
+
context.composed_contexts.each_value do |entry|
|
|
221
|
+
next unless entry.is_a?(Hash) && entry[:context].is_a?(RubyReactor::Context)
|
|
222
|
+
|
|
223
|
+
found = find_context(entry[:context], target_id)
|
|
224
|
+
return found if found
|
|
225
|
+
end
|
|
226
|
+
nil
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def save_root(_context)
|
|
230
|
+
return unless @root_context
|
|
231
|
+
|
|
232
|
+
storage.store_context(@root_context.context_id, ContextSerializer.serialize(@root_context),
|
|
233
|
+
@reactor_class_name)
|
|
234
|
+
rescue StandardError => e
|
|
235
|
+
RubyReactor.configuration.logger.warn(
|
|
236
|
+
"RubyReactor: async_step :#{@step_name} could not persist its parent context: #{e.message}"
|
|
237
|
+
)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def storage
|
|
241
|
+
RubyReactor.configuration.storage_adapter
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Machine-parseable, and carrying enough identity to correlate a
|
|
245
|
+
# worker-side outcome with the parent execution — which matters more here
|
|
246
|
+
# than elsewhere, because a fire-and-forget failure may have no other surface.
|
|
247
|
+
def log(level, event, **extra)
|
|
248
|
+
fields = {
|
|
249
|
+
event: "ruby_reactor.async_step.#{event}",
|
|
250
|
+
reactor: @reactor_class_name,
|
|
251
|
+
step: @step_name,
|
|
252
|
+
execution_id: @step_context_id
|
|
253
|
+
}.merge(extra)
|
|
254
|
+
|
|
255
|
+
RubyReactor.configuration.logger.public_send(
|
|
256
|
+
level, fields.map { |k, v| "#{k}=#{v.inspect}" }.join(" ")
|
|
257
|
+
)
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
end
|
|
@@ -15,6 +15,27 @@ module RubyReactor
|
|
|
15
15
|
raise NotImplementedError
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
+
# The durable outcome of one `async_step`, keyed by (parent context, step
|
|
19
|
+
# name). A separate worker writes it concurrently with the still-running
|
|
20
|
+
# parent, so it deliberately lives OUTSIDE the parent's context blob —
|
|
21
|
+
# writing into that blob from two processes would race.
|
|
22
|
+
#
|
|
23
|
+
# `record` is a plain hash: at minimum `status` ("dispatched" or
|
|
24
|
+
# "completed"); a completed record also carries the serialized outcome.
|
|
25
|
+
# The `dispatched` record is written before the job is enqueued, so it
|
|
26
|
+
# doubles as the re-attach marker on recovery.
|
|
27
|
+
def store_step_result(context_id, step_name, record, reactor_class_name)
|
|
28
|
+
raise NotImplementedError
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def retrieve_step_result(context_id, step_name, reactor_class_name)
|
|
32
|
+
raise NotImplementedError
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def scan_step_results(count: 1000)
|
|
36
|
+
raise NotImplementedError
|
|
37
|
+
end
|
|
38
|
+
|
|
18
39
|
def retrieve_map_results(map_id, reactor_class_name, strict_ordering: true)
|
|
19
40
|
raise NotImplementedError
|
|
20
41
|
end
|
|
@@ -63,7 +84,11 @@ module RubyReactor
|
|
|
63
84
|
raise NotImplementedError
|
|
64
85
|
end
|
|
65
86
|
|
|
66
|
-
def scan_reactors(pattern: "*", count: 50)
|
|
87
|
+
def scan_reactors(pattern: "*", count: 50, include_dispatched_children: false)
|
|
88
|
+
raise NotImplementedError
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def scan_reactors_page(pattern: "*", cursor: "0", count: 50, include_dispatched_children: false)
|
|
67
92
|
raise NotImplementedError
|
|
68
93
|
end
|
|
69
94
|
|