ruby_reactor 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/skills/demo-app-e2e-verify/SKILL.md +226 -0
- data/.claude/skills/speckit-demo-tests/SKILL.md +144 -0
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/.specify/memory/constitution.md +79 -12
- data/.specify/templates/tasks-template.md +7 -0
- data/CHANGELOG.md +125 -1
- data/CLAUDE.md +5 -0
- data/README.md +155 -25
- data/lib/ruby_reactor/adapters/active_job/router.rb +21 -6
- data/lib/ruby_reactor/adapters/active_job/step_worker.rb +20 -0
- data/lib/ruby_reactor/adapters/active_job/worker.rb +6 -1
- data/lib/ruby_reactor/adapters/sidekiq/router.rb +21 -6
- data/lib/ruby_reactor/adapters/sidekiq/step_worker.rb +17 -0
- data/lib/ruby_reactor/adapters/sidekiq/worker.rb +5 -2
- data/lib/ruby_reactor/async_waiter.rb +113 -0
- data/lib/ruby_reactor/configuration.rb +30 -1
- data/lib/ruby_reactor/context.rb +28 -0
- data/lib/ruby_reactor/context_serializer.rb +15 -0
- data/lib/ruby_reactor/dsl/async_macros.rb +176 -0
- data/lib/ruby_reactor/dsl/async_reactor_builder.rb +66 -0
- data/lib/ruby_reactor/dsl/compose_builder.rb +23 -4
- data/lib/ruby_reactor/dsl/interrupt_builder.rb +3 -2
- data/lib/ruby_reactor/dsl/interrupt_step_config.rb +7 -1
- data/lib/ruby_reactor/dsl/lockable.rb +3 -3
- data/lib/ruby_reactor/dsl/map_builder.rb +1 -2
- data/lib/ruby_reactor/dsl/reactor.rb +29 -10
- data/lib/ruby_reactor/dsl/step_builder.rb +30 -11
- data/lib/ruby_reactor/dsl/template_helpers.rb +11 -3
- data/lib/ruby_reactor/error/async_result_pending.rb +21 -0
- data/lib/ruby_reactor/error/async_wait_timeout_error.rb +10 -0
- data/lib/ruby_reactor/error/deprecated_dsl_error.rb +11 -0
- data/lib/ruby_reactor/executor/async_step_dispatch.rb +110 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +58 -45
- data/lib/ruby_reactor/executor/ordered_lock_support.rb +11 -11
- data/lib/ruby_reactor/executor/result_handler.rb +47 -11
- data/lib/ruby_reactor/executor/retry_manager.rb +13 -6
- data/lib/ruby_reactor/executor/step_executor.rb +77 -24
- data/lib/ruby_reactor/executor.rb +179 -23
- data/lib/ruby_reactor/lock.rb +19 -0
- data/lib/ruby_reactor/map/element_executor.rb +13 -2
- data/lib/ruby_reactor/map/helpers.rb +10 -8
- data/lib/ruby_reactor/map/result_enumerator.rb +7 -1
- data/lib/ruby_reactor/map/result_summary.rb +63 -0
- data/lib/ruby_reactor/map/sweeper.rb +1 -1
- data/lib/ruby_reactor/open_telemetry.rb +8 -5
- data/lib/ruby_reactor/ordered_lock.rb +3 -3
- data/lib/ruby_reactor/reactor.rb +25 -2
- data/lib/ruby_reactor/rspec/matchers.rb +61 -11
- data/lib/ruby_reactor/rspec/sidekiq_helpers.rb +2 -1
- data/lib/ruby_reactor/rspec/step_executor_patch.rb +2 -2
- data/lib/ruby_reactor/rspec/test_subject.rb +74 -19
- data/lib/ruby_reactor/semaphore.rb +10 -0
- data/lib/ruby_reactor/step/async_reactor_step.rb +207 -0
- data/lib/ruby_reactor/step/compose_step.rb +1 -1
- data/lib/ruby_reactor/step/map_step.rb +6 -2
- data/lib/ruby_reactor/step.rb +10 -4
- data/lib/ruby_reactor/step_signals.rb +33 -0
- data/lib/ruby_reactor/step_sweeper.rb +72 -0
- data/lib/ruby_reactor/step_worker.rb +260 -0
- data/lib/ruby_reactor/storage/adapter.rb +26 -1
- data/lib/ruby_reactor/storage/redis_adapter.rb +4 -71
- data/lib/ruby_reactor/storage/redis_locking.rb +7 -0
- data/lib/ruby_reactor/storage/redis_pub_sub.rb +31 -0
- data/lib/ruby_reactor/storage/redis_reactor_scan.rb +116 -0
- data/lib/ruby_reactor/storage/redis_step_results.rb +49 -0
- data/lib/ruby_reactor/sweeper.rb +7 -1
- data/lib/ruby_reactor/sweeper_job.rb +1 -0
- data/lib/ruby_reactor/template/result.rb +151 -5
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +117 -22
- data/lib/ruby_reactor/web/public/assets/index-BQvIWPdx.css +1 -0
- data/lib/ruby_reactor/web/public/assets/index-Dw4KV4QY.js +22 -0
- data/lib/ruby_reactor/web/public/index.html +2 -2
- data/lib/ruby_reactor/worker.rb +53 -3
- data/lib/ruby_reactor.rb +80 -10
- data/specs/active_job.md +1 -1
- metadata +23 -3
- data/lib/ruby_reactor/web/public/assets/index-CCnNVQy5.css +0 -1
- data/lib/ruby_reactor/web/public/assets/index-D7IBZvos.js +0 -21
|
@@ -94,12 +94,15 @@ module RubyReactor
|
|
|
94
94
|
|
|
95
95
|
def handle_retry_result(step_config, reactor_class, result)
|
|
96
96
|
case result
|
|
97
|
-
when RubyReactor::Success
|
|
97
|
+
when RubyReactor::Halt, RubyReactor::Skipped, RubyReactor::Success
|
|
98
|
+
# Halt and Skipped are Success subclasses, so they already take this
|
|
99
|
+
# path via inheritance; the explicit arms are readability plus a
|
|
100
|
+
# guard against a future hierarchy change (R5).
|
|
98
101
|
clear_retry_state
|
|
99
102
|
result
|
|
100
103
|
when RubyReactor::Failure
|
|
101
104
|
handle_failure_result(step_config, reactor_class, result)
|
|
102
|
-
when RetryQueuedResult, RubyReactor::
|
|
105
|
+
when RetryQueuedResult, RubyReactor::DispatchResult
|
|
103
106
|
# Pass through async results
|
|
104
107
|
result
|
|
105
108
|
else
|
|
@@ -126,8 +129,12 @@ module RubyReactor
|
|
|
126
129
|
@context
|
|
127
130
|
)
|
|
128
131
|
|
|
129
|
-
# Check if we should requeue (async retry)
|
|
130
|
-
|
|
132
|
+
# Check if we should requeue (async retry). The per-step `async` flag is
|
|
133
|
+
# gone: a step relocated by `background` fails inside the worker, where
|
|
134
|
+
# `inline_async_execution` already answers this — and a step failing
|
|
135
|
+
# BEFORE the hand-off point genuinely has no worker to requeue into, so
|
|
136
|
+
# it must retry synchronously.
|
|
137
|
+
is_async = reactor_class.async? ||
|
|
131
138
|
@context.root_context&.reactor_class&.async? ||
|
|
132
139
|
@context.inline_async_execution
|
|
133
140
|
|
|
@@ -142,9 +149,9 @@ module RubyReactor
|
|
|
142
149
|
def handle_async_retry(step_config, reactor_class, result)
|
|
143
150
|
requeue_result = requeue_job_for_step_retry(step_config, result.error, reactor_class)
|
|
144
151
|
|
|
145
|
-
# If it returned an
|
|
152
|
+
# If it returned an DispatchResult, we are truly async.
|
|
146
153
|
# Otherwise, it ran inline and we should return the result of that execution.
|
|
147
|
-
if requeue_result.is_a?(RubyReactor::
|
|
154
|
+
if requeue_result.is_a?(RubyReactor::DispatchResult)
|
|
148
155
|
RetryQueuedResult.new(
|
|
149
156
|
step_config.name,
|
|
150
157
|
@context.retry_context.attempts_for_step(step_config.name),
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
module RubyReactor
|
|
4
4
|
class Executor
|
|
5
5
|
class StepExecutor
|
|
6
|
+
include AsyncStepDispatch
|
|
7
|
+
|
|
6
8
|
def initialize(context:, dependency_graph:, reactor_class:, managers:)
|
|
7
9
|
@context = context
|
|
8
10
|
@dependency_graph = dependency_graph
|
|
@@ -30,15 +32,15 @@ module RubyReactor
|
|
|
30
32
|
result = execute_step(step_config)
|
|
31
33
|
|
|
32
34
|
# If step execution was handed off to async, return the async result
|
|
33
|
-
return result if result.is_a?(RubyReactor::
|
|
35
|
+
return result if result.is_a?(RubyReactor::DispatchResult)
|
|
34
36
|
|
|
35
37
|
# If a step returns RetryQueuedResult, we need to stop and return it
|
|
36
38
|
return result if result.is_a?(RetryQueuedResult)
|
|
37
39
|
|
|
38
|
-
# If a step returns
|
|
40
|
+
# If a step returns Halt, stop the reactor cleanly (no
|
|
39
41
|
# compensation). Must be checked BEFORE Failure / Success because
|
|
40
|
-
#
|
|
41
|
-
return result if result.is_a?(RubyReactor::
|
|
42
|
+
# Halt is a Success subclass.
|
|
43
|
+
return result if result.is_a?(RubyReactor::Halt)
|
|
42
44
|
|
|
43
45
|
# If a step returns Failure, we need to stop execution and return it
|
|
44
46
|
return result if result.is_a?(RubyReactor::Failure)
|
|
@@ -46,13 +48,17 @@ module RubyReactor
|
|
|
46
48
|
# If a step returns InterruptResult, we need to stop execution and return it
|
|
47
49
|
return result if result.is_a?(RubyReactor::InterruptResult)
|
|
48
50
|
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
51
|
+
# A Skipped step (or a plain Success) continues the loop — Skipped
|
|
52
|
+
# is a Success subclass, so this also fires the durable checkpoint
|
|
53
|
+
# for it, same as a plain success.
|
|
54
|
+
#
|
|
55
|
+
# Only a continue-Success/Skipped reaches here (Async/Retry/Halt/
|
|
56
|
+
# Failure/Interrupt all returned above; nil is inline-async test
|
|
57
|
+
# mode). It is the one outcome where the loop proceeds to more
|
|
58
|
+
# steps with no other save in between — every terminal/handoff
|
|
59
|
+
# result persists via its own path. Write a durable checkpoint so
|
|
60
|
+
# a crash re-runs at most this one step. Ordering: side-effect ->
|
|
61
|
+
# record result (inside execute_step) -> checkpoint here.
|
|
56
62
|
@on_step_complete&.call if result.is_a?(RubyReactor::Success)
|
|
57
63
|
end
|
|
58
64
|
end
|
|
@@ -62,25 +68,36 @@ module RubyReactor
|
|
|
62
68
|
end
|
|
63
69
|
|
|
64
70
|
def execute_step(step_config)
|
|
65
|
-
# If we're already in inline async execution mode (inside Worker),
|
|
66
|
-
# treat async steps as sync to avoid infinite recursion
|
|
67
|
-
|
|
68
71
|
if @dependency_graph.completed.include?(step_config.name)
|
|
69
72
|
return RubyReactor.Success(@context.get_result(step_config.name))
|
|
70
73
|
end
|
|
71
74
|
|
|
72
|
-
|
|
75
|
+
# Decided BEFORE argument resolution: resolving can block (or park) on
|
|
76
|
+
# an async `result(:name)`, and when the step body is about to be
|
|
77
|
+
# dispatched elsewhere — a `before:` hand-off, or an `async_step`'s
|
|
78
|
+
# own worker — that wait belongs to the process that will actually run
|
|
79
|
+
# it, not this one. (`async_reactor` still resolves here: its resolved
|
|
80
|
+
# values are the child's INPUTS, needed at dispatch.)
|
|
81
|
+
deferred_body = step_config.async_dispatch == :step || handoff_at?(step_config, :before)
|
|
82
|
+
resolved_arguments = deferred_body ? {} : resolve_arguments(step_config)
|
|
73
83
|
|
|
74
84
|
@middlewares.on(:start_step, step_config.name, resolved_arguments, @context)
|
|
75
85
|
completed = false
|
|
76
86
|
begin
|
|
77
87
|
result = if step_config.interrupt?
|
|
78
88
|
handle_interrupt_step(step_config)
|
|
79
|
-
elsif step_config.
|
|
80
|
-
|
|
89
|
+
elsif step_config.async_dispatch == :step
|
|
90
|
+
dispatch_async_step(step_config)
|
|
91
|
+
elsif handoff_at?(step_config, :before)
|
|
92
|
+
# `before: :x` hands off INSTEAD of running :x, leaving its
|
|
93
|
+
# graph node incomplete for the worker to pick up.
|
|
94
|
+
handle_background_handoff(step_config)
|
|
81
95
|
else
|
|
82
96
|
execute_step_with_retry(step_config, resolved_arguments)
|
|
83
97
|
end
|
|
98
|
+
# `after: :x` hands off once :x's result is recorded — the step really
|
|
99
|
+
# did run here, and only what remains moves to the worker.
|
|
100
|
+
result = handle_background_handoff(step_config) if handoff_after?(step_config, result)
|
|
84
101
|
completed = true
|
|
85
102
|
if result.is_a?(RubyReactor::Failure)
|
|
86
103
|
@middlewares.on(:failed_step, step_config.name, result, @context)
|
|
@@ -96,6 +113,37 @@ module RubyReactor
|
|
|
96
113
|
|
|
97
114
|
private
|
|
98
115
|
|
|
116
|
+
# The reactor's single hand-off point, `{ mode: :after|:before, step: }`.
|
|
117
|
+
# Nil for a reactor that never declares `background`.
|
|
118
|
+
def background_handoff
|
|
119
|
+
return @background_handoff if defined?(@background_handoff)
|
|
120
|
+
|
|
121
|
+
@background_handoff =
|
|
122
|
+
(@reactor_class.background_handoff if @reactor_class.respond_to?(:background_handoff))
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Hand-off is keyed to REACHING the named step, not to where the
|
|
126
|
+
# declaration sits in the class body. A step whose `where`/guard says it
|
|
127
|
+
# must not run never triggers it — the hand-off only ever relocates work
|
|
128
|
+
# that is actually going to happen. Inside the worker the whole thing is
|
|
129
|
+
# suppressed (`inline_async_execution`) so it cannot re-trigger.
|
|
130
|
+
def handoff_at?(step_config, mode)
|
|
131
|
+
point = background_handoff
|
|
132
|
+
return false unless point && point[:mode] == mode && point[:step] == step_config.name
|
|
133
|
+
return false if @context.inline_async_execution
|
|
134
|
+
|
|
135
|
+
step_config.should_run?(@context)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Post-execution trigger for `after:`. Only a plain continue-Success means
|
|
139
|
+
# the named step actually completed here — a Failure, Skipped, interrupt,
|
|
140
|
+
# queued retry or an already-async result each own the flow instead.
|
|
141
|
+
def handoff_after?(step_config, result)
|
|
142
|
+
return false unless result.is_a?(RubyReactor::Success) && !result.is_a?(RubyReactor::Skipped)
|
|
143
|
+
|
|
144
|
+
handoff_at?(step_config, :after)
|
|
145
|
+
end
|
|
146
|
+
|
|
99
147
|
def reconstruct_failure(data)
|
|
100
148
|
return data if data.is_a?(RubyReactor::Failure)
|
|
101
149
|
return nil unless data.is_a?(Hash)
|
|
@@ -125,7 +173,7 @@ module RubyReactor
|
|
|
125
173
|
safe_execute_step_sync(step_config, resolved_arguments)
|
|
126
174
|
end
|
|
127
175
|
|
|
128
|
-
unless result.is_a?(RetryQueuedResult) || result.is_a?(RubyReactor::
|
|
176
|
+
unless result.is_a?(RetryQueuedResult) || result.is_a?(RubyReactor::DispatchResult)
|
|
129
177
|
@result_handler.handle_step_result(step_config, result, resolved_arguments)
|
|
130
178
|
end
|
|
131
179
|
|
|
@@ -197,9 +245,12 @@ module RubyReactor
|
|
|
197
245
|
end
|
|
198
246
|
end
|
|
199
247
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
248
|
+
# Hand every step not yet executed to a worker job, and return the
|
|
249
|
+
# `DispatchResult` that halts `execute_all_steps` in the calling process.
|
|
250
|
+
# Shared verbatim by both `background` forms — they differ only in WHERE
|
|
251
|
+
# the trigger sits, never in what the hand-off does.
|
|
252
|
+
def handle_background_handoff(step_config)
|
|
253
|
+
log_async_event("background.handoff", step_config.name)
|
|
203
254
|
@context.current_step = step_config.name
|
|
204
255
|
@context.undo_stack = @compensation_manager.undo_stack
|
|
205
256
|
|
|
@@ -289,15 +340,17 @@ module RubyReactor
|
|
|
289
340
|
end
|
|
290
341
|
|
|
291
342
|
def run_step_implementation(step_config, arguments)
|
|
292
|
-
@context.
|
|
343
|
+
@context.append_execution_trace(
|
|
344
|
+
{ type: :run, step: step_config.name, timestamp: Time.now, arguments: arguments }
|
|
345
|
+
)
|
|
293
346
|
if step_config.has_run_block?
|
|
294
347
|
# Execute inline block
|
|
295
348
|
# If no arguments are defined for the step, pass the reactor inputs as arguments
|
|
296
349
|
args_to_pass = arguments.empty? ? @context.inputs : arguments
|
|
297
|
-
step_config.run_block.call(args_to_pass, @context)
|
|
350
|
+
catch(StepSignals::TAG) { step_config.run_block.call(args_to_pass, @context) }
|
|
298
351
|
elsif step_config.has_impl?
|
|
299
352
|
# Execute step class
|
|
300
|
-
step_config.impl.run(arguments, @context)
|
|
353
|
+
catch(StepSignals::TAG) { step_config.impl.run(arguments, @context) }
|
|
301
354
|
else
|
|
302
355
|
raise Error::ValidationError.new(
|
|
303
356
|
"Step '#{step_config.name}' has no implementation",
|
|
@@ -6,6 +6,7 @@ require_relative "executor/graph_manager"
|
|
|
6
6
|
require_relative "executor/retry_manager"
|
|
7
7
|
require_relative "executor/compensation_manager"
|
|
8
8
|
require_relative "executor/result_handler"
|
|
9
|
+
require_relative "executor/async_step_dispatch"
|
|
9
10
|
require_relative "executor/step_executor"
|
|
10
11
|
require_relative "executor/ordered_lock_support"
|
|
11
12
|
|
|
@@ -53,6 +54,7 @@ module RubyReactor
|
|
|
53
54
|
@acquired_semaphore = nil
|
|
54
55
|
@acquired_context_lock = nil
|
|
55
56
|
@context_lock_owner = nil
|
|
57
|
+
@parked = false
|
|
56
58
|
@contention_snooze = false
|
|
57
59
|
@skip_context_persist = false
|
|
58
60
|
@last_checkpoint_at = nil
|
|
@@ -101,15 +103,16 @@ module RubyReactor
|
|
|
101
103
|
input_validator = InputValidator.new(@reactor_class, @context)
|
|
102
104
|
input_validator.validate!
|
|
103
105
|
|
|
106
|
+
reset_held_lock_keys!
|
|
104
107
|
acquire_locks_with_telemetry
|
|
105
108
|
|
|
106
109
|
# Re-check the period gate now that we hold the lock. The pre-lock check
|
|
107
110
|
# is a fast path; this one closes the race where two callers both passed
|
|
108
111
|
# it and then serialized on the lock — without it the second caller would
|
|
109
112
|
# re-run work the first already marked. (No-op when no lock is configured.)
|
|
110
|
-
if (
|
|
113
|
+
if (halted = check_period_gate)
|
|
111
114
|
completed = true
|
|
112
|
-
return
|
|
115
|
+
return finalize_halt(halted)
|
|
113
116
|
end
|
|
114
117
|
|
|
115
118
|
@context.status = :running
|
|
@@ -132,6 +135,13 @@ module RubyReactor
|
|
|
132
135
|
RubyReactor::OrderedLock::WaitError => e
|
|
133
136
|
@contention_snooze = true
|
|
134
137
|
raise e
|
|
138
|
+
rescue Error::AsyncResultPending
|
|
139
|
+
# Only reachable when this executor runs nested inside a worker (a
|
|
140
|
+
# composed child; sync callers never park). Propagate to the ROOT
|
|
141
|
+
# resume, which owns the park. This child's own lock/semaphore (if any)
|
|
142
|
+
# ARE released below and re-competed for on redelivery.
|
|
143
|
+
@contention_snooze = true
|
|
144
|
+
raise
|
|
135
145
|
rescue StandardError => e
|
|
136
146
|
@result = @result_handler.handle_execution_error(e)
|
|
137
147
|
update_context_status(@result)
|
|
@@ -192,17 +202,25 @@ module RubyReactor
|
|
|
192
202
|
# and must not contend on the root's own key.
|
|
193
203
|
acquire_context_lock
|
|
194
204
|
|
|
205
|
+
reset_held_lock_keys!
|
|
206
|
+
|
|
195
207
|
# Resumes intentionally skip check_rate_limit (a paused run must not
|
|
196
208
|
# block itself on resume), so acquire lock/semaphore directly rather
|
|
197
|
-
# than via acquire_locks.
|
|
198
|
-
|
|
199
|
-
|
|
209
|
+
# than via acquire_locks. A context parked on an async result kept its
|
|
210
|
+
# primitives held across the gap — re-adopt them instead of re-competing.
|
|
211
|
+
parked = consume_parked_primitives!
|
|
212
|
+
if @reactor_class.respond_to?(:lock_config) && @reactor_class.lock_config
|
|
213
|
+
acquire_exclusive_lock(reattach: parked[:lock])
|
|
214
|
+
end
|
|
215
|
+
if @reactor_class.respond_to?(:semaphore_config) && @reactor_class.semaphore_config
|
|
216
|
+
acquire_semaphore(reattach_token: parked[:semaphore_token])
|
|
217
|
+
end
|
|
200
218
|
|
|
201
219
|
# Post-lock re-check (see execute) — closes the period race for the
|
|
202
220
|
# first run of a locked async reactor.
|
|
203
|
-
if first_run && (
|
|
221
|
+
if first_run && (halted = check_period_gate)
|
|
204
222
|
completed = true
|
|
205
|
-
return
|
|
223
|
+
return finalize_halt(halted)
|
|
206
224
|
end
|
|
207
225
|
|
|
208
226
|
prepare_for_resume
|
|
@@ -227,13 +245,21 @@ module RubyReactor
|
|
|
227
245
|
RubyReactor::OrderedLock::WaitError => e
|
|
228
246
|
@contention_snooze = true
|
|
229
247
|
raise e
|
|
248
|
+
rescue Error::AsyncResultPending => e
|
|
249
|
+
# An awaited async unit is not terminal yet: park. Exclusive lock and
|
|
250
|
+
# semaphore stay HELD (recorded on the context for the resuming job to
|
|
251
|
+
# re-adopt); the worker snoozes the job. The context lock is still
|
|
252
|
+
# released below — the redelivered job must be able to take it.
|
|
253
|
+
park_held_primitives!
|
|
254
|
+
@contention_snooze = true
|
|
255
|
+
raise e
|
|
230
256
|
rescue StandardError => e
|
|
231
257
|
handle_resume_error(e)
|
|
232
258
|
update_context_status(@result)
|
|
233
259
|
completed = true
|
|
234
260
|
@result
|
|
235
261
|
ensure
|
|
236
|
-
release_locks
|
|
262
|
+
release_locks unless @parked
|
|
237
263
|
@acquired_context_lock&.release
|
|
238
264
|
@acquired_context_lock = nil
|
|
239
265
|
leave_ordered_lock_scope
|
|
@@ -265,6 +291,25 @@ module RubyReactor
|
|
|
265
291
|
# Serialize context
|
|
266
292
|
serialized_context = ContextSerializer.serialize(@context)
|
|
267
293
|
storage.store_context(@context.context_id, serialized_context, reactor_class_name)
|
|
294
|
+
publish_completion_signal(storage)
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# Wake any parent blocked in the notified wait on this execution. Published
|
|
298
|
+
# AFTER the durable save, never before: the context row is the answer and
|
|
299
|
+
# the signal only saves the waiter a fallback interval. Unconditional —
|
|
300
|
+
# publishing to a channel with no subscribers is near-free, so there is no
|
|
301
|
+
# need for an "am I awaited?" marker.
|
|
302
|
+
def publish_completion_signal(storage)
|
|
303
|
+
return unless @context.finished?
|
|
304
|
+
|
|
305
|
+
log_completion
|
|
306
|
+
storage.publish(RubyReactor.async_reactor_channel(@context.context_id), @context.status.to_s)
|
|
307
|
+
rescue StandardError => e
|
|
308
|
+
# The signal is an optimisation; losing it costs the waiter one fallback
|
|
309
|
+
# interval and must never fail the run that just completed.
|
|
310
|
+
RubyReactor.configuration.logger.warn(
|
|
311
|
+
"RubyReactor: could not publish completion signal for #{@context.context_id}: #{e.message}"
|
|
312
|
+
)
|
|
268
313
|
end
|
|
269
314
|
|
|
270
315
|
# Durable per-step checkpoint. Unlike save_context (which serializes THIS
|
|
@@ -348,16 +393,16 @@ module RubyReactor
|
|
|
348
393
|
@context.current_step.nil? && @context.intermediate_results.empty?
|
|
349
394
|
end
|
|
350
395
|
|
|
351
|
-
# Record and persist a
|
|
396
|
+
# Record and persist a Halt result, then return it. Shared by the
|
|
352
397
|
# pre-lock and post-lock period gates in both execute and resume.
|
|
353
|
-
def
|
|
354
|
-
@result =
|
|
398
|
+
def finalize_halt(halted)
|
|
399
|
+
@result = halted
|
|
355
400
|
update_context_status(@result)
|
|
356
401
|
save_context
|
|
357
402
|
@result
|
|
358
403
|
end
|
|
359
404
|
|
|
360
|
-
# Returns a
|
|
405
|
+
# Returns a Halt result if the period bucket is already marked, else nil.
|
|
361
406
|
# Consulted before AND after lock acquisition on a first execution; genuine
|
|
362
407
|
# resumes never re-check (a paused run must not skip itself when its own
|
|
363
408
|
# marker eventually appears).
|
|
@@ -368,13 +413,13 @@ module RubyReactor
|
|
|
368
413
|
key = period_key(config)
|
|
369
414
|
return nil unless RubyReactor.configuration.storage_adapter.period_seen?(key)
|
|
370
415
|
|
|
371
|
-
RubyReactor::
|
|
416
|
+
RubyReactor::Halt.new(reason: :period, period_key: key)
|
|
372
417
|
end
|
|
373
418
|
|
|
374
419
|
def mark_period_on_success(result)
|
|
375
420
|
return unless @reactor_class.respond_to?(:period_config) && @reactor_class.period_config
|
|
376
421
|
return unless result.is_a?(RubyReactor::Success)
|
|
377
|
-
return if result.is_a?(RubyReactor::
|
|
422
|
+
return if result.is_a?(RubyReactor::Halt)
|
|
378
423
|
|
|
379
424
|
config = @reactor_class.period_config
|
|
380
425
|
ttl = RubyReactor::Period.ttl_seconds(config[:every])
|
|
@@ -386,6 +431,33 @@ module RubyReactor
|
|
|
386
431
|
RubyReactor::Period.key(base, config[:every])
|
|
387
432
|
end
|
|
388
433
|
|
|
434
|
+
# One machine-parseable line whenever an execution reaches a terminal
|
|
435
|
+
# state, carrying the parent link. A child dispatched fire-and-forget may
|
|
436
|
+
# have no other surface in its parent at all, so a failure entry also names
|
|
437
|
+
# the reason.
|
|
438
|
+
def log_completion
|
|
439
|
+
return unless @context.parent_context_id
|
|
440
|
+
|
|
441
|
+
fields = {
|
|
442
|
+
event: "ruby_reactor.async_reactor.completed",
|
|
443
|
+
reactor: @reactor_class&.name,
|
|
444
|
+
execution_id: @context.context_id,
|
|
445
|
+
parent_execution_id: @context.parent_context_id,
|
|
446
|
+
status: @context.status.to_s
|
|
447
|
+
}
|
|
448
|
+
fields[:failure] = failure_summary if @context.failed?
|
|
449
|
+
|
|
450
|
+
RubyReactor.configuration.logger.public_send(
|
|
451
|
+
@context.failed? ? :warn : :info,
|
|
452
|
+
fields.map { |k, v| "#{k}=#{v.inspect}" }.join(" ")
|
|
453
|
+
)
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def failure_summary
|
|
457
|
+
reason = @context.failure_reason
|
|
458
|
+
reason.respond_to?(:error) ? reason.error.to_s : reason.to_s
|
|
459
|
+
end
|
|
460
|
+
|
|
389
461
|
# Per-execution liveness lock on the root context id. Owner is a fresh UUID
|
|
390
462
|
# per execution (NOT the context_id): a duplicate delivery of the *same*
|
|
391
463
|
# context from a different worker must be blocked, so reentrancy by id would
|
|
@@ -422,7 +494,7 @@ module RubyReactor
|
|
|
422
494
|
defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
|
|
423
495
|
end
|
|
424
496
|
|
|
425
|
-
def acquire_exclusive_lock
|
|
497
|
+
def acquire_exclusive_lock(reattach: false)
|
|
426
498
|
config = @reactor_class.lock_config
|
|
427
499
|
key = config[:key_proc].call(@context.inputs)
|
|
428
500
|
|
|
@@ -436,9 +508,21 @@ module RubyReactor
|
|
|
436
508
|
wait: contention_wait(config[:wait]),
|
|
437
509
|
auto_extend: config.fetch(:auto_extend, true)
|
|
438
510
|
)
|
|
511
|
+
|
|
512
|
+
# Re-adopting a lock held across a parked gap: no :lock_acquired event —
|
|
513
|
+
# the original acquisition already emitted it, and the eventual release
|
|
514
|
+
# emits exactly one :lock_released. A lapsed TTL falls through to a
|
|
515
|
+
# fresh acquire.
|
|
516
|
+
if reattach && lock.reattach
|
|
517
|
+
@acquired_lock = lock
|
|
518
|
+
held_lock_keys << key
|
|
519
|
+
return
|
|
520
|
+
end
|
|
521
|
+
|
|
439
522
|
begin
|
|
440
523
|
lock.acquire
|
|
441
524
|
@acquired_lock = lock
|
|
525
|
+
held_lock_keys << key
|
|
442
526
|
middlewares.on(:lock_acquired, key, @context)
|
|
443
527
|
rescue RubyReactor::Lock::AcquisitionError => e
|
|
444
528
|
middlewares.on(:lock_failed, key, e, @context)
|
|
@@ -446,15 +530,29 @@ module RubyReactor
|
|
|
446
530
|
end
|
|
447
531
|
end
|
|
448
532
|
|
|
449
|
-
def acquire_semaphore
|
|
533
|
+
def acquire_semaphore(reattach_token: nil)
|
|
450
534
|
config = @reactor_class.semaphore_config
|
|
451
535
|
key = config[:key_proc].call(@context.inputs)
|
|
452
536
|
limit = config[:limit]
|
|
453
537
|
|
|
454
538
|
semaphore = RubyReactor::Semaphore.new(key, limit: limit, wait: contention_wait(config[:wait]))
|
|
539
|
+
|
|
540
|
+
# Same shape as the lock reattach above: keep the slot held across the
|
|
541
|
+
# parked gap, no duplicate :semaphore_acquired event, fall through to a
|
|
542
|
+
# fresh acquire when the token was lost in between.
|
|
543
|
+
if reattach_token && semaphore.reattach(reattach_token)
|
|
544
|
+
@acquired_semaphore = semaphore
|
|
545
|
+
held_lock_keys << key if limit == 1
|
|
546
|
+
return
|
|
547
|
+
end
|
|
548
|
+
|
|
455
549
|
begin
|
|
456
550
|
semaphore.acquire
|
|
457
551
|
@acquired_semaphore = semaphore
|
|
552
|
+
# Only a single-slot semaphore has the circular-wait shape the
|
|
553
|
+
# async_reactor deadlock guard can act on; higher limits are ordinary
|
|
554
|
+
# contention and must keep snoozing.
|
|
555
|
+
held_lock_keys << key if limit == 1
|
|
458
556
|
middlewares.on(:semaphore_acquired, key, limit, @context)
|
|
459
557
|
rescue RubyReactor::Semaphore::AcquisitionError => e
|
|
460
558
|
middlewares.on(:semaphore_failed, key, limit, e, @context)
|
|
@@ -471,10 +569,47 @@ module RubyReactor
|
|
|
471
569
|
configured_wait
|
|
472
570
|
end
|
|
473
571
|
|
|
572
|
+
# Park on a pending async result: keep exclusive lock / semaphore checked
|
|
573
|
+
# out through the gap, recording just enough on the (about-to-be-saved)
|
|
574
|
+
# context for the resuming job to re-adopt them. The lock's auto-extender
|
|
575
|
+
# dies with this process, so the parked gap is bounded by the lock TTL —
|
|
576
|
+
# the snooze redelivery (seconds) sits comfortably inside the default 60s.
|
|
577
|
+
def park_held_primitives!
|
|
578
|
+
@parked = true
|
|
579
|
+
parked = {}
|
|
580
|
+
|
|
581
|
+
if @acquired_lock
|
|
582
|
+
@acquired_lock.detach
|
|
583
|
+
parked[:lock] = true
|
|
584
|
+
@acquired_lock = nil
|
|
585
|
+
end
|
|
586
|
+
|
|
587
|
+
if @acquired_semaphore
|
|
588
|
+
parked[:semaphore_token] = @acquired_semaphore.token
|
|
589
|
+
@acquired_semaphore = nil
|
|
590
|
+
end
|
|
591
|
+
|
|
592
|
+
@context.private_data[:parked_primitives] = parked if parked.any?
|
|
593
|
+
end
|
|
594
|
+
|
|
595
|
+
# One-shot: the marker is deleted on read so a crash after this point
|
|
596
|
+
# degrades to a fresh acquire (reentrant by owner for the lock) rather
|
|
597
|
+
# than a stale reattach on some later, unrelated resume.
|
|
598
|
+
def consume_parked_primitives!
|
|
599
|
+
raw = @context.private_data.delete(:parked_primitives) ||
|
|
600
|
+
@context.private_data.delete("parked_primitives") || {}
|
|
601
|
+
|
|
602
|
+
{
|
|
603
|
+
lock: raw[:lock] || raw["lock"],
|
|
604
|
+
semaphore_token: raw[:semaphore_token] || raw["semaphore_token"]
|
|
605
|
+
}
|
|
606
|
+
end
|
|
607
|
+
|
|
474
608
|
def release_locks
|
|
475
609
|
if @acquired_semaphore
|
|
476
610
|
key = @acquired_semaphore.key
|
|
477
611
|
release_one("semaphore", @acquired_semaphore)
|
|
612
|
+
held_lock_keys.delete(key)
|
|
478
613
|
middlewares.on(:semaphore_released, key, @context)
|
|
479
614
|
end
|
|
480
615
|
@acquired_semaphore = nil
|
|
@@ -483,10 +618,29 @@ module RubyReactor
|
|
|
483
618
|
|
|
484
619
|
key = @acquired_lock.key
|
|
485
620
|
release_one("lock", @acquired_lock)
|
|
621
|
+
held_lock_keys.delete(key)
|
|
486
622
|
@acquired_lock = nil
|
|
487
623
|
middlewares.on(:lock_released, key, @context)
|
|
488
624
|
end
|
|
489
625
|
|
|
626
|
+
# Exclusive keys this EXECUTION currently holds, recorded on the root
|
|
627
|
+
# context so a dispatching step anywhere in the tree can see the whole
|
|
628
|
+
# chain. Read by the async_reactor deadlock guard; nothing else
|
|
629
|
+
# depends on it, so a stale entry can only cost a false positive — hence
|
|
630
|
+
# the reset on the way in.
|
|
631
|
+
def held_lock_keys
|
|
632
|
+
root = @context.root_context || @context
|
|
633
|
+
root.private_data[:held_lock_keys] ||= []
|
|
634
|
+
end
|
|
635
|
+
|
|
636
|
+
# A rehydrated context can carry keys from the process that died holding
|
|
637
|
+
# them. Only the root executor resets, and only on the way in.
|
|
638
|
+
def reset_held_lock_keys!
|
|
639
|
+
return unless (@context.root_context || @context).equal?(@context)
|
|
640
|
+
|
|
641
|
+
(@context.root_context || @context).private_data[:held_lock_keys] = []
|
|
642
|
+
end
|
|
643
|
+
|
|
490
644
|
def release_one(kind, primitive)
|
|
491
645
|
released = primitive.release
|
|
492
646
|
return if released
|
|
@@ -506,10 +660,10 @@ module RubyReactor
|
|
|
506
660
|
return unless result
|
|
507
661
|
|
|
508
662
|
case result
|
|
509
|
-
when RubyReactor::
|
|
663
|
+
when RubyReactor::DispatchResult
|
|
510
664
|
@context.status = :running
|
|
511
|
-
when RubyReactor::
|
|
512
|
-
@context.status = :
|
|
665
|
+
when RubyReactor::Halt
|
|
666
|
+
@context.status = :halted
|
|
513
667
|
when RubyReactor::Success
|
|
514
668
|
@context.status = :completed
|
|
515
669
|
when RubyReactor::Failure
|
|
@@ -542,14 +696,16 @@ module RubyReactor
|
|
|
542
696
|
@result = @step_executor.execute_all_steps
|
|
543
697
|
else
|
|
544
698
|
case result
|
|
545
|
-
#
|
|
699
|
+
# Halt must be listed before Success (Halt < Success) so the
|
|
546
700
|
# halt path wins over the "continue with remaining steps" path.
|
|
547
|
-
|
|
701
|
+
# Skipped is NOT listed here — it is a Success subclass and must
|
|
702
|
+
# continue with the remaining steps, same as a plain Success.
|
|
703
|
+
when RubyReactor::Halt,
|
|
548
704
|
RetryQueuedResult,
|
|
549
705
|
RubyReactor::Failure,
|
|
550
|
-
RubyReactor::
|
|
706
|
+
RubyReactor::DispatchResult,
|
|
551
707
|
RubyReactor::InterruptResult
|
|
552
|
-
# Terminal: step
|
|
708
|
+
# Terminal: step halted, requeued, failed, paused, or handed
|
|
553
709
|
# off to async. Return the result as-is.
|
|
554
710
|
@result = result
|
|
555
711
|
when RubyReactor::Success
|
data/lib/ruby_reactor/lock.rb
CHANGED
|
@@ -54,6 +54,25 @@ module RubyReactor
|
|
|
54
54
|
adapter.lock_release(@key, @owner)
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
+
# Park support: stop refreshing the TTL but LEAVE the key held — the
|
|
58
|
+
# parked execution keeps ownership through the gap, bounded by the TTL.
|
|
59
|
+
def detach
|
|
60
|
+
stop_extender
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Resume ownership after a parked gap: verify we are still the owner and
|
|
64
|
+
# refresh the TTL — WITHOUT incrementing the reentrancy count. The count
|
|
65
|
+
# was never decremented at park, so a plain `acquire` here would bump it
|
|
66
|
+
# to 2 and the final release would leave the key behind until TTL.
|
|
67
|
+
# Returns false when ownership lapsed (TTL expired mid-park); the caller
|
|
68
|
+
# falls back to a fresh acquire.
|
|
69
|
+
def reattach # rubocop:disable Naming/PredicateMethod
|
|
70
|
+
return false unless adapter.lock_extend(@key, @owner, @ttl)
|
|
71
|
+
|
|
72
|
+
start_extender if @auto_extend
|
|
73
|
+
true
|
|
74
|
+
end
|
|
75
|
+
|
|
57
76
|
def synchronize
|
|
58
77
|
acquire
|
|
59
78
|
yield
|
|
@@ -141,12 +141,23 @@ module RubyReactor
|
|
|
141
141
|
index = arguments[:index]
|
|
142
142
|
parent_class = arguments[:parent_reactor_class_name] # Using short name for variable
|
|
143
143
|
|
|
144
|
-
if result.
|
|
144
|
+
if result.halted?
|
|
145
|
+
# A Halt must not be collected as a (nil) value indistinguishable
|
|
146
|
+
# from an ordinary success — mark it so the enumerator reconstructs
|
|
147
|
+
# a real Halt for the consumer.
|
|
148
|
+
storage.store_map_result(map_id, index, { _halt: true, reason: result.reason },
|
|
149
|
+
parent_class, strict_ordering: arguments[:strict_ordering])
|
|
150
|
+
elsif result.success?
|
|
145
151
|
storage.store_map_result(map_id, index, ContextSerializer.serialize_value(result.value),
|
|
146
152
|
parent_class, strict_ordering: arguments[:strict_ordering])
|
|
147
153
|
else
|
|
148
154
|
executor.undo_all
|
|
149
|
-
|
|
155
|
+
# Store the whole serialized Failure, not just its message: step_name,
|
|
156
|
+
# backtrace, file_path and code_snippet are the only record of why this
|
|
157
|
+
# element failed once its context row expires, and the dashboard has
|
|
158
|
+
# nothing else to show for a non-fail_fast map.
|
|
159
|
+
storage.store_map_result(map_id, index,
|
|
160
|
+
{ _error: ContextSerializer.serialize_value(result) }, parent_class,
|
|
150
161
|
strict_ordering: arguments[:strict_ordering])
|
|
151
162
|
|
|
152
163
|
if arguments[:fail_fast]
|
|
@@ -95,14 +95,16 @@ module RubyReactor
|
|
|
95
95
|
|
|
96
96
|
# Manually update execution trace to reflect completion
|
|
97
97
|
# This is necessary because resume_execution continues from the NEXT step
|
|
98
|
-
# and the async step (which returned
|
|
99
|
-
parent_context.
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
98
|
+
# and the async step (which returned DispatchResult) needs to be marked as done with actual value
|
|
99
|
+
parent_context.append_execution_trace(
|
|
100
|
+
{
|
|
101
|
+
type: :result,
|
|
102
|
+
step: step_name_sym,
|
|
103
|
+
timestamp: Time.now,
|
|
104
|
+
value: final_result.value,
|
|
105
|
+
status: :success
|
|
106
|
+
}
|
|
107
|
+
)
|
|
106
108
|
|
|
107
109
|
parent_context.current_step = nil
|
|
108
110
|
executor.resume_execution
|