ruby_reactor 0.8.2 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/skills/speckit-review/SKILL.md +324 -0
- data/.release-please-manifest.json +1 -1
- data/.specify/extensions.yml +10 -0
- data/.specify/feature.json +1 -1
- data/.specify/workflows/speckit/workflow.yml +13 -1
- data/.specify/workflows/workflow-registry.json +2 -2
- data/CHANGELOG.md +94 -0
- data/CLAUDE.md +2 -2
- data/README.md +128 -93
- data/lib/ruby_reactor/adapters/active_job/router.rb +19 -0
- data/lib/ruby_reactor/adapters/sidekiq/router.rb +21 -0
- data/lib/ruby_reactor/context.rb +26 -0
- data/lib/ruby_reactor/context_serializer.rb +4 -2
- data/lib/ruby_reactor/dsl/async_reactor_builder.rb +3 -6
- data/lib/ruby_reactor/dsl/compose_builder.rb +3 -10
- data/lib/ruby_reactor/dsl/interrupt_builder.rb +14 -0
- data/lib/ruby_reactor/dsl/lockable.rb +76 -21
- data/lib/ruby_reactor/dsl/reactor.rb +8 -11
- data/lib/ruby_reactor/dsl/retryable.rb +45 -0
- data/lib/ruby_reactor/dsl/step_builder.rb +150 -13
- data/lib/ruby_reactor/error/async_result_pending.rb +1 -1
- data/lib/ruby_reactor/error/execution_parked.rb +16 -0
- data/lib/ruby_reactor/error/reactor_contention_park.rb +26 -0
- data/lib/ruby_reactor/error/step_contention_park.rb +26 -0
- data/lib/ruby_reactor/error/undeclared_input_error.rb +13 -0
- data/lib/ruby_reactor/executor/async_step_dispatch.rb +109 -3
- data/lib/ruby_reactor/executor/compensation_manager.rb +99 -17
- data/lib/ruby_reactor/executor/ordered_lock_support.rb +76 -44
- data/lib/ruby_reactor/executor/result_handler.rb +34 -13
- data/lib/ruby_reactor/executor/retry_manager.rb +20 -13
- data/lib/ruby_reactor/executor/step_coordination.rb +788 -0
- data/lib/ruby_reactor/executor/step_executor.rb +115 -11
- data/lib/ruby_reactor/executor.rb +90 -20
- data/lib/ruby_reactor/map/element_executor.rb +24 -2
- data/lib/ruby_reactor/map/helpers.rb +35 -11
- data/lib/ruby_reactor/max_retries_exhausted_failure.rb +2 -2
- data/lib/ruby_reactor/open_telemetry.rb +61 -24
- data/lib/ruby_reactor/retry_context.rb +31 -2
- data/lib/ruby_reactor/rspec/helpers.rb +15 -0
- data/lib/ruby_reactor/rspec/matchers.rb +92 -0
- data/lib/ruby_reactor/rspec/test_subject.rb +9 -5
- data/lib/ruby_reactor/step/async_reactor_step.rb +46 -26
- data/lib/ruby_reactor/step/compose_step.rb +22 -7
- data/lib/ruby_reactor/step/input_contract.rb +5 -0
- data/lib/ruby_reactor/step/inputs.rb +57 -0
- data/lib/ruby_reactor/step/map_step.rb +32 -22
- data/lib/ruby_reactor/step.rb +59 -11
- data/lib/ruby_reactor/step_sweeper.rb +29 -1
- data/lib/ruby_reactor/step_worker.rb +260 -37
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +72 -7
- data/lib/ruby_reactor/web/coordination_serializer.rb +120 -2
- data/lib/ruby_reactor/web/public/assets/{index-Dw4KV4QY.js → index-CeZU-ESu.js} +9 -9
- data/lib/ruby_reactor/web/public/index.html +1 -1
- data/lib/ruby_reactor/worker.rb +56 -30
- data/lib/ruby_reactor.rb +30 -6
- data/specs/{004-inheritable-step-class → 006-step-retry-declarations}/checklists/requirements.md +10 -9
- data/specs/006-step-retry-declarations/contracts/dsl-surface.md +89 -0
- data/specs/006-step-retry-declarations/data-model.md +58 -0
- data/specs/006-step-retry-declarations/plan.md +187 -0
- data/specs/006-step-retry-declarations/quickstart.md +80 -0
- data/specs/006-step-retry-declarations/research.md +194 -0
- data/specs/006-step-retry-declarations/spec.md +453 -0
- data/specs/006-step-retry-declarations/tasks.md +382 -0
- data/specs/future_improvements.md +250 -0
- data/specs/specs-inputs-by-method-md-piped-wigderson.md +77 -0
- metadata +20 -28
- data/specs/002-step-input-contracts/checklists/requirements.md +0 -49
- data/specs/002-step-input-contracts/contracts/dsl-surface.md +0 -193
- data/specs/002-step-input-contracts/data-model.md +0 -115
- data/specs/002-step-input-contracts/plan.md +0 -165
- data/specs/002-step-input-contracts/quickstart.md +0 -170
- data/specs/002-step-input-contracts/research.md +0 -233
- data/specs/002-step-input-contracts/spec.md +0 -359
- data/specs/002-step-input-contracts/tasks.md +0 -367
- data/specs/004-inheritable-step-class/contracts/step-lifecycle.md +0 -85
- data/specs/004-inheritable-step-class/data-model.md +0 -116
- data/specs/004-inheritable-step-class/plan.md +0 -174
- data/specs/004-inheritable-step-class/quickstart.md +0 -112
- data/specs/004-inheritable-step-class/research.md +0 -308
- data/specs/004-inheritable-step-class/spec.md +0 -316
- data/specs/004-inheritable-step-class/tasks.md +0 -258
- data/specs/active_job.md +0 -259
- data/specs/deferred-003-step-lock-declarations/checklists/requirements.md +0 -51
- data/specs/deferred-003-step-lock-declarations/contracts/dsl-surface.md +0 -154
- data/specs/deferred-003-step-lock-declarations/data-model.md +0 -131
- data/specs/deferred-003-step-lock-declarations/plan.md +0 -166
- data/specs/deferred-003-step-lock-declarations/quickstart.md +0 -169
- data/specs/deferred-003-step-lock-declarations/research.md +0 -196
- data/specs/deferred-003-step-lock-declarations/spec.md +0 -447
- data/specs/deferred-003-step-lock-declarations/tasks.md +0 -572
- data/specs/possible_feature.md +0 -22
data/lib/ruby_reactor/step.rb
CHANGED
|
@@ -5,17 +5,24 @@ module RubyReactor
|
|
|
5
5
|
#
|
|
6
6
|
# class MyStep < RubyReactor::Step
|
|
7
7
|
# input :amount, :integer
|
|
8
|
-
# def run = Success(charged: inputs
|
|
8
|
+
# def run = Success(charged: inputs.amount)
|
|
9
9
|
# end
|
|
10
10
|
#
|
|
11
11
|
# Lifecycle of every class-level call (`.run`/`.call`, `.undo`, `.compensate`):
|
|
12
12
|
#
|
|
13
13
|
# 1. Resolve `inputs`: the given arguments with the contract's defaults
|
|
14
|
-
# applied. `run`, `undo`, and `compensate` all see the same values
|
|
14
|
+
# applied. `run`, `undo`, and `compensate` all see the same values,
|
|
15
|
+
# through a read-only `Step::Inputs` (`inputs.amount`).
|
|
15
16
|
# 2. `.run` ONLY: enforce the declared input contract, raising
|
|
16
|
-
# `Error::InputValidationError` before any instance exists.
|
|
17
|
-
#
|
|
18
|
-
#
|
|
17
|
+
# `Error::InputValidationError` before any instance exists.
|
|
18
|
+
# 2b. `.run` ONLY: step-scoped coordination (`with_lock` etc, if declared)
|
|
19
|
+
# is acquired around the rest of `.run`, keyed on the just-validated
|
|
20
|
+
# inputs — after contract enforcement, so a step that will fail
|
|
21
|
+
# validation never takes a hold (research Finding 7 / Finding 8). A
|
|
22
|
+
# reactor never goes through here: it calls `.run_without_coordination`
|
|
23
|
+
# and coordinates the step itself (research D2).
|
|
24
|
+
# `.undo` and `.compensate` NEVER enforce the input contract: rollback
|
|
25
|
+
# must not fail on the very inputs that may have caused the failure.
|
|
19
26
|
# 3. Build a FRESH instance, never reused across actions. An ivar set in
|
|
20
27
|
# `run` is gone by the time `undo` runs on its own instance, so async
|
|
21
28
|
# execution running `run` and `undo` in different processes behaves
|
|
@@ -23,15 +30,21 @@ module RubyReactor
|
|
|
23
30
|
# 4. Invoke the matching instance method, translating any `StepSignals`
|
|
24
31
|
# throw (`success!`/`skip!`/`fail!`/`halt!`) into its result wrapper.
|
|
25
32
|
#
|
|
26
|
-
# No `prepend`/`
|
|
27
|
-
#
|
|
33
|
+
# No `prepend`/`define_method`/`method_missing` — every step in the class
|
|
34
|
+
# reads top to bottom as ordinary method calls. The two `extend`s are
|
|
35
|
+
# `Dsl::Lockable::ClassMethods` (the five coordination macros) and
|
|
36
|
+
# `Dsl::Retryable` (`retries`), self-contained modules whose only hook is
|
|
37
|
+
# an `inherited` that copies their declarations down (Finding 9).
|
|
28
38
|
class Step
|
|
29
39
|
include RubyReactor::StepSignals
|
|
40
|
+
extend RubyReactor::Dsl::Lockable::ClassMethods
|
|
41
|
+
extend RubyReactor::Dsl::Retryable
|
|
30
42
|
|
|
31
43
|
attr_reader :inputs, :context, :result, :reason
|
|
32
44
|
|
|
33
45
|
def initialize(inputs, context, result: nil, reason: nil)
|
|
34
|
-
|
|
46
|
+
contract = self.class.input_contract if self.class.declares_inputs?
|
|
47
|
+
@inputs = Inputs.new(inputs, contract: contract, owner: self.class.name || self.class.inspect)
|
|
35
48
|
@context = context
|
|
36
49
|
@result = result
|
|
37
50
|
@reason = reason
|
|
@@ -68,12 +81,28 @@ module RubyReactor
|
|
|
68
81
|
# rubocop:enable Naming/MethodName
|
|
69
82
|
|
|
70
83
|
class << self
|
|
71
|
-
|
|
84
|
+
# A DIRECT invocation — application code, or another step's body. It is
|
|
85
|
+
# its own unit of work: it takes this class's coordination itself and,
|
|
86
|
+
# having no queue to park into, waits then fails (FR-016/FR-023).
|
|
87
|
+
# `context` is optional: a stand-alone `ChargeStep.run(args)` is its own
|
|
88
|
+
# execution, with no reactor context to inherit ownership from.
|
|
89
|
+
def run(arguments, context = nil)
|
|
72
90
|
validated = enforce_contract!(arguments)
|
|
73
|
-
|
|
91
|
+
coordinate(validated, context) { run_without_coordination(validated, context) }
|
|
74
92
|
end
|
|
75
93
|
alias call run
|
|
76
94
|
|
|
95
|
+
# The reactor's entry (`StepConfig#call_body`): the executor or
|
|
96
|
+
# `StepWorker` has already taken this step's EFFECTIVE coordination —
|
|
97
|
+
# the class's declarations and any inline ones, in one fixed order — so
|
|
98
|
+
# taking the class's here again would split acquisition across two
|
|
99
|
+
# layers. Still enforces the contract (idempotent: it only applies
|
|
100
|
+
# defaults to already-valid arguments).
|
|
101
|
+
def run_without_coordination(arguments, context)
|
|
102
|
+
validated = enforce_contract!(arguments)
|
|
103
|
+
catch(StepSignals::TAG) { new(validated, context).run }
|
|
104
|
+
end
|
|
105
|
+
|
|
77
106
|
# Same `inputs` as `.run` (defaults applied), but NEVER enforces the contract.
|
|
78
107
|
def undo(result, arguments, context)
|
|
79
108
|
catch(StepSignals::TAG) { new(with_defaults(arguments), context, result: result).undo }
|
|
@@ -117,15 +146,34 @@ module RubyReactor
|
|
|
117
146
|
|
|
118
147
|
private
|
|
119
148
|
|
|
149
|
+
# Step-scoped coordination for a DIRECT call (`direct: true`): never
|
|
150
|
+
# parks, keeps no state on `context`, and so can never be mistaken for
|
|
151
|
+
# the coordination of the reactor step whose body made the call.
|
|
152
|
+
# After `enforce_contract!`, so a step that will fail validation never
|
|
153
|
+
# takes a hold (Finding 8). A bare step pays one check. `context` may be
|
|
154
|
+
# nil (a stand-alone `MyStep.run(args)`) — `StepCoordination#owner`
|
|
155
|
+
# handles that case.
|
|
156
|
+
def coordinate(validated, context, &block)
|
|
157
|
+
return block.call if Executor::StepCoordination.none?(self)
|
|
158
|
+
|
|
159
|
+
ctx = context if context.is_a?(RubyReactor::Context)
|
|
160
|
+
Executor::StepCoordination.new(
|
|
161
|
+
step_config: self, arguments: validated, context: ctx, reactor_class: ctx&.reactor_class,
|
|
162
|
+
middlewares: ctx&.middlewares || Executor.middlewares_for(ctx&.reactor_class), direct: true
|
|
163
|
+
).around_run(&block)
|
|
164
|
+
end
|
|
165
|
+
|
|
120
166
|
def own_input_contract
|
|
121
167
|
@own_input_contract ||= Step::InputContract.new(owner: self)
|
|
122
168
|
end
|
|
123
169
|
|
|
170
|
+
# `to_h`: a step body may hand its own `Inputs` on (`OtherStep.run(inputs, context)`).
|
|
124
171
|
def with_defaults(arguments)
|
|
125
|
-
input_contract.apply_defaults(arguments)
|
|
172
|
+
input_contract.apply_defaults(arguments.to_h)
|
|
126
173
|
end
|
|
127
174
|
|
|
128
175
|
def enforce_contract!(arguments)
|
|
176
|
+
arguments = arguments.to_h
|
|
129
177
|
return arguments unless declares_inputs?
|
|
130
178
|
|
|
131
179
|
input_contract.enforce!(arguments)
|
|
@@ -42,9 +42,10 @@ module RubyReactor
|
|
|
42
42
|
|
|
43
43
|
arguments = dispatch_arguments(record)
|
|
44
44
|
next unless arguments
|
|
45
|
+
next if parked?(record)
|
|
45
46
|
next if live?(arguments)
|
|
46
47
|
|
|
47
|
-
|
|
48
|
+
redispatch(record, arguments)
|
|
48
49
|
redispatched += 1
|
|
49
50
|
rescue StandardError => e
|
|
50
51
|
# One bad record must not abort the whole sweep.
|
|
@@ -56,6 +57,17 @@ module RubyReactor
|
|
|
56
57
|
|
|
57
58
|
private
|
|
58
59
|
|
|
60
|
+
# A swept park must not restart its contention counter: rebuilt from the
|
|
61
|
+
# record alone, a zeroed payload would let `lock_snooze_max_attempts`
|
|
62
|
+
# never bite. `perform_step_in(0, ...)` is the only re-dispatch that
|
|
63
|
+
# carries the count.
|
|
64
|
+
def redispatch(record, arguments)
|
|
65
|
+
attempts = record["contention_attempts"].to_i
|
|
66
|
+
return @async_router.perform_step_async(**arguments) unless attempts.positive?
|
|
67
|
+
|
|
68
|
+
@async_router.perform_step_in(0, **arguments, contention_attempts: attempts)
|
|
69
|
+
end
|
|
70
|
+
|
|
59
71
|
def dispatch_arguments(record)
|
|
60
72
|
values = record.values_at(*DISPATCH_KEYS)
|
|
61
73
|
return nil if values.any?(&:nil?)
|
|
@@ -63,6 +75,22 @@ module RubyReactor
|
|
|
63
75
|
DISPATCH_KEYS.map(&:to_sym).zip(values).to_h
|
|
64
76
|
end
|
|
65
77
|
|
|
78
|
+
# A unit parked on step-level contention (StepWorker#handle_contention)
|
|
79
|
+
# released its liveness lock on purpose and has a redelivery already
|
|
80
|
+
# scheduled. It looks exactly like a lost unit, so without this the sweep
|
|
81
|
+
# would dispatch a duplicate that runs the body a second time once the
|
|
82
|
+
# redelivery fires. Past the stamped window it is fair game again — a
|
|
83
|
+
# redelivery that eventually lands on a finished unit is dropped by
|
|
84
|
+
# `StepWorker#already_completed?`.
|
|
85
|
+
def parked?(record)
|
|
86
|
+
parked_until = record["parked_until"]
|
|
87
|
+
return false unless parked_until
|
|
88
|
+
|
|
89
|
+
Time.iso8601(parked_until) > Time.now
|
|
90
|
+
rescue ArgumentError, TypeError
|
|
91
|
+
false
|
|
92
|
+
end
|
|
93
|
+
|
|
66
94
|
def live?(arguments)
|
|
67
95
|
@storage.lock_held?(
|
|
68
96
|
RubyReactor.async_step_lock_key(arguments[:step_context_id], arguments[:step_name])
|
|
@@ -11,7 +11,23 @@ module RubyReactor
|
|
|
11
11
|
# load-bearing — the record is written BEFORE the signal, so a reader that
|
|
12
12
|
# misses the (at-most-once) signal still finds the answer on its next
|
|
13
13
|
# fallback re-check.
|
|
14
|
+
#
|
|
15
|
+
# Single writer: a context is written only by the execution that owns it,
|
|
16
|
+
# and this unit is not the parent's execution. It reads the parent's context
|
|
17
|
+
# and NEVER writes it back — the parent may be saving newer progress at this
|
|
18
|
+
# very moment, and an older snapshot written over it would revert that. The
|
|
19
|
+
# parent holds only the link written at dispatch (`:async_step_ref`); every
|
|
20
|
+
# fact about this unit — its run (arguments, attempts), its park state and
|
|
21
|
+
# its outcome — lives on its own Step Result Record, where the dashboard
|
|
22
|
+
# rebuilds it from the link. A body's changes to `context` are therefore
|
|
23
|
+
# local to this job and never persisted.
|
|
24
|
+
# rubocop:disable Metrics/ClassLength
|
|
14
25
|
class StepWorker
|
|
26
|
+
# Grace added to a park's stamped window, so ordinary queue latency on the
|
|
27
|
+
# redelivery does not make `StepSweeper` mistake a parked unit for a lost
|
|
28
|
+
# one (see `#mark_record_parked`).
|
|
29
|
+
PARK_SWEEP_GRACE = 30
|
|
30
|
+
|
|
15
31
|
class << self
|
|
16
32
|
def perform(arguments)
|
|
17
33
|
arguments = arguments.transform_keys(&:to_sym)
|
|
@@ -25,16 +41,18 @@ module RubyReactor
|
|
|
25
41
|
root_context_id: arguments[:root_context_id],
|
|
26
42
|
reactor_class_name: arguments[:reactor_class_name],
|
|
27
43
|
step_context_id: arguments[:step_context_id],
|
|
28
|
-
step_name: arguments[:step_name].to_sym
|
|
44
|
+
step_name: arguments[:step_name].to_sym,
|
|
45
|
+
contention_attempts: arguments[:contention_attempts] || 0
|
|
29
46
|
}
|
|
30
47
|
end
|
|
31
48
|
end
|
|
32
49
|
|
|
33
|
-
def initialize(root_context_id:, reactor_class_name:, step_context_id:, step_name:)
|
|
50
|
+
def initialize(root_context_id:, reactor_class_name:, step_context_id:, step_name:, contention_attempts: 0)
|
|
34
51
|
@root_context_id = root_context_id
|
|
35
52
|
@reactor_class_name = reactor_class_name
|
|
36
53
|
@step_context_id = step_context_id || root_context_id
|
|
37
54
|
@step_name = step_name
|
|
55
|
+
@contention_attempts = contention_attempts.to_i
|
|
38
56
|
end
|
|
39
57
|
|
|
40
58
|
# The lock is what makes a lost unit recoverable: the record alone cannot say
|
|
@@ -61,8 +79,15 @@ module RubyReactor
|
|
|
61
79
|
context.reactor_class&.validate_definition!
|
|
62
80
|
step_config = context.reactor_class&.steps&.[](@step_name)
|
|
63
81
|
return record_missing_step unless step_config
|
|
82
|
+
return if already_completed?(context)
|
|
64
83
|
|
|
65
84
|
complete(run_step(context, step_config), context)
|
|
85
|
+
rescue Executor::StepCoordination::Contended => e
|
|
86
|
+
handle_contention(e, context)
|
|
87
|
+
rescue Executor::StepCoordination::KeyError => e
|
|
88
|
+
log(:error, "failed", error: "#{e.class}: #{e.message}")
|
|
89
|
+
complete(RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name, retryable: false),
|
|
90
|
+
context)
|
|
66
91
|
rescue StandardError => e
|
|
67
92
|
# The unit's failure belongs in its record, where a reader can see it.
|
|
68
93
|
# Raising instead would hand the job to the backend's retry machinery to
|
|
@@ -71,6 +96,126 @@ module RubyReactor
|
|
|
71
96
|
complete(RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name), nil)
|
|
72
97
|
end
|
|
73
98
|
|
|
99
|
+
# Finding 6: `async_step` has no delayed re-enqueue of its own, so a
|
|
100
|
+
# contended step parks the same way a step-level contention park does
|
|
101
|
+
# elsewhere — reschedule via `perform_step_in`, bounded by
|
|
102
|
+
# `lock_snooze_max_attempts` (the `OrderedLock::WaitError` exemption
|
|
103
|
+
# mirrors `Worker#handle_snooze`). WITHOUT calling `complete`: the Step
|
|
104
|
+
# Result Record stays "dispatched" so a reader keeps waiting instead of
|
|
105
|
+
# seeing a phantom terminal state.
|
|
106
|
+
def handle_contention(contended, context = nil)
|
|
107
|
+
# `run_step`'s loop counted this round as an attempt before the body
|
|
108
|
+
# raised; a contention park is not a retry attempt, so give it back —
|
|
109
|
+
# mirrors `StepExecutor#handle_contention`. Left inflated, the count
|
|
110
|
+
# persists across redeliveries and `StepCoordination#retry_pending?`
|
|
111
|
+
# eventually reads the step as out of retries, advancing its
|
|
112
|
+
# ordered-lock position out from under an attempt still to come.
|
|
113
|
+
context&.retry_context&.decrement_attempt_for_step(@step_name)
|
|
114
|
+
|
|
115
|
+
config = RubyReactor.configuration
|
|
116
|
+
attempt = @contention_attempts + 1
|
|
117
|
+
uncapped = contended.original.is_a?(RubyReactor::OrderedLock::WaitError)
|
|
118
|
+
|
|
119
|
+
if !uncapped && config.lock_snooze_max_attempts != :infinity && attempt > config.lock_snooze_max_attempts
|
|
120
|
+
log(:warn, "contention_exhausted", key: contended.key, attempt: attempt)
|
|
121
|
+
# Terminal: an earlier park kept this step's ordered-lock position for
|
|
122
|
+
# a redelivery that is no longer coming — advance it, or every later
|
|
123
|
+
# position stalls until its poison pill. `complete` then writes a
|
|
124
|
+
# fresh terminal record, which drops the position and waiting marker.
|
|
125
|
+
Executor::StepCoordination.discard_parked_state!(context) if context
|
|
126
|
+
complete(
|
|
127
|
+
RubyReactor::Failure(
|
|
128
|
+
"async_step :#{@step_name} gave up on #{contended.primitive} '#{contended.key}' after " \
|
|
129
|
+
"#{attempt} contention attempts",
|
|
130
|
+
step_name: @step_name, reactor_name: @reactor_class_name, retryable: false,
|
|
131
|
+
exception_class: contended.original.class.name
|
|
132
|
+
), context
|
|
133
|
+
)
|
|
134
|
+
return
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
delay = RubyReactor::Worker.snooze_delay(config, contended)
|
|
138
|
+
log(:info, "parked", key: contended.key, primitive: contended.primitive, attempt: attempt, delay: delay)
|
|
139
|
+
mark_record_parked(context, delay, attempt, contended)
|
|
140
|
+
RubyReactor.configuration.async_router.perform_step_in(
|
|
141
|
+
delay, root_context_id: @root_context_id, reactor_class_name: @reactor_class_name,
|
|
142
|
+
step_context_id: @step_context_id, step_name: @step_name, contention_attempts: attempt
|
|
143
|
+
)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# The liveness lock only drops a CONCURRENT duplicate. A parked unit whose
|
|
147
|
+
# stamped window lapsed can be swept while its scheduled redelivery is
|
|
148
|
+
# merely late, so the two deliveries can run one after the other and repeat
|
|
149
|
+
# the side effect. The record is the durable answer: once it is terminal,
|
|
150
|
+
# the body must not run again.
|
|
151
|
+
#
|
|
152
|
+
# ponytail: the record check, not a durable delivery lease — it closes the
|
|
153
|
+
# repeat-after-completion case. Two deliveries that both arrive while the
|
|
154
|
+
# unit is still only parked still both run the body (serialized by the
|
|
155
|
+
# step's own coordination). Add a lease if that shows up in practice.
|
|
156
|
+
def already_completed?(context)
|
|
157
|
+
record = storage.retrieve_step_result(@step_context_id, @step_name, step_result_namespace(context))
|
|
158
|
+
return false unless record && record["status"] == "completed"
|
|
159
|
+
|
|
160
|
+
log(:info, "duplicate_dropped")
|
|
161
|
+
true
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# A park releases the liveness lock (`perform`'s ensure) and leaves the
|
|
165
|
+
# Step Result Record at "dispatched" — which is EXACTLY the shape
|
|
166
|
+
# `StepSweeper` reads as "this unit's job was lost". Left unmarked it
|
|
167
|
+
# re-dispatches immediately, and when the parked redelivery then fires the
|
|
168
|
+
# body runs a second time: two jobs, sequential, so the liveness lock
|
|
169
|
+
# (which only drops CONCURRENT duplicates) never sees them collide.
|
|
170
|
+
# Stamping the window the redelivery is due in lets the sweeper tell
|
|
171
|
+
# parked from lost.
|
|
172
|
+
#
|
|
173
|
+
# ponytail: a fixed grace covers ordinary queue latency; a park whose
|
|
174
|
+
# redelivery is lost is recovered one grace period late rather than never.
|
|
175
|
+
# Make it configurable only if real queue lag exceeds it.
|
|
176
|
+
#
|
|
177
|
+
# The record is also where the unit's OWN park state lives (005 R-09):
|
|
178
|
+
# its ordered-lock position (`load_step_context` restores it on the
|
|
179
|
+
# redelivery) and what it waits on (the dashboard's "waiting"). Never the
|
|
180
|
+
# parent's root blob: the parent may be checkpointing newer progress right
|
|
181
|
+
# now, and this worker is not its writer (F5).
|
|
182
|
+
def mark_record_parked(context, delay, attempt, contended)
|
|
183
|
+
namespace = step_result_namespace(context)
|
|
184
|
+
record = storage.retrieve_step_result(@step_context_id, @step_name, namespace)
|
|
185
|
+
return unless record
|
|
186
|
+
|
|
187
|
+
record["parked_until"] = (Time.now + delay + PARK_SWEEP_GRACE).iso8601
|
|
188
|
+
# The counter lives in the job payload, which a sweeper re-dispatch
|
|
189
|
+
# rebuilds from the record alone — without it a swept park restarts at
|
|
190
|
+
# zero and `lock_snooze_max_attempts` never bites.
|
|
191
|
+
record["contention_attempts"] = attempt
|
|
192
|
+
position = ordered_lock_position(context)
|
|
193
|
+
record["ordered_lock"] = position if position
|
|
194
|
+
record["waiting"] = { "step" => @step_name, "primitive" => contended.primitive, "key" => contended.key,
|
|
195
|
+
"attempts" => attempt }
|
|
196
|
+
record.merge!(run_fields)
|
|
197
|
+
storage.store_step_result(@step_context_id, @step_name, record, namespace)
|
|
198
|
+
rescue StandardError => e
|
|
199
|
+
RubyReactor.configuration.logger.warn(
|
|
200
|
+
"RubyReactor: async_step :#{@step_name} could not mark its record parked: #{e.message}"
|
|
201
|
+
)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def ordered_lock_position(context)
|
|
205
|
+
stash = context&.private_data&.[](:step_ordered_locks) || context&.private_data&.[]("step_ordered_locks")
|
|
206
|
+
return nil unless stash
|
|
207
|
+
|
|
208
|
+
stash[@step_name.to_s] || stash[@step_name.to_sym]
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Records are namespaced by the reactor that OWNS the step (what
|
|
212
|
+
# `AsyncStepDispatch#async_step_class_name` wrote them under), which for a
|
|
213
|
+
# composed child is not the root this job was handed.
|
|
214
|
+
def step_result_namespace(context)
|
|
215
|
+
owner = context&.reactor_class
|
|
216
|
+
owner ? RubyReactor.reactor_storage_name(owner) : @reactor_class_name
|
|
217
|
+
end
|
|
218
|
+
|
|
74
219
|
def acquire_liveness_lock
|
|
75
220
|
# Inline testing re-enters this frame synchronously, so the lock would
|
|
76
221
|
# self-contend; it only guards cross-process delivery, impossible inline.
|
|
@@ -93,37 +238,89 @@ module RubyReactor
|
|
|
93
238
|
end
|
|
94
239
|
|
|
95
240
|
def run_step(context, step_config)
|
|
241
|
+
# A suppressed step never coordinates (FR-012) — decided before the
|
|
242
|
+
# arguments are validated or any hold is taken, exactly as
|
|
243
|
+
# `StepExecutor#execute_step_sync` orders it. `complete` persists the
|
|
244
|
+
# nil result, so the reader sees the same skipped unit a same-process
|
|
245
|
+
# step would produce.
|
|
246
|
+
unless step_config.should_run?(context)
|
|
247
|
+
log(:info, "skipped")
|
|
248
|
+
return RubyReactor.Success(nil)
|
|
249
|
+
end
|
|
250
|
+
|
|
96
251
|
arguments = resolve_arguments(step_config, context)
|
|
252
|
+
# Reactor-side `argument`/`validate_args` rules gate the step BEFORE its
|
|
253
|
+
# coordination is acquired, exactly as `StepExecutor#execute_step_sync`
|
|
254
|
+
# orders them — an async_step must not take a lock (or spend a rate-limit
|
|
255
|
+
# slot) for arguments it is about to reject.
|
|
256
|
+
invalid = validate_arguments(step_config, arguments)
|
|
257
|
+
return invalid if invalid
|
|
258
|
+
|
|
97
259
|
log(:info, "running")
|
|
260
|
+
# What `StepExecutor#run_step_implementation` records as a `:run` trace
|
|
261
|
+
# entry for a same-process step, kept on this unit's record instead (see
|
|
262
|
+
# the class comment): the dashboard resolves the step's coordination key
|
|
263
|
+
# and its inspector's arguments from it.
|
|
264
|
+
contract = step_config.input_contract
|
|
265
|
+
@run = { "started_at" => Time.now.iso8601(6),
|
|
266
|
+
"arguments" => ContextSerializer.serialize_value(contract ? contract.redact(arguments) : arguments) }
|
|
98
267
|
|
|
99
268
|
attempt = 0
|
|
100
269
|
result = nil
|
|
101
270
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
271
|
+
# Under `with_step`, exactly as `StepExecutor#execute_step_sync` runs a
|
|
272
|
+
# same-process step: the coordination hooks this worker fires must be
|
|
273
|
+
# attributable to the step, which reads `context.current_step`.
|
|
274
|
+
context.with_step(@step_name) do
|
|
275
|
+
loop do
|
|
276
|
+
attempt += 1
|
|
277
|
+
@run["attempts"] = attempt
|
|
278
|
+
# Mirror the count onto the context: `StepCoordination` reads
|
|
279
|
+
# `retry_context` to decide whether an ordered-lock position should be
|
|
280
|
+
# held for a pending retry, and this worker is the one path that never
|
|
281
|
+
# goes through `RetryManager#prepare_retry_attempt`.
|
|
282
|
+
context.retry_context.increment_attempt_for_step(@step_name)
|
|
283
|
+
result = execute_step_body(step_config, arguments, context)
|
|
284
|
+
break unless retry?(step_config, result, attempt)
|
|
285
|
+
|
|
286
|
+
delay = backoff_delay(step_config, attempt)
|
|
287
|
+
log(:warn, "retrying", attempt: attempt, delay: delay)
|
|
288
|
+
sleep(delay)
|
|
289
|
+
end
|
|
110
290
|
end
|
|
111
291
|
|
|
112
292
|
result
|
|
113
293
|
end
|
|
114
294
|
|
|
295
|
+
# Same check and same structured, non-retryable shape `StepExecutor`
|
|
296
|
+
# produces — the same arguments fail the same rules on every attempt.
|
|
297
|
+
def validate_arguments(step_config, arguments)
|
|
298
|
+
return nil unless step_config.args_validator
|
|
299
|
+
|
|
300
|
+
validation_result = step_config.args_validator.call(arguments)
|
|
301
|
+
return nil if validation_result.success?
|
|
302
|
+
|
|
303
|
+
error = validation_result.error
|
|
304
|
+
error.step_name = @step_name
|
|
305
|
+
error.step_arguments = arguments
|
|
306
|
+
log(:warn, "invalid_arguments", error: "#{error.class}: #{error.message}")
|
|
307
|
+
RubyReactor.Failure(error, validation_errors: error.field_errors, step_name: @step_name,
|
|
308
|
+
step_arguments: arguments, reactor_name: @reactor_class_name,
|
|
309
|
+
retryable: false)
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# The same single enforcement site `StepExecutor` uses
|
|
313
|
+
# (`StepCoordination.run_step`), so the worker path cannot drift from the
|
|
314
|
+
# in-process one. `Contended`/`KeyError` — this step's own — propagate
|
|
315
|
+
# unrescued: `perform_unit` handles them (park, or a non-retryable Failure).
|
|
115
316
|
def execute_step_body(step_config, arguments, context)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
args = step_config.inline_contract.enforce!(args) if step_config.inline_contract
|
|
120
|
-
step_config.run_block.call(args, context)
|
|
121
|
-
elsif step_config.has_impl?
|
|
122
|
-
step_config.impl.run(arguments, context)
|
|
123
|
-
else
|
|
124
|
-
RubyReactor.Failure("Step '#{@step_name}' has no implementation")
|
|
125
|
-
end
|
|
317
|
+
unless step_config.has_run_block? || step_config.has_impl?
|
|
318
|
+
return RubyReactor.Failure("Step '#{@step_name}' has no implementation")
|
|
319
|
+
end
|
|
126
320
|
|
|
321
|
+
result = Executor::StepCoordination.run_step(step_config, arguments, context: context,
|
|
322
|
+
reactor_class: context.reactor_class,
|
|
323
|
+
middlewares: context.middlewares)
|
|
127
324
|
normalize(result)
|
|
128
325
|
rescue Error::InputValidationError => e
|
|
129
326
|
# Same shape the executor builds, and never retried: the same arguments
|
|
@@ -131,6 +328,8 @@ module RubyReactor
|
|
|
131
328
|
RubyReactor.Failure(e, validation_errors: e.field_errors, step_name: @step_name,
|
|
132
329
|
step_arguments: e.step_arguments || {}, reactor_name: @reactor_class_name,
|
|
133
330
|
retryable: false)
|
|
331
|
+
rescue Executor::StepCoordination::Contended, Executor::StepCoordination::KeyError
|
|
332
|
+
raise
|
|
134
333
|
rescue StandardError => e
|
|
135
334
|
RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name)
|
|
136
335
|
end
|
|
@@ -182,14 +381,21 @@ module RubyReactor
|
|
|
182
381
|
record["signal"] = "halt"
|
|
183
382
|
record["reason"] = result.reason
|
|
184
383
|
end
|
|
185
|
-
|
|
384
|
+
record.merge!(run_fields)
|
|
385
|
+
# Namespaced by the reactor that OWNS the step — what
|
|
386
|
+
# `AsyncStepDispatch` wrote the `dispatched` record under and what the
|
|
387
|
+
# reader's `Template::Result` looks under. For an async_step inside a
|
|
388
|
+
# composed child that is the child, not the root name this job carries.
|
|
389
|
+
storage.store_step_result(@step_context_id, @step_name, record, step_result_namespace(context || @step_context))
|
|
186
390
|
log(result.success? ? :info : :warn, result.success? ? "completed" : "completed_with_failure")
|
|
187
391
|
storage.publish(RubyReactor.async_step_channel(@step_context_id, @step_name), "done")
|
|
188
392
|
result
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
# This delivery's run, once the body was reached: `started_at`,
|
|
396
|
+
# `arguments` (redacted, serialized) and `attempts`. Empty before that.
|
|
397
|
+
def run_fields
|
|
398
|
+
@run || {}
|
|
193
399
|
end
|
|
194
400
|
|
|
195
401
|
def record_missing_parent
|
|
@@ -219,16 +425,43 @@ module RubyReactor
|
|
|
219
425
|
return nil unless data
|
|
220
426
|
|
|
221
427
|
root = ContextSerializer.deserialize_hash(data)
|
|
222
|
-
@root_context = root
|
|
223
428
|
found = find_context(root, @step_context_id)
|
|
429
|
+
# Kept so the paths that deliberately pass no context to `complete`
|
|
430
|
+
# still write the record under the owning reactor's namespace.
|
|
431
|
+
@step_context = found
|
|
432
|
+
restore_parked_position(found) if found
|
|
224
433
|
# The step runs in its own job; nothing it reaches should hand off again.
|
|
225
434
|
found&.inline_async_execution = true
|
|
435
|
+
# Per-job owner, NEVER the root context id (US4-5, research D5):
|
|
436
|
+
# ownership never crosses a process hand-off, so this job's coordination
|
|
437
|
+
# never re-enters the dispatching execution's holds and never blocks a
|
|
438
|
+
# SECOND async_step dispatch on the same key from proceeding once this
|
|
439
|
+
# one releases. Derived, not random: `StepCoordination` can detach a
|
|
440
|
+
# lock across a contention park, and `perform_step_in` redelivers this
|
|
441
|
+
# same unit — a fresh uuid per redelivery would make `lock.reattach`
|
|
442
|
+
# fail against its own detached hold until the TTL expired.
|
|
443
|
+
found&.coordination_owner = "async_step:#{@step_context_id}:#{@step_name}"
|
|
444
|
+
# `ContextSerializer` does not carry `middlewares`, and nothing else in
|
|
445
|
+
# this worker builds them — without this the step's coordination hooks
|
|
446
|
+
# fire into an empty runner, contradicting "identical hooks in a worker".
|
|
447
|
+
found&.middlewares ||= Executor.middlewares_for(found.reactor_class)
|
|
226
448
|
found
|
|
227
449
|
rescue RubyReactor::Error::DeserializationError, RubyReactor::Error::SchemaVersionError => e
|
|
228
450
|
log(:error, "parent_context_unreadable", error: "#{e.class}: #{e.message}")
|
|
229
451
|
nil
|
|
230
452
|
end
|
|
231
453
|
|
|
454
|
+
# A parked unit's ordered-lock position lives on its own record, not in
|
|
455
|
+
# the parent's blob (see `mark_record_parked`). Put it back where
|
|
456
|
+
# `StepCoordination` looks, so the redelivery re-reads the SAME nonce.
|
|
457
|
+
def restore_parked_position(context)
|
|
458
|
+
record = storage.retrieve_step_result(@step_context_id, @step_name, step_result_namespace(context))
|
|
459
|
+
position = record && record["ordered_lock"]
|
|
460
|
+
return unless position
|
|
461
|
+
|
|
462
|
+
(context.private_data[:step_ordered_locks] ||= {})[@step_name.to_s] = position.transform_keys(&:to_sym)
|
|
463
|
+
end
|
|
464
|
+
|
|
232
465
|
def find_context(context, target_id)
|
|
233
466
|
return context if context.context_id == target_id
|
|
234
467
|
|
|
@@ -241,17 +474,6 @@ module RubyReactor
|
|
|
241
474
|
nil
|
|
242
475
|
end
|
|
243
476
|
|
|
244
|
-
def save_root(_context)
|
|
245
|
-
return unless @root_context
|
|
246
|
-
|
|
247
|
-
storage.store_context(@root_context.context_id, ContextSerializer.serialize(@root_context),
|
|
248
|
-
@reactor_class_name)
|
|
249
|
-
rescue StandardError => e
|
|
250
|
-
RubyReactor.configuration.logger.warn(
|
|
251
|
-
"RubyReactor: async_step :#{@step_name} could not persist its parent context: #{e.message}"
|
|
252
|
-
)
|
|
253
|
-
end
|
|
254
|
-
|
|
255
477
|
def storage
|
|
256
478
|
RubyReactor.configuration.storage_adapter
|
|
257
479
|
end
|
|
@@ -272,4 +494,5 @@ module RubyReactor
|
|
|
272
494
|
)
|
|
273
495
|
end
|
|
274
496
|
end
|
|
497
|
+
# rubocop:enable Metrics/ClassLength
|
|
275
498
|
end
|
data/lib/ruby_reactor/version.rb
CHANGED