ruby_reactor 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/CHANGELOG.md +14 -0
- data/README.md +242 -75
- data/lib/ruby_reactor/configuration.rb +66 -2
- data/lib/ruby_reactor/context_serializer.rb +9 -4
- data/lib/ruby_reactor/dsl/template_helpers.rb +7 -1
- data/lib/ruby_reactor/executor/ordered_lock_support.rb +1 -1
- data/lib/ruby_reactor/executor/retry_manager.rb +7 -2
- data/lib/ruby_reactor/executor/step_executor.rb +25 -5
- data/lib/ruby_reactor/executor.rb +85 -3
- data/lib/ruby_reactor/lock.rb +13 -0
- data/lib/ruby_reactor/map/collector.rb +41 -0
- data/lib/ruby_reactor/map/dispatcher.rb +42 -0
- data/lib/ruby_reactor/map/element_executor.rb +39 -0
- data/lib/ruby_reactor/map/helpers.rb +10 -3
- data/lib/ruby_reactor/map/sweeper.rb +110 -0
- data/lib/ruby_reactor/reactor.rb +7 -5
- data/lib/ruby_reactor/sidekiq_adapter.rb +9 -8
- data/lib/ruby_reactor/sidekiq_workers/sweeper_worker.rb +73 -0
- data/lib/ruby_reactor/sidekiq_workers/worker.rb +42 -34
- data/lib/ruby_reactor/step/map_step.rb +18 -2
- data/lib/ruby_reactor/step.rb +4 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +83 -60
- data/lib/ruby_reactor/storage/redis_locking.rb +8 -0
- data/lib/ruby_reactor/sweeper.rb +58 -0
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor.rb +42 -0
- metadata +4 -1
|
@@ -11,6 +11,7 @@ module RubyReactor
|
|
|
11
11
|
@result_handler = managers[:result_handler]
|
|
12
12
|
@compensation_manager = managers[:compensation_manager]
|
|
13
13
|
@middlewares = managers[:middlewares] || context.middlewares || Executor.middlewares_for(reactor_class)
|
|
14
|
+
@on_step_complete = managers[:on_step_complete]
|
|
14
15
|
end
|
|
15
16
|
|
|
16
17
|
def execute_all_steps
|
|
@@ -45,8 +46,14 @@ module RubyReactor
|
|
|
45
46
|
# If a step returns InterruptResult, we need to stop execution and return it
|
|
46
47
|
return result if result.is_a?(RubyReactor::InterruptResult)
|
|
47
48
|
|
|
48
|
-
#
|
|
49
|
-
|
|
49
|
+
# Only a continue-Success reaches here (Async/Retry/Skipped/Failure/
|
|
50
|
+
# Interrupt all returned above; nil is inline-async test mode). It is
|
|
51
|
+
# the one outcome where the loop proceeds to more steps with no other
|
|
52
|
+
# save in between — every terminal/handoff result persists via its own
|
|
53
|
+
# path. Write a durable checkpoint so a crash re-runs at most this one
|
|
54
|
+
# step. Ordering: side-effect -> record result (inside execute_step) ->
|
|
55
|
+
# checkpoint here.
|
|
56
|
+
@on_step_complete&.call if result.is_a?(RubyReactor::Success)
|
|
50
57
|
end
|
|
51
58
|
end
|
|
52
59
|
|
|
@@ -198,20 +205,33 @@ module RubyReactor
|
|
|
198
205
|
|
|
199
206
|
# Use root context if available to ensure we serialize the full tree
|
|
200
207
|
context_to_serialize = @context.root_context || @context
|
|
201
|
-
reactor_class_name = context_to_serialize.reactor_class
|
|
208
|
+
reactor_class_name = RubyReactor.reactor_storage_name(context_to_serialize.reactor_class)
|
|
202
209
|
|
|
203
210
|
# Inject OTel context before serialization
|
|
204
211
|
@middlewares.on(:before_async_enqueue, context_to_serialize)
|
|
205
212
|
|
|
206
|
-
|
|
213
|
+
# Storage is load-bearing: the job payload is identity-only, so the root
|
|
214
|
+
# context MUST be persisted BEFORE the job is enqueued (F2). The reactor
|
|
215
|
+
# class name used for the storage key must match the one handed to the
|
|
216
|
+
# worker, so compute it once and reuse it for both.
|
|
217
|
+
checkpoint_root!(context_to_serialize, reactor_class_name)
|
|
207
218
|
|
|
208
219
|
configuration.async_router.perform_async(
|
|
209
|
-
|
|
220
|
+
context_to_serialize.context_id,
|
|
210
221
|
reactor_class_name,
|
|
211
222
|
intermediate_results: @context.intermediate_results
|
|
212
223
|
)
|
|
213
224
|
end
|
|
214
225
|
|
|
226
|
+
# Persist the root context under its storage key. Mirrors Executor#checkpoint!
|
|
227
|
+
# but lives here because handle_async_step runs inside the StepExecutor and
|
|
228
|
+
# must serialize AFTER the before_async_enqueue middleware has injected its
|
|
229
|
+
# OTel context.
|
|
230
|
+
def checkpoint_root!(root, reactor_class_name)
|
|
231
|
+
storage = RubyReactor::Configuration.instance.storage_adapter
|
|
232
|
+
storage.store_context(root.context_id, ContextSerializer.serialize(root), reactor_class_name)
|
|
233
|
+
end
|
|
234
|
+
|
|
215
235
|
def handle_interrupt_step(step_config)
|
|
216
236
|
# Check if we have a result for this step (resuming)
|
|
217
237
|
if @context.intermediate_results.key?(step_config.name)
|
|
@@ -38,14 +38,24 @@ module RubyReactor
|
|
|
38
38
|
retry_manager: @retry_manager,
|
|
39
39
|
result_handler: @result_handler,
|
|
40
40
|
compensation_manager: @compensation_manager,
|
|
41
|
-
middlewares: @middlewares
|
|
41
|
+
middlewares: @middlewares,
|
|
42
|
+
# Save-per-step durable checkpoint. checkpoint! resolves the ROOT
|
|
43
|
+
# context, so this same callback — wired into every executor including
|
|
44
|
+
# the nested ones ComposeStep builds — always advances the root blob
|
|
45
|
+
# (F8): a mid-child crash re-runs one sub-step, not the whole child.
|
|
46
|
+
# `throttle: true` lets checkpoint_min_interval coalesce these mid-run
|
|
47
|
+
# writes (default 0 = write every step); the terminal save still runs.
|
|
48
|
+
on_step_complete: -> { checkpoint!(throttle: true) }
|
|
42
49
|
}
|
|
43
50
|
)
|
|
44
51
|
@result = nil
|
|
45
52
|
@acquired_lock = nil
|
|
46
53
|
@acquired_semaphore = nil
|
|
54
|
+
@acquired_context_lock = nil
|
|
55
|
+
@context_lock_owner = nil
|
|
47
56
|
@contention_snooze = false
|
|
48
57
|
@skip_context_persist = false
|
|
58
|
+
@last_checkpoint_at = nil
|
|
49
59
|
end
|
|
50
60
|
|
|
51
61
|
def self.resolve_middlewares(reactor_class)
|
|
@@ -150,7 +160,7 @@ module RubyReactor
|
|
|
150
160
|
end
|
|
151
161
|
end
|
|
152
162
|
|
|
153
|
-
def resume_execution # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity
|
|
163
|
+
def resume_execution # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity
|
|
154
164
|
middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
|
|
155
165
|
completed = false
|
|
156
166
|
|
|
@@ -175,6 +185,13 @@ module RubyReactor
|
|
|
175
185
|
@context.status = :running
|
|
176
186
|
check_rate_limit if first_run
|
|
177
187
|
|
|
188
|
+
# Per-context liveness lock: serializes duplicate deliveries of the same
|
|
189
|
+
# root context (e.g. a sweeper re-enqueue racing a still-live worker) and
|
|
190
|
+
# doubles as the sweeper's "worker alive" signal. Only the ROOT executor
|
|
191
|
+
# holds it — composed/nested children resume inline under the root worker
|
|
192
|
+
# and must not contend on the root's own key.
|
|
193
|
+
acquire_context_lock
|
|
194
|
+
|
|
178
195
|
# Resumes intentionally skip check_rate_limit (a paused run must not
|
|
179
196
|
# block itself on resume), so acquire lock/semaphore directly rather
|
|
180
197
|
# than via acquire_locks.
|
|
@@ -217,6 +234,8 @@ module RubyReactor
|
|
|
217
234
|
@result
|
|
218
235
|
ensure
|
|
219
236
|
release_locks
|
|
237
|
+
@acquired_context_lock&.release
|
|
238
|
+
@acquired_context_lock = nil
|
|
220
239
|
leave_ordered_lock_scope
|
|
221
240
|
save_context unless skip_context_persist?
|
|
222
241
|
|
|
@@ -241,13 +260,40 @@ module RubyReactor
|
|
|
241
260
|
|
|
242
261
|
def save_context
|
|
243
262
|
storage = RubyReactor::Configuration.instance.storage_adapter
|
|
244
|
-
reactor_class_name =
|
|
263
|
+
reactor_class_name = RubyReactor.reactor_storage_name(@reactor_class)
|
|
245
264
|
|
|
246
265
|
# Serialize context
|
|
247
266
|
serialized_context = ContextSerializer.serialize(@context)
|
|
248
267
|
storage.store_context(@context.context_id, serialized_context, reactor_class_name)
|
|
249
268
|
end
|
|
250
269
|
|
|
270
|
+
# Durable per-step checkpoint. Unlike save_context (which serializes THIS
|
|
271
|
+
# executor's @context — the observability path, F1), checkpoint! always
|
|
272
|
+
# serializes and stores the ROOT context under the root's key — the unit the
|
|
273
|
+
# async worker rehydrates by id. For a top-level reactor root == @context; for
|
|
274
|
+
# a composed/nested child it stores the root with the child's live state
|
|
275
|
+
# embedded via composed_contexts. TTL is re-stamped on every write (Phase 4).
|
|
276
|
+
def checkpoint!(throttle: false)
|
|
277
|
+
return if throttle && !checkpoint_due?
|
|
278
|
+
|
|
279
|
+
root = @context.root_context || @context
|
|
280
|
+
storage = RubyReactor::Configuration.instance.storage_adapter
|
|
281
|
+
reactor_class_name = RubyReactor.reactor_storage_name(root.reactor_class)
|
|
282
|
+
storage.store_context(root.context_id, ContextSerializer.serialize(root), reactor_class_name)
|
|
283
|
+
@last_checkpoint_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# Whether a throttled (per-step) checkpoint is due. With checkpoint_min_interval
|
|
287
|
+
# <= 0 (default) every step checkpoints; otherwise mid-run checkpoints are
|
|
288
|
+
# coalesced to at most one per interval. The first step of a run always writes
|
|
289
|
+
# (@last_checkpoint_at is nil), and the run's terminal save is never throttled.
|
|
290
|
+
def checkpoint_due?
|
|
291
|
+
interval = RubyReactor.configuration.checkpoint_min_interval.to_f
|
|
292
|
+
return true if interval <= 0 || @last_checkpoint_at.nil?
|
|
293
|
+
|
|
294
|
+
(Process.clock_gettime(Process::CLOCK_MONOTONIC) - @last_checkpoint_at) >= interval
|
|
295
|
+
end
|
|
296
|
+
|
|
251
297
|
def persist_context?
|
|
252
298
|
@context.status.to_s != "pending" ||
|
|
253
299
|
@context.execution_trace.any? ||
|
|
@@ -340,6 +386,42 @@ module RubyReactor
|
|
|
340
386
|
RubyReactor::Period.key(base, config[:every])
|
|
341
387
|
end
|
|
342
388
|
|
|
389
|
+
# Per-execution liveness lock on the root context id. Owner is a fresh UUID
|
|
390
|
+
# per execution (NOT the context_id): a duplicate delivery of the *same*
|
|
391
|
+
# context from a different worker must be blocked, so reentrancy by id would
|
|
392
|
+
# defeat the guard. Only the root executor acquires — a composed/nested child
|
|
393
|
+
# resumes inline under the root worker and shares the root's lock, so it must
|
|
394
|
+
# not try to re-acquire the same key with a different owner (self-deadlock).
|
|
395
|
+
def acquire_context_lock
|
|
396
|
+
root = @context.root_context || @context
|
|
397
|
+
return unless root.equal?(@context) # only the root executor holds it
|
|
398
|
+
# In Sidekiq::Testing.inline! the retry/snooze `perform_in` re-enters the
|
|
399
|
+
# worker synchronously, nested inside this still-running frame that holds
|
|
400
|
+
# the lock — it would self-contend forever. The lock guards concurrent
|
|
401
|
+
# cross-process delivery, which cannot happen under inline testing, so skip.
|
|
402
|
+
return if inline_testing_mode?
|
|
403
|
+
|
|
404
|
+
lock = RubyReactor::Lock.new(
|
|
405
|
+
"async:#{root.context_id}",
|
|
406
|
+
owner: @context_lock_owner ||= SecureRandom.uuid,
|
|
407
|
+
ttl: RubyReactor.configuration.context_lock_ttl,
|
|
408
|
+
wait: 0, # fail fast -> snooze; never block the worker thread
|
|
409
|
+
auto_extend: true # keep the liveness signal fresh while we run
|
|
410
|
+
)
|
|
411
|
+
lock.acquire
|
|
412
|
+
@acquired_context_lock = lock
|
|
413
|
+
rescue RubyReactor::Lock::AcquisitionError => e
|
|
414
|
+
# We lost the race to a live original holding this context's lock. We did
|
|
415
|
+
# no work, so we must NOT persist on the way out — saving our (older)
|
|
416
|
+
# rehydrated snapshot would clobber the original's newer checkpoint.
|
|
417
|
+
@skip_context_persist = true
|
|
418
|
+
raise RubyReactor::Lock::ContextLockContention.new(e.message, context_lock_key: "async:#{root.context_id}")
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
def inline_testing_mode?
|
|
422
|
+
defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
|
|
423
|
+
end
|
|
424
|
+
|
|
343
425
|
def acquire_exclusive_lock
|
|
344
426
|
config = @reactor_class.lock_config
|
|
345
427
|
key = config[:key_proc].call(@context.inputs)
|
data/lib/ruby_reactor/lock.rb
CHANGED
|
@@ -4,6 +4,19 @@ module RubyReactor
|
|
|
4
4
|
class Lock
|
|
5
5
|
class AcquisitionError < StandardError; end
|
|
6
6
|
|
|
7
|
+
# Raised specifically for the per-context liveness lock (`async:<id>`).
|
|
8
|
+
# Carries the bare key so the worker can exempt it from the snooze cap:
|
|
9
|
+
# a duplicate of the *same* execution may legitimately wait arbitrarily
|
|
10
|
+
# long for the live original to finish.
|
|
11
|
+
class ContextLockContention < AcquisitionError
|
|
12
|
+
attr_reader :context_lock_key
|
|
13
|
+
|
|
14
|
+
def initialize(message, context_lock_key:)
|
|
15
|
+
super(message)
|
|
16
|
+
@context_lock_key = context_lock_key
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
7
20
|
# Minimum interval between auto-extend pings; protects very small TTLs.
|
|
8
21
|
MIN_EXTEND_INTERVAL = 1.0
|
|
9
22
|
|
|
@@ -8,6 +8,42 @@ module RubyReactor
|
|
|
8
8
|
def self.perform(arguments)
|
|
9
9
|
arguments = arguments.transform_keys(&:to_sym)
|
|
10
10
|
map_id = arguments[:map_id]
|
|
11
|
+
|
|
12
|
+
# Serialize concurrent collector deliveries for the SAME map (eager queue +
|
|
13
|
+
# counter-zero trigger + sweeper re-trigger could otherwise all resume the
|
|
14
|
+
# parent at once and both write its context). A dedicated map_collect lock
|
|
15
|
+
# is used rather than the parent's own lock so it never conflicts with the
|
|
16
|
+
# context lock the parent's resume_execution acquires for itself.
|
|
17
|
+
lock = acquire_collect_lock(map_id)
|
|
18
|
+
return if lock == :contended
|
|
19
|
+
|
|
20
|
+
begin
|
|
21
|
+
perform_collection(arguments)
|
|
22
|
+
ensure
|
|
23
|
+
lock.release if lock.respond_to?(:release)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def self.acquire_collect_lock(map_id)
|
|
28
|
+
return :inline if inline_testing_mode?
|
|
29
|
+
|
|
30
|
+
lock = RubyReactor::Lock.new(
|
|
31
|
+
"map_collect:#{map_id}",
|
|
32
|
+
owner: SecureRandom.uuid, ttl: RubyReactor.configuration.context_lock_ttl,
|
|
33
|
+
wait: 0, auto_extend: true
|
|
34
|
+
)
|
|
35
|
+
lock.acquire
|
|
36
|
+
lock
|
|
37
|
+
rescue RubyReactor::Lock::AcquisitionError
|
|
38
|
+
:contended
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def self.inline_testing_mode?
|
|
42
|
+
defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.perform_collection(arguments)
|
|
46
|
+
map_id = arguments[:map_id]
|
|
11
47
|
parent_context_id = arguments[:parent_context_id]
|
|
12
48
|
parent_reactor_class_name = arguments[:parent_reactor_class_name]
|
|
13
49
|
step_name = arguments[:step_name]
|
|
@@ -18,6 +54,11 @@ module RubyReactor
|
|
|
18
54
|
parent_context_data = storage.retrieve_context(parent_context_id, parent_reactor_class_name)
|
|
19
55
|
parent_context = RubyReactor::Context.deserialize_from_retry(parent_context_data)
|
|
20
56
|
|
|
57
|
+
# Idempotency: if the parent already recorded this map step's result, a
|
|
58
|
+
# prior collector already resumed it. Re-resuming would double-execute the
|
|
59
|
+
# steps after the map. Skip.
|
|
60
|
+
return if parent_context.intermediate_results.key?(step_name.to_sym)
|
|
61
|
+
|
|
21
62
|
# Check if all tasks are completed
|
|
22
63
|
metadata = storage.retrieve_map_metadata(map_id, parent_reactor_class_name)
|
|
23
64
|
total_count = metadata ? metadata["count"].to_i : 0
|
|
@@ -104,6 +104,48 @@ module RubyReactor
|
|
|
104
104
|
end
|
|
105
105
|
end
|
|
106
106
|
|
|
107
|
+
# Re-dispatch a SPECIFIC index whose result slot is missing (Phase 5c, used
|
|
108
|
+
# by the map sweeper). Index-driven rather than offset-driven: resolve the
|
|
109
|
+
# source from the stored parent context and pick source[index]. Idempotent
|
|
110
|
+
# because store_map_result HSETs by index — a re-run overwrites slot `index`,
|
|
111
|
+
# never duplicates.
|
|
112
|
+
def self.requeue_index(map_meta, index)
|
|
113
|
+
storage = RubyReactor.configuration.storage_adapter
|
|
114
|
+
parent_class_name = map_meta["parent_reactor_class_name"]
|
|
115
|
+
parent_context = load_parent_context_from_storage(map_meta["parent_context_id"], parent_class_name, storage)
|
|
116
|
+
|
|
117
|
+
arguments = {
|
|
118
|
+
map_id: map_meta["map_id"],
|
|
119
|
+
step_name: map_meta["step_name"],
|
|
120
|
+
strict_ordering: map_meta["strict_ordering"],
|
|
121
|
+
parent_context_id: map_meta["parent_context_id"],
|
|
122
|
+
parent_reactor_class_name: parent_class_name,
|
|
123
|
+
fail_fast: map_meta["fail_fast"],
|
|
124
|
+
batch_size: map_meta["batch_size"]
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
source = resolve_source(arguments, parent_context)
|
|
128
|
+
element = element_at(source, index)
|
|
129
|
+
|
|
130
|
+
queue_element_job(element, index, {
|
|
131
|
+
map_id: map_meta["map_id"],
|
|
132
|
+
arguments: arguments,
|
|
133
|
+
context: parent_context,
|
|
134
|
+
reactor_class_info: map_meta["reactor_class_info"],
|
|
135
|
+
step_name: map_meta["step_name"]
|
|
136
|
+
})
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def self.element_at(source, index)
|
|
140
|
+
if source.is_a?(Array)
|
|
141
|
+
source[index]
|
|
142
|
+
elsif source.respond_to?(:offset) && source.respond_to?(:limit)
|
|
143
|
+
source.offset(index).limit(1).to_a.first
|
|
144
|
+
else
|
|
145
|
+
source.drop(index).first
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
107
149
|
def self.queue_element_job(element, index, options)
|
|
108
150
|
arguments = options[:arguments]
|
|
109
151
|
context = options[:context]
|
|
@@ -8,6 +8,45 @@ module RubyReactor
|
|
|
8
8
|
def self.perform(arguments)
|
|
9
9
|
arguments = arguments.transform_keys(&:to_sym)
|
|
10
10
|
|
|
11
|
+
# Per-element liveness lock (Phase 5b): its presence is the map sweeper's
|
|
12
|
+
# "element alive" signal, and it serializes duplicate deliveries so a
|
|
13
|
+
# re-run can't double-decrement the counter (M3). A duplicate of a live
|
|
14
|
+
# element is dropped — the live original stores the result and finalizes.
|
|
15
|
+
lock = acquire_element_lock(arguments)
|
|
16
|
+
return if lock == :contended
|
|
17
|
+
|
|
18
|
+
begin
|
|
19
|
+
perform_element(arguments)
|
|
20
|
+
ensure
|
|
21
|
+
lock.release if lock.respond_to?(:release)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.acquire_element_lock(arguments)
|
|
26
|
+
# In Sidekiq::Testing.inline! an element's async-retry perform_map_element_in
|
|
27
|
+
# re-enters synchronously inside this frame; the lock would self-contend.
|
|
28
|
+
# It only guards concurrent cross-process delivery, impossible inline.
|
|
29
|
+
return :inline if inline_testing_mode?
|
|
30
|
+
|
|
31
|
+
lock = RubyReactor::Lock.new(
|
|
32
|
+
"map_element:#{arguments[:map_id]}:#{arguments[:index]}",
|
|
33
|
+
owner: SecureRandom.uuid, ttl: RubyReactor.configuration.context_lock_ttl,
|
|
34
|
+
wait: 0, auto_extend: true
|
|
35
|
+
)
|
|
36
|
+
lock.acquire
|
|
37
|
+
lock
|
|
38
|
+
rescue RubyReactor::Lock::AcquisitionError
|
|
39
|
+
RubyReactor.configuration.logger.info(
|
|
40
|
+
"RubyReactor map element #{arguments[:map_id]}:#{arguments[:index]} already in flight; dropping duplicate"
|
|
41
|
+
)
|
|
42
|
+
:contended
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.inline_testing_mode?
|
|
46
|
+
defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def self.perform_element(arguments)
|
|
11
50
|
context = hydrate_or_create_context(arguments)
|
|
12
51
|
# The element already runs inside its own background worker, so any async
|
|
13
52
|
# steps (and async retries) must execute inline here rather than handing
|
|
@@ -108,10 +108,17 @@ module RubyReactor
|
|
|
108
108
|
executor.resume_execution
|
|
109
109
|
end
|
|
110
110
|
|
|
111
|
+
# Checkpoint the ROOT, not the sub (F9/C2). When the map is embedded in a
|
|
112
|
+
# composed sub-reactor, parent_context is the *sub*; storing only the sub
|
|
113
|
+
# would leave the root blob stale and a rehydrate-by-root-id resume would
|
|
114
|
+
# lose the map's completion. Resolve the root (which embeds the sub's
|
|
115
|
+
# post-map state via composed_contexts) and store that. For a top-level
|
|
116
|
+
# map parent_context IS the root, so this is unchanged.
|
|
117
|
+
root = parent_context.root_context || parent_context
|
|
111
118
|
storage.store_context(
|
|
112
|
-
|
|
113
|
-
ContextSerializer.serialize(
|
|
114
|
-
|
|
119
|
+
root.context_id,
|
|
120
|
+
ContextSerializer.serialize(root),
|
|
121
|
+
RubyReactor.reactor_storage_name(root.reactor_class)
|
|
115
122
|
)
|
|
116
123
|
end
|
|
117
124
|
end
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
module Map
|
|
5
|
+
# Recovers map fan-out from a hard kill (Phase 5d). Maps are the path most
|
|
6
|
+
# exposed to a lost job: one missing element result hangs the whole map and
|
|
7
|
+
# its parent forever. The unifying signal is the results hash — index-keyed
|
|
8
|
+
# and idempotent (HSET) — so completion is authoritative on `missing`, not on
|
|
9
|
+
# the fragile counter:
|
|
10
|
+
#
|
|
11
|
+
# missing = (0...count) - HKEYS(results)
|
|
12
|
+
#
|
|
13
|
+
# For each active map:
|
|
14
|
+
# * missing indices with NO live element lock are re-dispatched (M1/M4/M5).
|
|
15
|
+
# * if nothing is missing but the parent never resumed, the collector is
|
|
16
|
+
# re-triggered (M2) — gated so it never fires while a collector or the
|
|
17
|
+
# parent is alive, or after the parent already collected.
|
|
18
|
+
#
|
|
19
|
+
# `run_once` is pure and idempotent; the host wires the cadence (same contract
|
|
20
|
+
# as RubyReactor::Sweeper).
|
|
21
|
+
class Sweeper
|
|
22
|
+
def self.run_once(limit: 1000)
|
|
23
|
+
new.run_once(limit: limit)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def initialize(storage: nil, async_router: nil, logger: nil)
|
|
27
|
+
@storage = storage || RubyReactor.configuration.storage_adapter
|
|
28
|
+
@async_router = async_router || RubyReactor.configuration.async_router
|
|
29
|
+
@logger = logger || RubyReactor.configuration.logger
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Returns { redispatched:, recollected: } counts.
|
|
33
|
+
def run_once(limit: 1000)
|
|
34
|
+
redispatched = 0
|
|
35
|
+
recollected = 0
|
|
36
|
+
|
|
37
|
+
@storage.scan_maps(count: limit).each do |meta|
|
|
38
|
+
missing = missing_indices(meta)
|
|
39
|
+
if missing.any?
|
|
40
|
+
redispatched += redispatch_missing(meta, missing)
|
|
41
|
+
elsif recollect?(meta)
|
|
42
|
+
retrigger_collector(meta)
|
|
43
|
+
recollected += 1
|
|
44
|
+
end
|
|
45
|
+
rescue StandardError => e
|
|
46
|
+
@logger.warn("RubyReactor::Map::Sweeper failed on map #{meta["map_id"]}: #{e.class}: #{e.message}")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
{ redispatched: redispatched, recollected: recollected }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
private
|
|
53
|
+
|
|
54
|
+
def missing_indices(meta)
|
|
55
|
+
@storage.missing_map_indices(meta["map_id"], meta["count"].to_i, meta["parent_reactor_class_name"])
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def redispatch_missing(meta, missing)
|
|
59
|
+
count = 0
|
|
60
|
+
missing.each do |index|
|
|
61
|
+
next if @storage.lock_held?("map_element:#{meta["map_id"]}:#{index}") # element alive
|
|
62
|
+
|
|
63
|
+
RubyReactor::Map::Dispatcher.requeue_index(meta, index)
|
|
64
|
+
count += 1
|
|
65
|
+
end
|
|
66
|
+
count
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# All results are in. Re-trigger the collector only if no collector/parent is
|
|
70
|
+
# alive and the parent has not already collected this step.
|
|
71
|
+
def recollect?(meta)
|
|
72
|
+
return false if @storage.lock_held?("map_collect:#{meta["map_id"]}") # a collector is running
|
|
73
|
+
return false if parent_live_lock?(meta) # parent execution alive
|
|
74
|
+
return false if parent_already_collected?(meta)
|
|
75
|
+
|
|
76
|
+
true
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# N1: a nested map's parent is a map element running under a `map_element:`
|
|
80
|
+
# lock, not an `async:` lock. Derive the right key from metadata.
|
|
81
|
+
def parent_live_lock?(meta)
|
|
82
|
+
if meta["parent_is_map_element"]
|
|
83
|
+
@storage.lock_held?("map_element:#{meta["outer_map_id"]}:#{meta["outer_index"]}")
|
|
84
|
+
else
|
|
85
|
+
@storage.lock_held?("async:#{meta["parent_context_id"]}")
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def parent_already_collected?(meta)
|
|
90
|
+
data = @storage.retrieve_context(meta["parent_context_id"], meta["parent_reactor_class_name"])
|
|
91
|
+
return false unless data
|
|
92
|
+
|
|
93
|
+
results = data["intermediate_results"] || {}
|
|
94
|
+
status = data["status"].to_s
|
|
95
|
+
results.key?(meta["step_name"].to_s) || %w[completed failed skipped].include?(status)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def retrigger_collector(meta)
|
|
99
|
+
@async_router.perform_map_collection_async(
|
|
100
|
+
parent_context_id: meta["parent_context_id"],
|
|
101
|
+
map_id: meta["map_id"],
|
|
102
|
+
parent_reactor_class_name: meta["parent_reactor_class_name"],
|
|
103
|
+
step_name: meta["step_name"],
|
|
104
|
+
strict_ordering: meta["strict_ordering"],
|
|
105
|
+
timeout: 3600
|
|
106
|
+
)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
data/lib/ruby_reactor/reactor.rb
CHANGED
|
@@ -111,10 +111,11 @@ module RubyReactor
|
|
|
111
111
|
# For async reactors, queue a job for the whole reactor
|
|
112
112
|
@context.status = :running
|
|
113
113
|
Executor.middlewares_for(self.class).on(:before_async_enqueue, @context)
|
|
114
|
+
# Persist BEFORE enqueue — the job payload is identity-only (F2).
|
|
114
115
|
save_context
|
|
115
116
|
|
|
116
|
-
|
|
117
|
-
|
|
117
|
+
@result = configuration.async_router.perform_async(@context.context_id,
|
|
118
|
+
RubyReactor.reactor_storage_name(self.class),
|
|
118
119
|
intermediate_results: @context.intermediate_results)
|
|
119
120
|
|
|
120
121
|
# Even if it's an AsyncResult, it might have finished inline (e.g. Sidekiq::Testing.inline!)
|
|
@@ -312,10 +313,11 @@ module RubyReactor
|
|
|
312
313
|
|
|
313
314
|
def perform_async_run
|
|
314
315
|
@context.status = :running
|
|
316
|
+
# Persist BEFORE enqueue — the job payload is identity-only (F2).
|
|
315
317
|
save_context
|
|
316
318
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
+
@result = configuration.async_router.perform_async(@context.context_id,
|
|
320
|
+
RubyReactor.reactor_storage_name(self.class),
|
|
319
321
|
intermediate_results: @context.intermediate_results)
|
|
320
322
|
|
|
321
323
|
check_for_inline_completion
|
|
@@ -424,7 +426,7 @@ module RubyReactor
|
|
|
424
426
|
|
|
425
427
|
def save_context
|
|
426
428
|
storage = configuration.storage_adapter
|
|
427
|
-
reactor_class_name =
|
|
429
|
+
reactor_class_name = RubyReactor.reactor_storage_name(self.class)
|
|
428
430
|
serialized_context = ContextSerializer.serialize(@context)
|
|
429
431
|
storage.store_context(@context.context_id, serialized_context, reactor_class_name)
|
|
430
432
|
end
|
|
@@ -2,18 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
module RubyReactor
|
|
4
4
|
class SidekiqAdapter
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
# Identity-only payload: the worker rehydrates the live context from storage
|
|
6
|
+
# by (context_id, reactor_class_name). The caller already holds context_id, so
|
|
7
|
+
# there is no blob to deserialize here.
|
|
8
|
+
def self.perform_async(context_id, reactor_class_name = nil, intermediate_results: {})
|
|
9
|
+
job_id = SidekiqWorkers::Worker.perform_async(context_id, reactor_class_name)
|
|
8
10
|
RubyReactor::AsyncResult.new(job_id: job_id, intermediate_results: intermediate_results,
|
|
9
|
-
execution_id:
|
|
11
|
+
execution_id: context_id)
|
|
10
12
|
end
|
|
11
13
|
|
|
12
|
-
def self.perform_in(delay,
|
|
13
|
-
job_id = SidekiqWorkers::Worker.perform_in(delay,
|
|
14
|
-
context = ContextSerializer.deserialize(serialized_context)
|
|
14
|
+
def self.perform_in(delay, context_id, reactor_class_name = nil, intermediate_results: {})
|
|
15
|
+
job_id = SidekiqWorkers::Worker.perform_in(delay, context_id, reactor_class_name)
|
|
15
16
|
RubyReactor::AsyncResult.new(job_id: job_id, intermediate_results: intermediate_results,
|
|
16
|
-
execution_id:
|
|
17
|
+
execution_id: context_id)
|
|
17
18
|
end
|
|
18
19
|
|
|
19
20
|
# rubocop:disable Metrics/ParameterLists
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
|
|
6
|
+
module RubyReactor
|
|
7
|
+
module SidekiqWorkers
|
|
8
|
+
# Self-rescheduling recovery tick. Each run sweeps both the top-level reactor
|
|
9
|
+
# sweeper and the map sweeper, then schedules the next tick — a perpetual
|
|
10
|
+
# chain the host kicks once via `RubyReactor.start_sweeper!`.
|
|
11
|
+
#
|
|
12
|
+
# super_fetch safety. Sidekiq Enterprise `super_fetch` reliably re-runs a job
|
|
13
|
+
# whose worker died mid-execution. For a self-rescheduling chain that is a
|
|
14
|
+
# hazard: a tick can crash AFTER enqueuing its successor but BEFORE acking, so
|
|
15
|
+
# super_fetch recovers the crashed tick *alongside* the successor it already
|
|
16
|
+
# scheduled — the chain forks and then doubles every interval. We therefore do
|
|
17
|
+
# NOT rely on "exactly one job exists". The next tick is claimed by a
|
|
18
|
+
# per-time-window lock: every duplicate computes the SAME target window and
|
|
19
|
+
# only one wins the claim, so recovered/duplicated ticks collapse back to a
|
|
20
|
+
# single chain. The claim lock is never released — it simply expires — so no
|
|
21
|
+
# delete can race two duplicates into both winning.
|
|
22
|
+
class SweeperWorker
|
|
23
|
+
include ::Sidekiq::Worker
|
|
24
|
+
|
|
25
|
+
# retry: false — the sweep is idempotent and self-rescheduling, so a failed
|
|
26
|
+
# tick must not pile up Sidekiq retries; the next tick (or a super_fetch
|
|
27
|
+
# recovery) re-runs it anyway.
|
|
28
|
+
sidekiq_options retry: false, queue: RubyReactor.configuration.sidekiq_queue
|
|
29
|
+
|
|
30
|
+
def perform
|
|
31
|
+
config = RubyReactor.configuration
|
|
32
|
+
return unless config.sweeper_enabled
|
|
33
|
+
|
|
34
|
+
run_sweeps(config)
|
|
35
|
+
ensure
|
|
36
|
+
# Always chain forward (unless disabled), even after an error above, so a
|
|
37
|
+
# single bad sweep can't kill recovery. The window lock keeps this from
|
|
38
|
+
# forking under super_fetch.
|
|
39
|
+
self.class.schedule_next if RubyReactor.configuration.sweeper_enabled
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def run_sweeps(config)
|
|
43
|
+
RubyReactor::Sweeper.run_once(limit: config.sweeper_limit)
|
|
44
|
+
RubyReactor::Map::Sweeper.run_once(limit: config.sweeper_limit)
|
|
45
|
+
rescue StandardError => e
|
|
46
|
+
config.logger.error("RubyReactor::SweeperWorker sweep failed: #{e.class}: #{e.message}")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Enqueue the next tick for the upcoming time window, claiming that window
|
|
50
|
+
# so concurrent/duplicate/recovered ticks produce exactly one successor.
|
|
51
|
+
# Idempotent: also safe to call from `start_sweeper!` on every process boot.
|
|
52
|
+
def self.schedule_next
|
|
53
|
+
interval = RubyReactor.configuration.sweeper_interval
|
|
54
|
+
window = (Time.now.to_i / interval) + 1
|
|
55
|
+
|
|
56
|
+
lock = RubyReactor::Lock.new(
|
|
57
|
+
"sweeper:window:#{window}",
|
|
58
|
+
owner: SecureRandom.uuid,
|
|
59
|
+
ttl: interval * 2, # outlive the window; expires on its own (never released)
|
|
60
|
+
wait: 0,
|
|
61
|
+
auto_extend: false
|
|
62
|
+
)
|
|
63
|
+
lock.acquire # raises AcquisitionError if this window is already claimed
|
|
64
|
+
|
|
65
|
+
delay = (window * interval) - Time.now.to_i
|
|
66
|
+
perform_in([delay, 1].max)
|
|
67
|
+
rescue RubyReactor::Lock::AcquisitionError
|
|
68
|
+
# Another tick already scheduled this window — collapse the duplicate.
|
|
69
|
+
nil
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|