ruby_reactor 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/CHANGELOG.md +14 -0
- data/README.md +179 -26
- data/lib/ruby_reactor/configuration.rb +66 -2
- data/lib/ruby_reactor/context_serializer.rb +9 -4
- data/lib/ruby_reactor/dsl/compose_builder.rb +20 -0
- data/lib/ruby_reactor/dsl/lockable.rb +41 -1
- data/lib/ruby_reactor/executor/ordered_lock_support.rb +307 -0
- data/lib/ruby_reactor/executor/retry_manager.rb +7 -2
- data/lib/ruby_reactor/executor/step_executor.rb +25 -5
- data/lib/ruby_reactor/executor.rb +166 -52
- data/lib/ruby_reactor/lock.rb +13 -0
- data/lib/ruby_reactor/map/collector.rb +41 -0
- data/lib/ruby_reactor/map/dispatcher.rb +42 -0
- data/lib/ruby_reactor/map/element_executor.rb +39 -0
- data/lib/ruby_reactor/map/helpers.rb +10 -3
- data/lib/ruby_reactor/map/sweeper.rb +110 -0
- data/lib/ruby_reactor/ordered_lock.rb +158 -0
- data/lib/ruby_reactor/reactor.rb +48 -5
- data/lib/ruby_reactor/rspec/helpers.rb +6 -0
- data/lib/ruby_reactor/rspec/matchers.rb +66 -0
- data/lib/ruby_reactor/rspec/sidekiq_helpers.rb +70 -0
- data/lib/ruby_reactor/rspec/storage_reset.rb +23 -0
- data/lib/ruby_reactor/rspec/test_subject.rb +14 -28
- data/lib/ruby_reactor/rspec.rb +37 -0
- data/lib/ruby_reactor/sidekiq_adapter.rb +9 -8
- data/lib/ruby_reactor/sidekiq_workers/sweeper_worker.rb +73 -0
- data/lib/ruby_reactor/sidekiq_workers/worker.rb +82 -36
- data/lib/ruby_reactor/step/map_step.rb +18 -2
- data/lib/ruby_reactor/storage/redis_adapter.rb +84 -60
- data/lib/ruby_reactor/storage/redis_locking.rb +8 -0
- data/lib/ruby_reactor/storage/redis_ordered_locking.rb +382 -0
- data/lib/ruby_reactor/sweeper.rb +58 -0
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor.rb +43 -0
- metadata +9 -1
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
# Strict-ordering primitive. A monotonically increasing nonce is assigned at
|
|
5
|
+
# enqueue time; the worker can proceed only when its nonce equals
|
|
6
|
+
# `last_completed + 1`. Otherwise the worker raises {WaitError}, which the
|
|
7
|
+
# Sidekiq worker rescues and re-snoozes via `perform_in`.
|
|
8
|
+
#
|
|
9
|
+
# See `with_ordered_lock` for usage from a reactor.
|
|
10
|
+
class OrderedLock
|
|
11
|
+
# Raised by the gate check when the worker's nonce is ahead of
|
|
12
|
+
# `last_completed + 1`. Carries `retry_after_seconds`, a hint derived from
|
|
13
|
+
# the poison-pill timeout on the *blocker* nonce.
|
|
14
|
+
class WaitError < StandardError
|
|
15
|
+
attr_reader :retry_after_seconds, :key, :nonce, :last_completed
|
|
16
|
+
|
|
17
|
+
def initialize(key:, nonce:, last_completed:, retry_after_seconds:)
|
|
18
|
+
@key = key
|
|
19
|
+
@nonce = nonce
|
|
20
|
+
@last_completed = last_completed
|
|
21
|
+
@retry_after_seconds = retry_after_seconds
|
|
22
|
+
super("OrderedLock '#{key}' nonce #{nonce} waiting on #{last_completed + 1}")
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Default poison-pill: if the blocker nonce was assigned more than
|
|
27
|
+
# `poison_pill_timeout` seconds ago and never advanced, the gate treats it
|
|
28
|
+
# as dead and advances past it. Prevents permanent head-of-line blocking
|
|
29
|
+
# from a crashed caller that INCRed but never enqueued.
|
|
30
|
+
DEFAULT_POISON_PILL_TIMEOUT = 600
|
|
31
|
+
|
|
32
|
+
# TTL on the Redis counter keys. Bumped on every assign so an active
|
|
33
|
+
# sequence never expires; only fully-drained ones GC themselves.
|
|
34
|
+
DEFAULT_TTL = 86_400
|
|
35
|
+
|
|
36
|
+
attr_reader :key, :nonce, :epoch, :poison_pill_timeout, :strict
|
|
37
|
+
|
|
38
|
+
def initialize(key, nonce: nil, epoch: nil, poison_pill_timeout: DEFAULT_POISON_PILL_TIMEOUT, # rubocop:disable Metrics/ParameterLists
|
|
39
|
+
ttl: DEFAULT_TTL, strict: true)
|
|
40
|
+
@key = key
|
|
41
|
+
@nonce = nonce
|
|
42
|
+
@epoch = epoch
|
|
43
|
+
@poison_pill_timeout = poison_pill_timeout
|
|
44
|
+
@ttl = ttl
|
|
45
|
+
@strict = strict
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Atomic INCR on the `next` counter. Caller-side; runs during
|
|
49
|
+
# `Reactor.run` BEFORE `perform_async`. Returns `[nonce, epoch]` — the nonce
|
|
50
|
+
# we own plus the generation it belongs to (used to fence stale stragglers).
|
|
51
|
+
def self.assign(key, ttl: DEFAULT_TTL)
|
|
52
|
+
adapter = RubyReactor.configuration.storage_adapter
|
|
53
|
+
adapter.ordered_lock_assign(key, ttl: ttl)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Gate check. Returns `:go`, `:drained_go`, `:skip_chain_failed`,
|
|
57
|
+
# `:stale_batch`, or raises {WaitError}.
|
|
58
|
+
# - `:go` — proceed to run steps.
|
|
59
|
+
# - `:drained_go` — the batch fully drained and GC'd while this caller slept.
|
|
60
|
+
# A genuine late straggler should run; a Sidekiq redelivery of an
|
|
61
|
+
# already-terminal context should be skipped. The executor disambiguates
|
|
62
|
+
# via the stored context status.
|
|
63
|
+
# - `:skip_chain_failed` — only in strict mode: an earlier nonce in this
|
|
64
|
+
# sequence terminated with a Failure, so this run is short-circuited
|
|
65
|
+
# with `Skipped(reason: :ordered_lock_chain_failed)` without executing.
|
|
66
|
+
# - `:stale_batch` — this run's epoch no longer matches the key's current
|
|
67
|
+
# generation: its batch fully drained and the numbering was reused by a
|
|
68
|
+
# newer batch. The run is short-circuited with
|
|
69
|
+
# `Skipped(reason: :ordered_lock_stale_batch)` and must not participate.
|
|
70
|
+
# - `:poison_advance` is collapsed to `:go` from the caller's perspective.
|
|
71
|
+
def check!
|
|
72
|
+
raise ArgumentError, "OrderedLock#check! requires a nonce" unless @nonce
|
|
73
|
+
|
|
74
|
+
state, retry_after, last_completed, first_failed = adapter.ordered_lock_can_proceed(
|
|
75
|
+
@key,
|
|
76
|
+
nonce: @nonce,
|
|
77
|
+
poison_pill_timeout: @poison_pill_timeout,
|
|
78
|
+
epoch: @epoch.to_i
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
case state
|
|
82
|
+
when "go", "poison_advance"
|
|
83
|
+
chain_failed?(first_failed) ? :skip_chain_failed : :go
|
|
84
|
+
when "drained_go"
|
|
85
|
+
# Batch fully drained and GC'd while this caller slept. A genuine late
|
|
86
|
+
# straggler may run (poison semantics); a redelivery of an
|
|
87
|
+
# already-terminal context must not. The executor disambiguates via the
|
|
88
|
+
# stored context status. (fail_key is GC'd here, so no chain check.)
|
|
89
|
+
:drained_go
|
|
90
|
+
when "stale"
|
|
91
|
+
:stale_batch
|
|
92
|
+
when "wait"
|
|
93
|
+
raise WaitError.new(
|
|
94
|
+
key: @key,
|
|
95
|
+
nonce: @nonce,
|
|
96
|
+
last_completed: last_completed,
|
|
97
|
+
retry_after_seconds: retry_after
|
|
98
|
+
)
|
|
99
|
+
else
|
|
100
|
+
raise "Unexpected OrderedLock state: #{state.inspect}"
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Move `last_completed` forward. Idempotent: only the nonce equal to
|
|
105
|
+
# `last_completed + 1` advances; others are no-ops (the poison-pill path
|
|
106
|
+
# may have already skipped us).
|
|
107
|
+
#
|
|
108
|
+
# Call on terminal status only (success, permanent failure, escalated skip).
|
|
109
|
+
# Retryable failures must NOT advance — the same nonce keeps owning until
|
|
110
|
+
# the job either succeeds or exhausts its retry budget.
|
|
111
|
+
#
|
|
112
|
+
# `failed:` records this nonce as the chain-failure marker (only the FIRST
|
|
113
|
+
# failure sticks). In strict mode the marker causes subsequent nonces to
|
|
114
|
+
# short-circuit with Skipped.
|
|
115
|
+
def advance!(failed: false)
|
|
116
|
+
raise ArgumentError, "OrderedLock#advance! requires a nonce" unless @nonce
|
|
117
|
+
|
|
118
|
+
adapter.ordered_lock_advance(@key, nonce: @nonce, failed: failed, epoch: @epoch.to_i, ttl: @ttl)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Restamp this nonce's `assigned_at` to "now" while its steps execute, so a
|
|
122
|
+
# successor does not poison-advance past a blocker that is merely slow (not
|
|
123
|
+
# dead). Called on an interval by a background heartbeat thread for the
|
|
124
|
+
# duration of step execution. No-op if the nonce's timer was already deleted
|
|
125
|
+
# by a terminal advance, or if the batch has gone stale (epoch fence).
|
|
126
|
+
def heartbeat!
|
|
127
|
+
return unless @nonce
|
|
128
|
+
|
|
129
|
+
adapter.ordered_lock_heartbeat(@key, nonce: @nonce, epoch: @epoch.to_i)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Read-only inspection. `{ next:, last_completed:, in_flight: [...] }`.
|
|
133
|
+
def self.peek(key)
|
|
134
|
+
RubyReactor.configuration.storage_adapter.ordered_lock_peek(key)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Manual ops escape hatch — force-advance past a stuck nonce.
|
|
138
|
+
def self.skip!(key, nonce:)
|
|
139
|
+
RubyReactor.configuration.storage_adapter.ordered_lock_skip(key, nonce: nonce)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Nuke all counters for a key. Ops only; concurrent enqueues during reset
|
|
143
|
+
# produce undefined ordering.
|
|
144
|
+
def self.reset!(key)
|
|
145
|
+
RubyReactor.configuration.storage_adapter.ordered_lock_reset(key)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
private
|
|
149
|
+
|
|
150
|
+
def chain_failed?(first_failed)
|
|
151
|
+
@strict && first_failed.to_i.positive? && @nonce > first_failed.to_i
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def adapter
|
|
155
|
+
RubyReactor.configuration.storage_adapter
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
data/lib/ruby_reactor/reactor.rb
CHANGED
|
@@ -102,14 +102,20 @@ module RubyReactor
|
|
|
102
102
|
return validation_result
|
|
103
103
|
end
|
|
104
104
|
|
|
105
|
+
# Assign-at-enqueue: ordered_lock nonce is INCRed atomically here so
|
|
106
|
+
# the order matches the caller's order, not whichever worker happens to
|
|
107
|
+
# pick the job up first.
|
|
108
|
+
assign_ordered_lock_nonce!
|
|
109
|
+
|
|
105
110
|
if self.class.async? && !@context.inline_async_execution
|
|
106
111
|
# For async reactors, queue a job for the whole reactor
|
|
107
112
|
@context.status = :running
|
|
108
113
|
Executor.middlewares_for(self.class).on(:before_async_enqueue, @context)
|
|
114
|
+
# Persist BEFORE enqueue — the job payload is identity-only (F2).
|
|
109
115
|
save_context
|
|
110
116
|
|
|
111
|
-
|
|
112
|
-
|
|
117
|
+
@result = configuration.async_router.perform_async(@context.context_id,
|
|
118
|
+
RubyReactor.reactor_storage_name(self.class),
|
|
113
119
|
intermediate_results: @context.intermediate_results)
|
|
114
120
|
|
|
115
121
|
# Even if it's an AsyncResult, it might have finished inline (e.g. Sidekiq::Testing.inline!)
|
|
@@ -307,10 +313,11 @@ module RubyReactor
|
|
|
307
313
|
|
|
308
314
|
def perform_async_run
|
|
309
315
|
@context.status = :running
|
|
316
|
+
# Persist BEFORE enqueue — the job payload is identity-only (F2).
|
|
310
317
|
save_context
|
|
311
318
|
|
|
312
|
-
|
|
313
|
-
|
|
319
|
+
@result = configuration.async_router.perform_async(@context.context_id,
|
|
320
|
+
RubyReactor.reactor_storage_name(self.class),
|
|
314
321
|
intermediate_results: @context.intermediate_results)
|
|
315
322
|
|
|
316
323
|
check_for_inline_completion
|
|
@@ -419,10 +426,46 @@ module RubyReactor
|
|
|
419
426
|
|
|
420
427
|
def save_context
|
|
421
428
|
storage = configuration.storage_adapter
|
|
422
|
-
reactor_class_name =
|
|
429
|
+
reactor_class_name = RubyReactor.reactor_storage_name(self.class)
|
|
423
430
|
serialized_context = ContextSerializer.serialize(@context)
|
|
424
431
|
storage.store_context(@context.context_id, serialized_context, reactor_class_name)
|
|
425
432
|
end
|
|
433
|
+
|
|
434
|
+
def assign_ordered_lock_nonce!
|
|
435
|
+
return unless self.class.respond_to?(:ordered_lock_config) && self.class.ordered_lock_config
|
|
436
|
+
return if @context.private_data[:ordered_lock] || @context.private_data["ordered_lock"]
|
|
437
|
+
|
|
438
|
+
config = self.class.ordered_lock_config
|
|
439
|
+
key = config[:key_proc].call(@context.inputs)
|
|
440
|
+
|
|
441
|
+
# Synchronous nested `Reactor.run` of an ordered-lock reactor on the same
|
|
442
|
+
# key would deadlock: the outer nonce holds the slot, an inner nonce
|
|
443
|
+
# would never advance until the outer completes — but the outer is
|
|
444
|
+
# blocked waiting for the inner to return. Mirror the compose behavior:
|
|
445
|
+
# silently skip nonce assignment (the inner runs without gate/advance)
|
|
446
|
+
# and log a warning so this isn't invisible.
|
|
447
|
+
active = Executor::OrderedLockSupport.active_keys
|
|
448
|
+
if active.include?(key)
|
|
449
|
+
RubyReactor.configuration.logger.warn(
|
|
450
|
+
"RubyReactor: nested `Reactor.run` of #{self.class.name || "<anonymous>"} on " \
|
|
451
|
+
"ordered-lock key '#{key}' from inside another ordered-lock reactor on the same " \
|
|
452
|
+
"key — nonce assignment skipped, inner run executes without ordering enforcement. " \
|
|
453
|
+
"Use a different key or move the inner call to a top-level invocation if you need ordering."
|
|
454
|
+
)
|
|
455
|
+
return
|
|
456
|
+
end
|
|
457
|
+
|
|
458
|
+
nonce, epoch = RubyReactor::OrderedLock.assign(key, ttl: config[:ttl])
|
|
459
|
+
|
|
460
|
+
@context.private_data[:ordered_lock] = {
|
|
461
|
+
key: key,
|
|
462
|
+
nonce: nonce,
|
|
463
|
+
epoch: epoch,
|
|
464
|
+
poison_pill_timeout: config[:poison_pill_timeout],
|
|
465
|
+
ttl: config[:ttl],
|
|
466
|
+
strict: config.fetch(:strict, true)
|
|
467
|
+
}
|
|
468
|
+
end
|
|
426
469
|
end
|
|
427
470
|
# rubocop:enable Metrics/ClassLength
|
|
428
471
|
end
|
|
@@ -2,7 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
module RubyReactor
|
|
4
4
|
module RSpec
|
|
5
|
+
# Globally-included helpers. Only methods whose names clearly belong to
|
|
6
|
+
# RubyReactor's test surface live here (`test_reactor`). Sidekiq-coupled
|
|
7
|
+
# helpers live in `SidekiqHelpers` and are scoped to `type: :reactor`.
|
|
5
8
|
module Helpers
|
|
9
|
+
# Build a `TestSubject` around a reactor invocation. Captures the run for
|
|
10
|
+
# later introspection via matchers; runs the reactor lazily on first
|
|
11
|
+
# query unless `.run` is called explicitly.
|
|
6
12
|
def test_reactor(reactor_class, inputs, context: {}, async: nil, process_jobs: true)
|
|
7
13
|
TestSubject.new(
|
|
8
14
|
reactor_class: reactor_class,
|
|
@@ -415,6 +415,72 @@ module RubyReactor
|
|
|
415
415
|
end
|
|
416
416
|
end
|
|
417
417
|
|
|
418
|
+
# Asserts the last-assigned ordered_lock nonce for a key. Subject is
|
|
419
|
+
# the user-provided ordered_lock key (without the `ordered_lock:` prefix).
|
|
420
|
+
#
|
|
421
|
+
# expect("orders:42").to have_ordered_lock_next(3)
|
|
422
|
+
::RSpec::Matchers.define :have_ordered_lock_next do |expected|
|
|
423
|
+
match { |key| Matchers.coordination_adapter.ordered_lock_peek(key)[:next] == expected }
|
|
424
|
+
|
|
425
|
+
failure_message do |key|
|
|
426
|
+
state = Matchers.coordination_adapter.ordered_lock_peek(key)
|
|
427
|
+
"expected ordered_lock '#{key}' next to be #{expected}, got #{state[:next]} " \
|
|
428
|
+
"(last_completed: #{state[:last_completed]}, in_flight: #{state[:in_flight].inspect})"
|
|
429
|
+
end
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
# Asserts the last-advanced cursor for an ordered_lock key.
|
|
433
|
+
#
|
|
434
|
+
# expect("orders:42").to have_ordered_lock_last_completed(2)
|
|
435
|
+
::RSpec::Matchers.define :have_ordered_lock_last_completed do |expected|
|
|
436
|
+
match do |key|
|
|
437
|
+
Matchers.coordination_adapter.ordered_lock_peek(key)[:last_completed] == expected
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
failure_message do |key|
|
|
441
|
+
state = Matchers.coordination_adapter.ordered_lock_peek(key)
|
|
442
|
+
"expected ordered_lock '#{key}' last_completed to be #{expected}, " \
|
|
443
|
+
"got #{state[:last_completed]} (next: #{state[:next]}, in_flight: #{state[:in_flight].inspect})"
|
|
444
|
+
end
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
# Asserts the exact set of in-flight nonces for an ordered_lock key.
|
|
448
|
+
# Order-insensitive — the matcher sorts both sides.
|
|
449
|
+
#
|
|
450
|
+
# expect("orders:42").to have_ordered_lock_in_flight(2, 3)
|
|
451
|
+
::RSpec::Matchers.define :have_ordered_lock_in_flight do |*expected|
|
|
452
|
+
match do |key|
|
|
453
|
+
actual = Matchers.coordination_adapter.ordered_lock_peek(key)[:in_flight].sort
|
|
454
|
+
actual == expected.flatten.map(&:to_i).sort
|
|
455
|
+
end
|
|
456
|
+
|
|
457
|
+
failure_message do |key|
|
|
458
|
+
state = Matchers.coordination_adapter.ordered_lock_peek(key)
|
|
459
|
+
"expected ordered_lock '#{key}' in_flight to be #{expected.flatten.sort.inspect}, " \
|
|
460
|
+
"got #{state[:in_flight].inspect} (next: #{state[:next]}, last_completed: #{state[:last_completed]})"
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
# Asserts an ordered_lock key has fully drained — counters GC'd, no
|
|
465
|
+
# in-flight nonces. After a clean drain `peek` returns all zeros.
|
|
466
|
+
#
|
|
467
|
+
# expect("orders:42").to be_ordered_lock_drained
|
|
468
|
+
::RSpec::Matchers.define :be_ordered_lock_drained do
|
|
469
|
+
match do |key|
|
|
470
|
+
state = Matchers.coordination_adapter.ordered_lock_peek(key)
|
|
471
|
+
state[:next].zero? && state[:last_completed].zero? && state[:in_flight].empty?
|
|
472
|
+
end
|
|
473
|
+
|
|
474
|
+
failure_message do |key|
|
|
475
|
+
state = Matchers.coordination_adapter.ordered_lock_peek(key)
|
|
476
|
+
"expected ordered_lock '#{key}' to be drained, but state is #{state.inspect}"
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
failure_message_when_negated do |key|
|
|
480
|
+
"expected ordered_lock '#{key}' not to be drained, but counters are all zero"
|
|
481
|
+
end
|
|
482
|
+
end
|
|
483
|
+
|
|
418
484
|
# Add more matchers as per plan
|
|
419
485
|
# rubocop:enable Metrics/BlockLength
|
|
420
486
|
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
module RSpec
|
|
5
|
+
# Async-job manipulation helpers. Names like `drain_async_jobs` are too
|
|
6
|
+
# generic to live in the global spec namespace, so this module is only
|
|
7
|
+
# auto-included into examples tagged `type: :reactor`. Specs that need it
|
|
8
|
+
# outside that tag should `include RubyReactor::RSpec::SidekiqHelpers`
|
|
9
|
+
# explicitly.
|
|
10
|
+
module SidekiqHelpers
|
|
11
|
+
# Drain every queued async job across all RubyReactor worker classes
|
|
12
|
+
# until the queues are empty. Recursive — handles jobs that re-enqueue
|
|
13
|
+
# themselves (e.g. ordered_lock snoozes) and worker chains that queue
|
|
14
|
+
# additional jobs.
|
|
15
|
+
def drain_async_jobs(max_iterations: 100)
|
|
16
|
+
SidekiqHelpers.drain_async_jobs(max_iterations: max_iterations)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# All currently-pending async jobs, wrapped in `PendingJob` so callers
|
|
20
|
+
# can perform individual jobs out-of-order (e.g. to assert
|
|
21
|
+
# ordered_lock's snooze behavior) without touching Sidekiq internals.
|
|
22
|
+
def pending_async_jobs
|
|
23
|
+
SidekiqHelpers.pending_async_jobs
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
PendingJob = Struct.new(:worker_class, :raw) do
|
|
27
|
+
def perform!
|
|
28
|
+
worker_class.jobs.delete(raw)
|
|
29
|
+
worker_class.new.perform(*raw["args"])
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def args
|
|
33
|
+
raw["args"]
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def self.worker_classes
|
|
38
|
+
@worker_classes ||= [
|
|
39
|
+
RubyReactor::SidekiqWorkers::Worker,
|
|
40
|
+
RubyReactor::SidekiqWorkers::MapElementWorker,
|
|
41
|
+
RubyReactor::SidekiqWorkers::MapCollectorWorker
|
|
42
|
+
]
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.drain_async_jobs(max_iterations: 100)
|
|
46
|
+
return unless defined?(Sidekiq::Testing)
|
|
47
|
+
|
|
48
|
+
max_iterations.times do
|
|
49
|
+
processed_any = false
|
|
50
|
+
worker_classes.each do |worker_class|
|
|
51
|
+
while (job = worker_class.jobs.shift)
|
|
52
|
+
worker_class.new.perform(*job["args"])
|
|
53
|
+
processed_any = true
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
break unless processed_any
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def self.pending_async_jobs
|
|
62
|
+
return [] unless defined?(Sidekiq::Testing)
|
|
63
|
+
|
|
64
|
+
worker_classes.flat_map do |worker_class|
|
|
65
|
+
worker_class.jobs.map { |raw| PendingJob.new(worker_class, raw) }
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyReactor
|
|
4
|
+
module RSpec
|
|
5
|
+
# Test-only `reset!` impls layered onto storage adapters at framework
|
|
6
|
+
# load time. Kept out of `lib/ruby_reactor/storage/*` so production code
|
|
7
|
+
# never gains a "wipe everything" entry point.
|
|
8
|
+
module StorageReset
|
|
9
|
+
module RedisAdapterReset
|
|
10
|
+
def reset!
|
|
11
|
+
@redis.flushdb
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def self.install!
|
|
16
|
+
return if @installed
|
|
17
|
+
|
|
18
|
+
::RubyReactor::Storage::RedisAdapter.prepend(RedisAdapterReset)
|
|
19
|
+
@installed = true
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -246,6 +246,8 @@ module RubyReactor
|
|
|
246
246
|
end
|
|
247
247
|
end
|
|
248
248
|
RubyReactor::Success.new(val)
|
|
249
|
+
when "skipped"
|
|
250
|
+
skipped_result(ctx)
|
|
249
251
|
when "running"
|
|
250
252
|
# Try to determine if it is truly running or if we just missed the completion
|
|
251
253
|
if @process_jobs && defined?(Sidekiq::Testing)
|
|
@@ -269,6 +271,17 @@ module RubyReactor
|
|
|
269
271
|
end
|
|
270
272
|
end
|
|
271
273
|
|
|
274
|
+
# A clean halt: either a `with_period` gate or a step returning
|
|
275
|
+
# `RubyReactor.Skipped(...)`. The sync run already produced the exact
|
|
276
|
+
# Skipped (reason/step intact) — surface it. For async runs the worker
|
|
277
|
+
# swallows the return value, so rebuild from the trace.
|
|
278
|
+
def skipped_result(ctx)
|
|
279
|
+
return @run_result if @run_result.is_a?(RubyReactor::Skipped)
|
|
280
|
+
|
|
281
|
+
entry = ctx.execution_trace.reverse.find { |t| t[:type].to_s == "skipped" }
|
|
282
|
+
RubyReactor::Skipped.new(reason: entry&.dig(:reason), step_name: entry&.dig(:step))
|
|
283
|
+
end
|
|
284
|
+
|
|
272
285
|
def success?
|
|
273
286
|
ensure_executed!
|
|
274
287
|
@reactor_instance.context.status.to_s == "completed"
|
|
@@ -419,34 +432,7 @@ module RubyReactor
|
|
|
419
432
|
def process_pending_jobs
|
|
420
433
|
return unless defined?(Sidekiq::Testing)
|
|
421
434
|
|
|
422
|
-
|
|
423
|
-
# This handles batched map execution where jobs queue more jobs
|
|
424
|
-
max_iterations = 100
|
|
425
|
-
iterations = 0
|
|
426
|
-
|
|
427
|
-
while iterations < max_iterations
|
|
428
|
-
iterations += 1
|
|
429
|
-
jobs_processed = false
|
|
430
|
-
|
|
431
|
-
# Known worker classes to check
|
|
432
|
-
worker_classes = [
|
|
433
|
-
RubyReactor::SidekiqWorkers::Worker,
|
|
434
|
-
RubyReactor::SidekiqWorkers::MapElementWorker,
|
|
435
|
-
RubyReactor::SidekiqWorkers::MapCollectorWorker
|
|
436
|
-
]
|
|
437
|
-
|
|
438
|
-
worker_classes.each do |worker_class|
|
|
439
|
-
while worker_class.jobs.any?
|
|
440
|
-
job = worker_class.jobs.shift
|
|
441
|
-
worker_class.new.perform(*job["args"])
|
|
442
|
-
jobs_processed = true
|
|
443
|
-
end
|
|
444
|
-
end
|
|
445
|
-
|
|
446
|
-
break unless jobs_processed
|
|
447
|
-
end
|
|
448
|
-
|
|
449
|
-
# Final reload
|
|
435
|
+
SidekiqHelpers.drain_async_jobs
|
|
450
436
|
@reactor_instance = @reactor_class.find(@reactor_instance.context.context_id)
|
|
451
437
|
end
|
|
452
438
|
|
data/lib/ruby_reactor/rspec.rb
CHANGED
|
@@ -2,17 +2,54 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "rspec/helpers"
|
|
4
4
|
require_relative "rspec/matchers"
|
|
5
|
+
require_relative "rspec/sidekiq_helpers"
|
|
6
|
+
require_relative "rspec/storage_reset"
|
|
5
7
|
require_relative "rspec/test_subject"
|
|
6
8
|
|
|
7
9
|
module RubyReactor
|
|
8
10
|
module RSpec
|
|
11
|
+
# Examples opt into RubyReactor's RSpec setup (Sidekiq fake mode,
|
|
12
|
+
# storage wipe, snooze knob reset) by declaring `type: :reactor`.
|
|
13
|
+
REACTOR_METADATA = { type: :reactor }.freeze
|
|
14
|
+
|
|
15
|
+
DEFAULT_SNOOZE_BASE_DELAY = 5
|
|
16
|
+
DEFAULT_SNOOZE_JITTER = 5
|
|
17
|
+
DEFAULT_SNOOZE_MAX_ATTEMPTS = 20
|
|
18
|
+
|
|
9
19
|
def self.configure(config)
|
|
10
20
|
require_relative "rspec/step_executor_patch"
|
|
11
21
|
|
|
12
22
|
config.include RubyReactor::RSpec::Helpers
|
|
13
23
|
config.include RubyReactor::RSpec::Matchers
|
|
24
|
+
config.include RubyReactor::RSpec::SidekiqHelpers, REACTOR_METADATA
|
|
14
25
|
|
|
15
26
|
::RubyReactor::Executor::StepExecutor.prepend(RubyReactor::RSpec::StepExecutorPatch)
|
|
27
|
+
StorageReset.install!
|
|
28
|
+
|
|
29
|
+
config.before(:each, REACTOR_METADATA) do
|
|
30
|
+
RubyReactor::RSpec.prepare_example!
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Idempotent setup invoked before each `type: :reactor` example. Restores
|
|
35
|
+
# Sidekiq fake mode, clears the queues, wipes the storage adapter, and
|
|
36
|
+
# rolls back snooze knobs so cross-example bleed-through can't happen.
|
|
37
|
+
def self.prepare_example!
|
|
38
|
+
if defined?(::Sidekiq::Testing)
|
|
39
|
+
begin
|
|
40
|
+
::Sidekiq::Testing.fake! unless ::Sidekiq::Testing.fake?
|
|
41
|
+
rescue ::Sidekiq::Testing::TestModeAlreadySetError
|
|
42
|
+
# Nested fake!/inline! block already active in this thread; leave it.
|
|
43
|
+
end
|
|
44
|
+
::Sidekiq::Worker.clear_all
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
adapter = ::RubyReactor.configuration.storage_adapter
|
|
48
|
+
adapter.reset! if adapter.respond_to?(:reset!)
|
|
49
|
+
|
|
50
|
+
::RubyReactor.configuration.lock_snooze_base_delay = DEFAULT_SNOOZE_BASE_DELAY
|
|
51
|
+
::RubyReactor.configuration.lock_snooze_jitter = DEFAULT_SNOOZE_JITTER
|
|
52
|
+
::RubyReactor.configuration.lock_snooze_max_attempts = DEFAULT_SNOOZE_MAX_ATTEMPTS
|
|
16
53
|
end
|
|
17
54
|
end
|
|
18
55
|
end
|
|
@@ -2,18 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
module RubyReactor
|
|
4
4
|
class SidekiqAdapter
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
# Identity-only payload: the worker rehydrates the live context from storage
|
|
6
|
+
# by (context_id, reactor_class_name). The caller already holds context_id, so
|
|
7
|
+
# there is no blob to deserialize here.
|
|
8
|
+
def self.perform_async(context_id, reactor_class_name = nil, intermediate_results: {})
|
|
9
|
+
job_id = SidekiqWorkers::Worker.perform_async(context_id, reactor_class_name)
|
|
8
10
|
RubyReactor::AsyncResult.new(job_id: job_id, intermediate_results: intermediate_results,
|
|
9
|
-
execution_id:
|
|
11
|
+
execution_id: context_id)
|
|
10
12
|
end
|
|
11
13
|
|
|
12
|
-
def self.perform_in(delay,
|
|
13
|
-
job_id = SidekiqWorkers::Worker.perform_in(delay,
|
|
14
|
-
context = ContextSerializer.deserialize(serialized_context)
|
|
14
|
+
def self.perform_in(delay, context_id, reactor_class_name = nil, intermediate_results: {})
|
|
15
|
+
job_id = SidekiqWorkers::Worker.perform_in(delay, context_id, reactor_class_name)
|
|
15
16
|
RubyReactor::AsyncResult.new(job_id: job_id, intermediate_results: intermediate_results,
|
|
16
|
-
execution_id:
|
|
17
|
+
execution_id: context_id)
|
|
17
18
|
end
|
|
18
19
|
|
|
19
20
|
# rubocop:disable Metrics/ParameterLists
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
|
|
6
|
+
module RubyReactor
|
|
7
|
+
module SidekiqWorkers
|
|
8
|
+
# Self-rescheduling recovery tick. Each run sweeps both the top-level reactor
|
|
9
|
+
# sweeper and the map sweeper, then schedules the next tick — a perpetual
|
|
10
|
+
# chain the host kicks once via `RubyReactor.start_sweeper!`.
|
|
11
|
+
#
|
|
12
|
+
# super_fetch safety. Sidekiq Enterprise `super_fetch` reliably re-runs a job
|
|
13
|
+
# whose worker died mid-execution. For a self-rescheduling chain that is a
|
|
14
|
+
# hazard: a tick can crash AFTER enqueuing its successor but BEFORE acking, so
|
|
15
|
+
# super_fetch recovers the crashed tick *alongside* the successor it already
|
|
16
|
+
# scheduled — the chain forks and then doubles every interval. We therefore do
|
|
17
|
+
# NOT rely on "exactly one job exists". The next tick is claimed by a
|
|
18
|
+
# per-time-window lock: every duplicate computes the SAME target window and
|
|
19
|
+
# only one wins the claim, so recovered/duplicated ticks collapse back to a
|
|
20
|
+
# single chain. The claim lock is never released — it simply expires — so no
|
|
21
|
+
# delete can race two duplicates into both winning.
|
|
22
|
+
class SweeperWorker
|
|
23
|
+
include ::Sidekiq::Worker
|
|
24
|
+
|
|
25
|
+
# retry: false — the sweep is idempotent and self-rescheduling, so a failed
|
|
26
|
+
# tick must not pile up Sidekiq retries; the next tick (or a super_fetch
|
|
27
|
+
# recovery) re-runs it anyway.
|
|
28
|
+
sidekiq_options retry: false, queue: RubyReactor.configuration.sidekiq_queue
|
|
29
|
+
|
|
30
|
+
def perform
|
|
31
|
+
config = RubyReactor.configuration
|
|
32
|
+
return unless config.sweeper_enabled
|
|
33
|
+
|
|
34
|
+
run_sweeps(config)
|
|
35
|
+
ensure
|
|
36
|
+
# Always chain forward (unless disabled), even after an error above, so a
|
|
37
|
+
# single bad sweep can't kill recovery. The window lock keeps this from
|
|
38
|
+
# forking under super_fetch.
|
|
39
|
+
self.class.schedule_next if RubyReactor.configuration.sweeper_enabled
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def run_sweeps(config)
|
|
43
|
+
RubyReactor::Sweeper.run_once(limit: config.sweeper_limit)
|
|
44
|
+
RubyReactor::Map::Sweeper.run_once(limit: config.sweeper_limit)
|
|
45
|
+
rescue StandardError => e
|
|
46
|
+
config.logger.error("RubyReactor::SweeperWorker sweep failed: #{e.class}: #{e.message}")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Enqueue the next tick for the upcoming time window, claiming that window
|
|
50
|
+
# so concurrent/duplicate/recovered ticks produce exactly one successor.
|
|
51
|
+
# Idempotent: also safe to call from `start_sweeper!` on every process boot.
|
|
52
|
+
def self.schedule_next
|
|
53
|
+
interval = RubyReactor.configuration.sweeper_interval
|
|
54
|
+
window = (Time.now.to_i / interval) + 1
|
|
55
|
+
|
|
56
|
+
lock = RubyReactor::Lock.new(
|
|
57
|
+
"sweeper:window:#{window}",
|
|
58
|
+
owner: SecureRandom.uuid,
|
|
59
|
+
ttl: interval * 2, # outlive the window; expires on its own (never released)
|
|
60
|
+
wait: 0,
|
|
61
|
+
auto_extend: false
|
|
62
|
+
)
|
|
63
|
+
lock.acquire # raises AcquisitionError if this window is already claimed
|
|
64
|
+
|
|
65
|
+
delay = (window * interval) - Time.now.to_i
|
|
66
|
+
perform_in([delay, 1].max)
|
|
67
|
+
rescue RubyReactor::Lock::AcquisitionError
|
|
68
|
+
# Another tick already scheduled this window — collapse the duplicate.
|
|
69
|
+
nil
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|