ruby_reactor 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/.specify/memory/constitution.md +17 -14
- data/CHANGELOG.md +254 -0
- data/CLAUDE.md +1 -1
- data/README.md +66 -15
- data/lib/ruby_reactor/adapters/active_job/map_element_rollback_worker.rb +20 -0
- data/lib/ruby_reactor/adapters/active_job/router.rb +32 -4
- data/lib/ruby_reactor/adapters/sidekiq/map_element_rollback_worker.rb +16 -0
- data/lib/ruby_reactor/adapters/sidekiq/router.rb +32 -4
- data/lib/ruby_reactor/context.rb +14 -1
- data/lib/ruby_reactor/dsl/async_macros.rb +30 -1
- data/lib/ruby_reactor/dsl/async_reactor_builder.rb +12 -6
- data/lib/ruby_reactor/dsl/compose_builder.rb +12 -6
- data/lib/ruby_reactor/dsl/definition_warnings.rb +30 -0
- data/lib/ruby_reactor/dsl/interrupt_builder.rb +1 -3
- data/lib/ruby_reactor/dsl/map_builder.rb +27 -9
- data/lib/ruby_reactor/dsl/step_builder.rb +93 -34
- data/lib/ruby_reactor/error/argument_resolution_error.rb +19 -0
- data/lib/ruby_reactor/error/rescuable.rb +33 -0
- data/lib/ruby_reactor/error/rollback_handed_off.rb +27 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +63 -35
- data/lib/ruby_reactor/executor/result_handler.rb +85 -63
- data/lib/ruby_reactor/executor/retry_manager.rb +4 -3
- data/lib/ruby_reactor/executor/step_coordination.rb +11 -8
- data/lib/ruby_reactor/executor/step_executor.rb +61 -49
- data/lib/ruby_reactor/executor.rb +221 -7
- data/lib/ruby_reactor/map/collector.rb +31 -89
- data/lib/ruby_reactor/map/dispatcher.rb +65 -8
- data/lib/ruby_reactor/map/element_executor.rb +21 -11
- data/lib/ruby_reactor/map/element_rollback.rb +158 -0
- data/lib/ruby_reactor/map/helpers.rb +11 -125
- data/lib/ruby_reactor/map/result_enumerator.rb +18 -12
- data/lib/ruby_reactor/map/step_rollback.rb +136 -0
- data/lib/ruby_reactor/map/sweeper.rb +66 -8
- data/lib/ruby_reactor/map.rb +17 -0
- data/lib/ruby_reactor/max_retries_exhausted_failure.rb +4 -2
- data/lib/ruby_reactor/reactor.rb +65 -2
- data/lib/ruby_reactor/rspec/active_job_helpers.rb +13 -2
- data/lib/ruby_reactor/rspec/matchers.rb +59 -24
- data/lib/ruby_reactor/rspec/sidekiq_helpers.rb +7 -3
- data/lib/ruby_reactor/rspec/test_subject.rb +13 -3
- data/lib/ruby_reactor/rspec.rb +3 -6
- data/lib/ruby_reactor/step/compose_step.rb +10 -1
- data/lib/ruby_reactor/step/map_step.rb +144 -36
- data/lib/ruby_reactor/step.rb +7 -0
- data/lib/ruby_reactor/step_worker.rb +46 -22
- data/lib/ruby_reactor/storage/adapter.rb +18 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +27 -2
- data/lib/ruby_reactor/storage/redis_map_rollback.rb +153 -0
- data/lib/ruby_reactor/storage/redis_reactor_scan.rb +1 -1
- data/lib/ruby_reactor/sweeper.rb +3 -1
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +11 -1
- data/lib/ruby_reactor/web/public/assets/{index-CeZU-ESu.js → index-C6ejszCQ.js} +12 -12
- data/lib/ruby_reactor/web/public/assets/index-DM5darhy.css +1 -0
- data/lib/ruby_reactor/web/public/index.html +2 -2
- data/lib/ruby_reactor/worker.rb +7 -3
- data/lib/ruby_reactor.rb +17 -6
- data/specs/007-execution-flow-analysis/analysis/README.md +147 -0
- data/specs/007-execution-flow-analysis/analysis/execution-order.md +359 -0
- data/specs/007-execution-flow-analysis/analysis/findings-and-options.md +502 -0
- data/specs/007-execution-flow-analysis/analysis/invariants.md +109 -0
- data/specs/007-execution-flow-analysis/checklists/requirements.md +39 -0
- data/specs/007-execution-flow-analysis/contracts/report-structure.md +71 -0
- data/specs/007-execution-flow-analysis/data-model.md +83 -0
- data/specs/007-execution-flow-analysis/evidence/harness.rb +231 -0
- data/specs/007-execution-flow-analysis/evidence/output.txt +333 -0
- data/specs/007-execution-flow-analysis/evidence/probes/01_plain.rb +122 -0
- data/specs/007-execution-flow-analysis/evidence/probes/02_compose.rb +182 -0
- data/specs/007-execution-flow-analysis/evidence/probes/03_map.rb +235 -0
- data/specs/007-execution-flow-analysis/evidence/probes/04_async.rb +132 -0
- data/specs/007-execution-flow-analysis/evidence/probes/05_background.rb +58 -0
- data/specs/007-execution-flow-analysis/evidence/probes/06_coordination.rb +158 -0
- data/specs/007-execution-flow-analysis/evidence/probes/07_interrupts_manual.rb +185 -0
- data/specs/007-execution-flow-analysis/evidence/run.rb +15 -0
- data/specs/007-execution-flow-analysis/plan.md +127 -0
- data/specs/007-execution-flow-analysis/quickstart.md +51 -0
- data/specs/007-execution-flow-analysis/research.md +202 -0
- data/specs/007-execution-flow-analysis/spec.md +270 -0
- data/specs/007-execution-flow-analysis/tasks.md +257 -0
- data/specs/008-rollback-reliability/checklists/requirements.md +43 -0
- data/specs/008-rollback-reliability/contracts/api-surface.md +126 -0
- data/specs/008-rollback-reliability/contracts/rollback-semantics.md +76 -0
- data/specs/008-rollback-reliability/data-model.md +139 -0
- data/specs/008-rollback-reliability/plan.md +233 -0
- data/specs/008-rollback-reliability/quickstart.md +105 -0
- data/specs/008-rollback-reliability/research.md +653 -0
- data/specs/008-rollback-reliability/spec.md +561 -0
- data/specs/008-rollback-reliability/tasks.md +1110 -0
- data/specs/009-distributed-map-undo/checklists/requirements.md +53 -0
- data/specs/009-distributed-map-undo/contracts/api-surface.md +125 -0
- data/specs/009-distributed-map-undo/contracts/rollback-protocol.md +157 -0
- data/specs/009-distributed-map-undo/data-model.md +151 -0
- data/specs/009-distributed-map-undo/plan.md +323 -0
- data/specs/009-distributed-map-undo/pr-description.md +78 -0
- data/specs/009-distributed-map-undo/quickstart.md +154 -0
- data/specs/009-distributed-map-undo/research.md +585 -0
- data/specs/009-distributed-map-undo/spec.md +360 -0
- data/specs/009-distributed-map-undo/tasks.md +1060 -0
- data/specs/future_improvements.md +64 -28
- metadata +54 -3
- data/lib/ruby_reactor/web/public/assets/index-BQvIWPdx.css +0 -1
|
@@ -4,8 +4,10 @@ module RubyReactor
|
|
|
4
4
|
module RSpec
|
|
5
5
|
# rubocop:disable Metrics/ModuleLength
|
|
6
6
|
module Matchers
|
|
7
|
+
extend ::RSpec::Matchers::DSL
|
|
8
|
+
|
|
7
9
|
# rubocop:disable Metrics/BlockLength
|
|
8
|
-
|
|
10
|
+
matcher :be_success do
|
|
9
11
|
match do |subject|
|
|
10
12
|
subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
|
|
11
13
|
subject.success?
|
|
@@ -63,7 +65,7 @@ module RubyReactor
|
|
|
63
65
|
end
|
|
64
66
|
end
|
|
65
67
|
|
|
66
|
-
|
|
68
|
+
matcher :be_failure do
|
|
67
69
|
match do |subject|
|
|
68
70
|
subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
|
|
69
71
|
subject.failure?
|
|
@@ -74,7 +76,7 @@ module RubyReactor
|
|
|
74
76
|
end
|
|
75
77
|
end
|
|
76
78
|
|
|
77
|
-
|
|
79
|
+
matcher :have_run_step do |step_name|
|
|
78
80
|
match do |subject|
|
|
79
81
|
subject.ensure_executed!
|
|
80
82
|
@trace = subject.reactor_instance.context.execution_trace
|
|
@@ -127,7 +129,7 @@ module RubyReactor
|
|
|
127
129
|
end
|
|
128
130
|
end
|
|
129
131
|
|
|
130
|
-
|
|
132
|
+
matcher :have_retried_step do |step_name|
|
|
131
133
|
match do |subject|
|
|
132
134
|
subject.ensure_executed!
|
|
133
135
|
attempts = subject.reactor_instance.context.retry_context.attempts_for_step(step_name)
|
|
@@ -151,7 +153,7 @@ module RubyReactor
|
|
|
151
153
|
end
|
|
152
154
|
end
|
|
153
155
|
|
|
154
|
-
|
|
156
|
+
matcher :have_validation_error do |field|
|
|
155
157
|
match do |subject|
|
|
156
158
|
subject.ensure_executed!
|
|
157
159
|
return false unless subject.failure?
|
|
@@ -173,7 +175,7 @@ module RubyReactor
|
|
|
173
175
|
end
|
|
174
176
|
|
|
175
177
|
# Matcher to check if reactor is paused at an interrupt
|
|
176
|
-
|
|
178
|
+
matcher :be_paused do
|
|
177
179
|
match do |subject|
|
|
178
180
|
subject.ensure_executed!
|
|
179
181
|
subject.paused?
|
|
@@ -188,9 +190,26 @@ module RubyReactor
|
|
|
188
190
|
end
|
|
189
191
|
end
|
|
190
192
|
|
|
193
|
+
# A run whose rollback handed off at a fan-out map and has not finished
|
|
194
|
+
# (009): its element rollback jobs are still pending.
|
|
195
|
+
matcher :be_rolling_back do
|
|
196
|
+
match do |subject|
|
|
197
|
+
subject.ensure_executed!
|
|
198
|
+
subject.rolling_back?
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
failure_message do |subject|
|
|
202
|
+
"expected reactor to be rolling back, but status was #{subject.reactor_instance.context.status}"
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
failure_message_when_negated do |_subject|
|
|
206
|
+
"expected reactor not to be rolling back, but it is"
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
191
210
|
# Matcher to check if reactor is paused at a specific interrupt step
|
|
192
211
|
# Works with both single and multiple concurrent interrupts
|
|
193
|
-
|
|
212
|
+
matcher :be_paused_at do |*step_names|
|
|
194
213
|
match do |subject|
|
|
195
214
|
subject.ensure_executed!
|
|
196
215
|
return false unless subject.paused?
|
|
@@ -221,7 +240,7 @@ module RubyReactor
|
|
|
221
240
|
end
|
|
222
241
|
|
|
223
242
|
# Matcher to check the exact set of ready interrupt steps
|
|
224
|
-
|
|
243
|
+
matcher :have_ready_interrupts do |*expected_steps|
|
|
225
244
|
match do |subject|
|
|
226
245
|
subject.ensure_executed!
|
|
227
246
|
return false unless subject.paused?
|
|
@@ -261,6 +280,15 @@ module RubyReactor
|
|
|
261
280
|
RubyReactor.configuration.storage_adapter
|
|
262
281
|
end
|
|
263
282
|
|
|
283
|
+
def self.rate_limit_total(key_base, period, since)
|
|
284
|
+
return coordination_adapter.rate_limit_count(key_base, period) unless since
|
|
285
|
+
|
|
286
|
+
seconds = RubyReactor::Period.period_seconds(period)
|
|
287
|
+
((since.to_i / seconds)..(Time.now.to_i / seconds)).sum do |bucket|
|
|
288
|
+
coordination_adapter.rate_limit_count(key_base, period, now: bucket * seconds)
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
|
|
264
292
|
# Distinguishes `RubyReactor::Halt` (a clean halt) from a plain
|
|
265
293
|
# `Success`. Works on any object with a `halted?` predicate.
|
|
266
294
|
#
|
|
@@ -268,7 +296,7 @@ module RubyReactor
|
|
|
268
296
|
# expect(result).to be_halted
|
|
269
297
|
# expect(result).to be_halted.because(:period)
|
|
270
298
|
# expect(result).to be_halted.at_step(:second)
|
|
271
|
-
|
|
299
|
+
matcher :be_halted do
|
|
272
300
|
match do |subject|
|
|
273
301
|
subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
|
|
274
302
|
actual = subject.respond_to?(:result) ? subject.result : subject
|
|
@@ -311,7 +339,7 @@ module RubyReactor
|
|
|
311
339
|
# Examples:
|
|
312
340
|
# expect(result).to be_skipped
|
|
313
341
|
# expect(subject).to be_skipped.at_step(:maybe_sync)
|
|
314
|
-
|
|
342
|
+
matcher :be_skipped do
|
|
315
343
|
match do |subject|
|
|
316
344
|
subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
|
|
317
345
|
actual = subject.respond_to?(:result) ? subject.result : subject
|
|
@@ -358,7 +386,7 @@ module RubyReactor
|
|
|
358
386
|
#
|
|
359
387
|
# expect("order:42").to be_locked
|
|
360
388
|
# expect("order:42").to be_locked.by("ctx-abc")
|
|
361
|
-
|
|
389
|
+
matcher :be_locked do
|
|
362
390
|
match do |key|
|
|
363
391
|
info = Matchers.coordination_adapter.lock_info("lock:#{key}")
|
|
364
392
|
next false unless info
|
|
@@ -391,7 +419,7 @@ module RubyReactor
|
|
|
391
419
|
# user-provided semaphore name (without the "semaphore:" prefix).
|
|
392
420
|
#
|
|
393
421
|
# expect("api_limit").to have_available_tokens(3)
|
|
394
|
-
|
|
422
|
+
matcher :have_available_tokens do |expected|
|
|
395
423
|
match do |name|
|
|
396
424
|
Matchers.coordination_adapter.semaphore_state(name)[:available] == expected
|
|
397
425
|
end
|
|
@@ -406,7 +434,7 @@ module RubyReactor
|
|
|
406
434
|
# Asserts the number of currently-checked-out semaphore tokens.
|
|
407
435
|
#
|
|
408
436
|
# expect("api_limit").to have_held_tokens(2)
|
|
409
|
-
|
|
437
|
+
matcher :have_held_tokens do |expected|
|
|
410
438
|
match do |name|
|
|
411
439
|
Matchers.coordination_adapter.semaphore_state(name)[:held] == expected
|
|
412
440
|
end
|
|
@@ -419,22 +447,29 @@ module RubyReactor
|
|
|
419
447
|
end
|
|
420
448
|
|
|
421
449
|
# Asserts the current rate-limit counter for a (key_base, period) pair.
|
|
422
|
-
# Use `.for(period_unit)` to specify which window.
|
|
450
|
+
# Use `.for(period_unit)` to specify which window. Add `.since(time)` to
|
|
451
|
+
# sum every bucket from `time` to now instead — buckets are fixed
|
|
452
|
+
# windows, so a run that straddles a boundary splits its count.
|
|
423
453
|
#
|
|
424
454
|
# expect("stripe:42").to have_rate_limit_count(3).for(:second)
|
|
425
|
-
|
|
455
|
+
# expect("stripe:42").to have_rate_limit_count(1).for(:minute).since(started_at)
|
|
456
|
+
matcher :have_rate_limit_count do |expected|
|
|
426
457
|
match do |key_base|
|
|
427
458
|
raise ArgumentError, "have_rate_limit_count requires .for(period)" unless @period
|
|
428
459
|
|
|
429
|
-
Matchers.
|
|
460
|
+
Matchers.rate_limit_total(key_base, @period, @since) == expected
|
|
430
461
|
end
|
|
431
462
|
|
|
432
463
|
chain :for do |period|
|
|
433
464
|
@period = period
|
|
434
465
|
end
|
|
435
466
|
|
|
467
|
+
chain :since do |time|
|
|
468
|
+
@since = time
|
|
469
|
+
end
|
|
470
|
+
|
|
436
471
|
failure_message do |key_base|
|
|
437
|
-
actual = Matchers.
|
|
472
|
+
actual = Matchers.rate_limit_total(key_base, @period, @since)
|
|
438
473
|
"expected rate-limit '#{key_base}' (#{@period}) count to be #{expected}, got #{actual}"
|
|
439
474
|
end
|
|
440
475
|
end
|
|
@@ -442,7 +477,7 @@ module RubyReactor
|
|
|
442
477
|
# Asserts that a `with_period` bucket has been marked. Use `.for(period)`.
|
|
443
478
|
#
|
|
444
479
|
# expect("daily_report:7").to be_period_marked.for(:day)
|
|
445
|
-
|
|
480
|
+
matcher :be_period_marked do
|
|
446
481
|
match do |key_base|
|
|
447
482
|
raise ArgumentError, "be_period_marked requires .for(period)" unless @period
|
|
448
483
|
|
|
@@ -466,7 +501,7 @@ module RubyReactor
|
|
|
466
501
|
# the user-provided ordered_lock key (without the `ordered_lock:` prefix).
|
|
467
502
|
#
|
|
468
503
|
# expect("orders:42").to have_ordered_lock_next(3)
|
|
469
|
-
|
|
504
|
+
matcher :have_ordered_lock_next do |expected|
|
|
470
505
|
match { |key| Matchers.coordination_adapter.ordered_lock_peek(key)[:next] == expected }
|
|
471
506
|
|
|
472
507
|
failure_message do |key|
|
|
@@ -479,7 +514,7 @@ module RubyReactor
|
|
|
479
514
|
# Asserts the last-advanced cursor for an ordered_lock key.
|
|
480
515
|
#
|
|
481
516
|
# expect("orders:42").to have_ordered_lock_last_completed(2)
|
|
482
|
-
|
|
517
|
+
matcher :have_ordered_lock_last_completed do |expected|
|
|
483
518
|
match do |key|
|
|
484
519
|
Matchers.coordination_adapter.ordered_lock_peek(key)[:last_completed] == expected
|
|
485
520
|
end
|
|
@@ -495,7 +530,7 @@ module RubyReactor
|
|
|
495
530
|
# Order-insensitive — the matcher sorts both sides.
|
|
496
531
|
#
|
|
497
532
|
# expect("orders:42").to have_ordered_lock_in_flight(2, 3)
|
|
498
|
-
|
|
533
|
+
matcher :have_ordered_lock_in_flight do |*expected|
|
|
499
534
|
match do |key|
|
|
500
535
|
actual = Matchers.coordination_adapter.ordered_lock_peek(key)[:in_flight].sort
|
|
501
536
|
actual == expected.flatten.map(&:to_i).sort
|
|
@@ -512,7 +547,7 @@ module RubyReactor
|
|
|
512
547
|
# in-flight nonces. After a clean drain `peek` returns all zeros.
|
|
513
548
|
#
|
|
514
549
|
# expect("orders:42").to be_ordered_lock_drained
|
|
515
|
-
|
|
550
|
+
matcher :be_ordered_lock_drained do
|
|
516
551
|
match do |key|
|
|
517
552
|
state = Matchers.coordination_adapter.ordered_lock_peek(key)
|
|
518
553
|
state[:next].zero? && state[:last_completed].zero? && state[:in_flight].empty?
|
|
@@ -535,7 +570,7 @@ module RubyReactor
|
|
|
535
570
|
#
|
|
536
571
|
# expect(result).to have_contended_at(:charge)
|
|
537
572
|
# expect(result).to have_contended_at(:charge).on("acct:1")
|
|
538
|
-
|
|
573
|
+
matcher :have_contended_at do |step_name|
|
|
539
574
|
match do |subject|
|
|
540
575
|
trace = execution_trace_for(subject)
|
|
541
576
|
@entry = trace.find do |t|
|
|
@@ -580,7 +615,7 @@ module RubyReactor
|
|
|
580
615
|
# expect(result).to have_rollback_failure(:charge)
|
|
581
616
|
# expect(subject).to have_rollback_failure(:charge).for_key("acct:1").because(:coordination_unavailable)
|
|
582
617
|
# expect(subject).not_to have_rollback_failure(:charge)
|
|
583
|
-
|
|
618
|
+
matcher :have_rollback_failure do |step_name|
|
|
584
619
|
match do |subject|
|
|
585
620
|
@entries = rollback_failures_for(subject)
|
|
586
621
|
@entries.any? do |e|
|
|
@@ -12,15 +12,18 @@ module RubyReactor
|
|
|
12
12
|
# until the queues are empty. Recursive — handles jobs that re-enqueue
|
|
13
13
|
# themselves (e.g. ordered_lock snoozes) and worker chains that queue
|
|
14
14
|
# additional jobs.
|
|
15
|
+
# Whichever test backend is active (Sidekiq fake mode or the ActiveJob
|
|
16
|
+
# `:test` adapter), via `AsyncTestHelpers`.
|
|
15
17
|
def drain_async_jobs(max_iterations: 100)
|
|
16
|
-
|
|
18
|
+
AsyncTestHelpers.drain_async_jobs(max_iterations: max_iterations)
|
|
17
19
|
end
|
|
18
20
|
|
|
19
21
|
# All currently-pending async jobs, wrapped in `PendingJob` so callers
|
|
20
22
|
# can perform individual jobs out-of-order (e.g. to assert
|
|
21
|
-
# ordered_lock's snooze behavior) without touching
|
|
23
|
+
# ordered_lock's snooze behavior) without touching backend internals.
|
|
24
|
+
# `worker_class` names the job's class on either backend.
|
|
22
25
|
def pending_async_jobs
|
|
23
|
-
|
|
26
|
+
AsyncTestHelpers.pending_async_jobs
|
|
24
27
|
end
|
|
25
28
|
|
|
26
29
|
PendingJob = Struct.new(:worker_class, :raw) do
|
|
@@ -39,6 +42,7 @@ module RubyReactor
|
|
|
39
42
|
RubyReactor::Adapters::Sidekiq::Worker,
|
|
40
43
|
RubyReactor::Adapters::Sidekiq::MapElementWorker,
|
|
41
44
|
RubyReactor::Adapters::Sidekiq::MapCollectorWorker,
|
|
45
|
+
RubyReactor::Adapters::Sidekiq::MapElementRollbackWorker,
|
|
42
46
|
RubyReactor::Adapters::Sidekiq::StepWorker
|
|
43
47
|
]
|
|
44
48
|
end
|
|
@@ -351,14 +351,14 @@ module RubyReactor
|
|
|
351
351
|
RubyReactor::Success.new(val)
|
|
352
352
|
when "halted", "skipped" # "skipped" is the legacy name for a halt
|
|
353
353
|
halted_result(ctx)
|
|
354
|
-
when "running"
|
|
354
|
+
when "running", "rolling_back"
|
|
355
355
|
# Try to determine if it is truly running or if we just missed the completion
|
|
356
356
|
if @process_jobs && AsyncTestHelpers.active?
|
|
357
357
|
# Force one more check
|
|
358
358
|
process_pending_jobs
|
|
359
359
|
# Reload status
|
|
360
360
|
@reactor_instance = @reactor_class.find(@reactor_instance.context.context_id)
|
|
361
|
-
return result unless @reactor_instance.context.status.to_s
|
|
361
|
+
return result unless %w[running rolling_back].include?(@reactor_instance.context.status.to_s)
|
|
362
362
|
end
|
|
363
363
|
|
|
364
364
|
# If still running, return a Pending/Running result instead of nil
|
|
@@ -405,6 +405,13 @@ module RubyReactor
|
|
|
405
405
|
@reactor_instance.context.status.to_s == "paused"
|
|
406
406
|
end
|
|
407
407
|
|
|
408
|
+
# Whether the run's rollback handed off at a fan-out map and has not
|
|
409
|
+
# finished (009). Reloads, so it reflects jobs performed since.
|
|
410
|
+
def rolling_back?
|
|
411
|
+
ensure_executed!
|
|
412
|
+
@reactor_instance.context.status.to_s == "rolling_back"
|
|
413
|
+
end
|
|
414
|
+
|
|
408
415
|
# Get the current step where the reactor is paused (interrupt step)
|
|
409
416
|
# Note: When multiple interrupts are ready, this returns just one of them.
|
|
410
417
|
# Use `ready_interrupt_steps` to get all ready interrupt steps.
|
|
@@ -523,9 +530,12 @@ module RubyReactor
|
|
|
523
530
|
|
|
524
531
|
def ensure_executed!
|
|
525
532
|
run unless @executed
|
|
533
|
+
return unless %w[running rolling_back].include?(@reactor_instance.context.status.to_s)
|
|
526
534
|
|
|
535
|
+
# Not finished: jobs the spec performed itself may have moved it on.
|
|
536
|
+
@reactor_instance = @reactor_class.find(@reactor_instance.context.context_id)
|
|
527
537
|
# Process jobs if status is running and processing is enabled
|
|
528
|
-
return unless @process_jobs && @reactor_instance.context.status.to_s
|
|
538
|
+
return unless @process_jobs && %w[running rolling_back].include?(@reactor_instance.context.status.to_s)
|
|
529
539
|
|
|
530
540
|
process_pending_jobs
|
|
531
541
|
end
|
data/lib/ruby_reactor/rspec.rb
CHANGED
|
@@ -21,15 +21,12 @@ module RubyReactor
|
|
|
21
21
|
def self.configure(config)
|
|
22
22
|
require_relative "rspec/step_executor_patch"
|
|
23
23
|
|
|
24
|
-
#
|
|
24
|
+
# Helpers, matchers and async-job helpers are only auto-included into
|
|
25
25
|
# examples tagged `type: :reactor`, so a host app's other specs keep a
|
|
26
26
|
# clean namespace. Specs that want them without the tag can
|
|
27
|
-
# `include RubyReactor::RSpec::Helpers` explicitly.
|
|
28
|
-
#
|
|
29
|
-
# Matchers stay unconditional: `::RSpec::Matchers.define` registers them
|
|
30
|
-
# globally at load time, so gating this include would not scope anything.
|
|
27
|
+
# `include RubyReactor::RSpec::Helpers` / `Matchers` explicitly.
|
|
31
28
|
config.include RubyReactor::RSpec::Helpers, REACTOR_METADATA
|
|
32
|
-
config.include RubyReactor::RSpec::Matchers
|
|
29
|
+
config.include RubyReactor::RSpec::Matchers, REACTOR_METADATA
|
|
33
30
|
config.include RubyReactor::RSpec::SidekiqHelpers, REACTOR_METADATA
|
|
34
31
|
|
|
35
32
|
::RubyReactor::Executor::StepExecutor.prepend(RubyReactor::RSpec::StepExecutorPatch)
|
|
@@ -34,7 +34,13 @@ module RubyReactor
|
|
|
34
34
|
return RubyReactor.Success() unless composed_data && composed_data[:context]
|
|
35
35
|
|
|
36
36
|
child_context = composed_data[:context]
|
|
37
|
-
|
|
37
|
+
# Linked like a running child, so a construct in it finds the run that
|
|
38
|
+
# owns it: a fan-out map's rollback resumes that run (009 S-3).
|
|
39
|
+
link_contexts(child_context, context)
|
|
40
|
+
# The child's own class: it survives serialization, while a Class in a
|
|
41
|
+
# stored undo record's arguments comes back as its name (a compose
|
|
42
|
+
# inside a map element is rolled back from its stored row).
|
|
43
|
+
executor = RubyReactor::Executor.new(child_context.reactor_class, {}, child_context)
|
|
38
44
|
executor.undo_all
|
|
39
45
|
executor.save_context
|
|
40
46
|
|
|
@@ -46,6 +52,9 @@ module RubyReactor
|
|
|
46
52
|
|
|
47
53
|
alias undo compensate
|
|
48
54
|
|
|
55
|
+
# An interrupted run is undone too: undo replays only the child's completed steps.
|
|
56
|
+
def self.undoes_partial_run? = true
|
|
57
|
+
|
|
49
58
|
private
|
|
50
59
|
|
|
51
60
|
def build_composed_inputs(mappings)
|
|
@@ -3,6 +3,13 @@
|
|
|
3
3
|
module RubyReactor
|
|
4
4
|
class Step
|
|
5
5
|
class MapStep < RubyReactor::Step
|
|
6
|
+
include RubyReactor::Map::StepRollback
|
|
7
|
+
|
|
8
|
+
# Seconds a rollback waits for an element's liveness lock. The last
|
|
9
|
+
# element to settle triggers the collector before its own job releases
|
|
10
|
+
# that lock, so a short wait covers the gap.
|
|
11
|
+
ELEMENT_LOCK_WAIT = 2
|
|
12
|
+
|
|
6
13
|
# Untyped, so no validation: declared only so `inputs.x` can read them.
|
|
7
14
|
input :source
|
|
8
15
|
input :mapped_reactor_class
|
|
@@ -10,14 +17,16 @@ module RubyReactor
|
|
|
10
17
|
input :strict_ordering, optional: true
|
|
11
18
|
input :batch_size, optional: true
|
|
12
19
|
input :collect_block, optional: true
|
|
13
|
-
input :
|
|
20
|
+
input :atomic, optional: true
|
|
14
21
|
input :fan_out, optional: true
|
|
15
22
|
|
|
16
23
|
def run
|
|
17
|
-
return RubyReactor::Failure("Map source cannot be nil") if inputs.source.nil?
|
|
18
|
-
|
|
19
24
|
# Initialize map state in context if not present
|
|
20
25
|
context.map_operations ||= {}
|
|
26
|
+
# Dispatched already: the owner run resumed (009 R-03) — adopt the
|
|
27
|
+
# settled outcome, never dispatch again (I-3).
|
|
28
|
+
return adopt_dispatched_map(context.current_step) if dispatched_map_id(context.current_step)
|
|
29
|
+
return RubyReactor::Failure("Map source cannot be nil") if inputs.source.nil?
|
|
21
30
|
|
|
22
31
|
if fan_out?
|
|
23
32
|
run_async(context.current_step)
|
|
@@ -26,11 +35,32 @@ module RubyReactor
|
|
|
26
35
|
end
|
|
27
36
|
end
|
|
28
37
|
|
|
38
|
+
# Compensating a failed map and undoing a completed one are the same work,
|
|
39
|
+
# as for compose: replay the undo stack of every element that COMPLETED
|
|
40
|
+
# (a failed element already rolled itself back; a halted or skipped one
|
|
41
|
+
# did nothing to undo), newest-started first. Elements are found through
|
|
42
|
+
# the index both modes write, so nothing per element lives in the parent
|
|
43
|
+
# (008 R-02). Runs only once every element has settled (R-04), from the
|
|
44
|
+
# execution that owns the map, so it is the elements' only writer.
|
|
45
|
+
#
|
|
46
|
+
# A map rolls back where it ran (009 R-01): a fan-out map one job per
|
|
47
|
+
# started element, `batch_size` per throw, handing the run off until
|
|
48
|
+
# they all report; an inline map in process, 100 elements per read.
|
|
29
49
|
def compensate
|
|
30
|
-
|
|
31
|
-
|
|
50
|
+
step_name = context.current_step
|
|
51
|
+
map_id = "#{context.context_id}:#{step_name}"
|
|
52
|
+
dispatched_map_id(step_name) ? distributed_rollback(map_id, step_name) : inline_rollback(map_id, step_name)
|
|
32
53
|
end
|
|
33
54
|
|
|
55
|
+
alias undo compensate
|
|
56
|
+
|
|
57
|
+
# An interrupted run is undone too: undo replays only each element's completed steps.
|
|
58
|
+
def self.undoes_partial_run? = true
|
|
59
|
+
|
|
60
|
+
# The undo record keeps no arguments: rollback reads the map's
|
|
61
|
+
# declaration and records, never the resolved source (009 R-14).
|
|
62
|
+
def self.rollback_arguments(_resolved) = {}
|
|
63
|
+
|
|
34
64
|
class << self
|
|
35
65
|
def build_mapped_inputs(mappings, context, element)
|
|
36
66
|
built = {}
|
|
@@ -79,6 +109,12 @@ module RubyReactor
|
|
|
79
109
|
|
|
80
110
|
private
|
|
81
111
|
|
|
112
|
+
# Read from the map step's static declaration, not from the undo record,
|
|
113
|
+
# which a fan-out map leaves empty (R-03).
|
|
114
|
+
def element_class
|
|
115
|
+
context.reactor_class.steps[context.current_step].arguments[:mapped_reactor_class][:source].value
|
|
116
|
+
end
|
|
117
|
+
|
|
82
118
|
# Fans out anywhere except inside a map element: an element's result and
|
|
83
119
|
# the map's completion counter are tracked by its own ElementExecutor job,
|
|
84
120
|
# so a nested hand-off there would escape that tracking. A reactor worker
|
|
@@ -90,16 +126,20 @@ module RubyReactor
|
|
|
90
126
|
inputs.fan_out
|
|
91
127
|
end
|
|
92
128
|
|
|
129
|
+
def atomic?
|
|
130
|
+
inputs.atomic.nil? || inputs.atomic
|
|
131
|
+
end
|
|
132
|
+
|
|
93
133
|
def run_inline
|
|
94
134
|
results = execute_inline_map
|
|
95
135
|
return results if results.is_a?(RubyReactor::Failure) || results.is_a?(RubyReactor::Halt)
|
|
96
136
|
|
|
97
|
-
|
|
137
|
+
collect_results(results)
|
|
98
138
|
end
|
|
99
139
|
|
|
100
140
|
def execute_inline_map
|
|
101
141
|
results = []
|
|
102
|
-
|
|
142
|
+
atomic = atomic?
|
|
103
143
|
|
|
104
144
|
inputs.source.each_with_index do |element, index|
|
|
105
145
|
result = execute_single_element(element, index)
|
|
@@ -108,12 +148,10 @@ module RubyReactor
|
|
|
108
148
|
# rather than being collected as a (nil) value.
|
|
109
149
|
return result if result.is_a?(RubyReactor::Halt)
|
|
110
150
|
|
|
111
|
-
if
|
|
112
|
-
return result # Stop immediately on first failure
|
|
113
|
-
end
|
|
151
|
+
return result if atomic && result.failure? # Stop immediately on first failure
|
|
114
152
|
|
|
115
|
-
#
|
|
116
|
-
results << (
|
|
153
|
+
# A non-atomic map collects Result objects; an atomic one, values.
|
|
154
|
+
results << (atomic ? result.value : result)
|
|
117
155
|
end
|
|
118
156
|
|
|
119
157
|
results
|
|
@@ -141,7 +179,8 @@ module RubyReactor
|
|
|
141
179
|
name: context.current_step,
|
|
142
180
|
type: :map_ref,
|
|
143
181
|
map_id: map_id,
|
|
144
|
-
element_reactor_class: inputs.mapped_reactor_class.name
|
|
182
|
+
element_reactor_class: inputs.mapped_reactor_class.name,
|
|
183
|
+
started: index + 1
|
|
145
184
|
}
|
|
146
185
|
|
|
147
186
|
executor = RubyReactor::Executor.new(inputs.mapped_reactor_class, {}, child_context)
|
|
@@ -155,20 +194,6 @@ module RubyReactor
|
|
|
155
194
|
child_context.inline_async_execution = parent_context.inline_async_execution
|
|
156
195
|
end
|
|
157
196
|
|
|
158
|
-
def process_results(results, collect_block, _fail_fast = true)
|
|
159
|
-
if collect_block
|
|
160
|
-
begin
|
|
161
|
-
# Collect block receives Result objects when fail_fast is false, values when true
|
|
162
|
-
return RubyReactor::Success(collect_block.call(results))
|
|
163
|
-
rescue StandardError => e
|
|
164
|
-
return RubyReactor::Failure(e)
|
|
165
|
-
end
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
# Simplified: both branches returned Success(results)
|
|
169
|
-
RubyReactor::Success(results)
|
|
170
|
-
end
|
|
171
|
-
|
|
172
197
|
def run_async(step_name)
|
|
173
198
|
map_id = "#{context.context_id}:#{step_name}"
|
|
174
199
|
context.map_operations[step_name.to_s] = map_id
|
|
@@ -188,10 +213,12 @@ module RubyReactor
|
|
|
188
213
|
element_reactor_class: inputs.mapped_reactor_class.name
|
|
189
214
|
}
|
|
190
215
|
|
|
216
|
+
# The owner's id: the run the caller holds, and the one the map's
|
|
217
|
+
# completion resumes (a composed child's own id is internal).
|
|
191
218
|
RubyReactor::DispatchResult.new(
|
|
192
219
|
job_id: job_id,
|
|
193
220
|
intermediate_results: context.intermediate_results,
|
|
194
|
-
execution_id:
|
|
221
|
+
execution_id: owner_context.context_id
|
|
195
222
|
)
|
|
196
223
|
end
|
|
197
224
|
|
|
@@ -200,10 +227,83 @@ module RubyReactor
|
|
|
200
227
|
storage.initialize_map_operation(
|
|
201
228
|
map_id, inputs.source.size, context.reactor_class.name,
|
|
202
229
|
strict_ordering: inputs.strict_ordering, reactor_class_info: reactor_class_info,
|
|
230
|
+
owner_context_id: owner_context.context_id,
|
|
231
|
+
owner_reactor_class_name: RubyReactor.reactor_storage_name(owner_context.reactor_class),
|
|
232
|
+
batch_size: effective_batch_size, atomic: atomic?,
|
|
203
233
|
**map_recovery_metadata(context.current_step)
|
|
204
234
|
)
|
|
205
235
|
end
|
|
206
236
|
|
|
237
|
+
# The top-level run: the one execution that writes the context tree, and
|
|
238
|
+
# the one the map's completion resumes (009 R-03).
|
|
239
|
+
def owner_context
|
|
240
|
+
context.root_context || context
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Declared, or `Map::DEFAULT_BATCH_SIZE`: no throw enqueues more (R-10).
|
|
244
|
+
def effective_batch_size
|
|
245
|
+
inputs.batch_size || RubyReactor::Map::DEFAULT_BATCH_SIZE
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def dispatched_map_id(step_name)
|
|
249
|
+
Utils::FetchIndifferent.call(context.map_operations || {}, step_name)
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Re-entry of a dispatched map (S-1). Unsettled (an early or duplicate
|
|
253
|
+
# resume): hand off again without dispatching. Settled: the failing
|
|
254
|
+
# element's Failure for an atomic map, else the collected results. The
|
|
255
|
+
# executor records either like any step result.
|
|
256
|
+
def adopt_dispatched_map(step_name)
|
|
257
|
+
map_id = dispatched_map_id(step_name)
|
|
258
|
+
storage = RubyReactor.configuration.storage_adapter
|
|
259
|
+
metadata = storage.retrieve_map_metadata(map_id, context.reactor_class.name)
|
|
260
|
+
return RubyReactor::Failure("map :#{step_name} records expired before it settled") unless metadata
|
|
261
|
+
|
|
262
|
+
total = metadata["count"].to_i
|
|
263
|
+
if storage.count_map_results(map_id, context.reactor_class.name) < total
|
|
264
|
+
return RubyReactor::DispatchResult.new(job_id: "map:#{map_id}", execution_id: owner_context.context_id,
|
|
265
|
+
intermediate_results: context.intermediate_results)
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
failed_id = storage.retrieve_map_failed_context_id(map_id, context.reactor_class.name)
|
|
269
|
+
return adopt_element_failure(step_name, failed_id) if failed_id
|
|
270
|
+
|
|
271
|
+
record_elements_started(step_name, total)
|
|
272
|
+
collect_results(RubyReactor::Map::ResultEnumerator.new(map_id, context.reactor_class.name,
|
|
273
|
+
strict_ordering: inputs.strict_ordering))
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# The element's own Failure, carrying its rollback failures: the executor
|
|
277
|
+
# compensates the map (its completed elements) and rolls the run back.
|
|
278
|
+
def adopt_element_failure(step_name, failed_id)
|
|
279
|
+
data = RubyReactor.configuration.storage_adapter.retrieve_context(
|
|
280
|
+
failed_id, RubyReactor.reactor_storage_name(element_class)
|
|
281
|
+
)
|
|
282
|
+
return RubyReactor::Failure("map :#{step_name} element failed; its context expired") unless data
|
|
283
|
+
|
|
284
|
+
reason = RubyReactor::Context.deserialize_from_retry(data).failure_reason
|
|
285
|
+
reason.is_a?(RubyReactor::Failure) ? reason : RubyReactor::Failure(reason)
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# The collect block gets Result objects for a non-atomic map, values for
|
|
289
|
+
# an atomic one; a raise is the map's failure.
|
|
290
|
+
def collect_results(results)
|
|
291
|
+
return RubyReactor::Success(results) unless inputs.collect_block
|
|
292
|
+
|
|
293
|
+
RubyReactor::Success(inputs.collect_block.call(results))
|
|
294
|
+
rescue RubyReactor::Error::Rescuable => e
|
|
295
|
+
RubyReactor.configuration.logger.error("Map collect block raised: #{e.message}")
|
|
296
|
+
RubyReactor::Failure(e)
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# Every index of a completed map ran: record the count on the map's
|
|
300
|
+
# reference, which outlives the element index, so a late rollback names
|
|
301
|
+
# each element whose context expired.
|
|
302
|
+
def record_elements_started(step_name, total)
|
|
303
|
+
ref = Utils::FetchIndifferent.call(context.composed_contexts, step_name)
|
|
304
|
+
ref[:started] = total if ref
|
|
305
|
+
end
|
|
306
|
+
|
|
207
307
|
# Recovery metadata for the map sweeper. When this map runs inside a map
|
|
208
308
|
# element (context.map_metadata present), it is a NESTED map: its parent
|
|
209
309
|
# holds the element's `map_element:` lock, not an `async:` lock (N1).
|
|
@@ -219,13 +319,13 @@ module RubyReactor
|
|
|
219
319
|
end
|
|
220
320
|
|
|
221
321
|
def dispatch_async_map(map_id, _reactor_class_info, step_name)
|
|
222
|
-
# Every async map runs through the per-element Dispatcher path
|
|
223
|
-
#
|
|
224
|
-
#
|
|
225
|
-
#
|
|
226
|
-
#
|
|
227
|
-
#
|
|
228
|
-
batch_size =
|
|
322
|
+
# Every async map runs through the per-element Dispatcher path: each
|
|
323
|
+
# element runs in its own worker, with the map counter/collector
|
|
324
|
+
# tracking completion. This lets elements with async steps or async
|
|
325
|
+
# retries hand off correctly instead of being forced to run
|
|
326
|
+
# synchronously in a single worker. Without a declared batch_size, at
|
|
327
|
+
# most `Map::DEFAULT_BATCH_SIZE` are enqueued per throw (009 R-10).
|
|
328
|
+
batch_size = effective_batch_size
|
|
229
329
|
|
|
230
330
|
RubyReactor::Map::Dispatcher.perform(
|
|
231
331
|
map_id: map_id,
|
|
@@ -237,18 +337,26 @@ module RubyReactor
|
|
|
237
337
|
argument_mappings: inputs.argument_mappings,
|
|
238
338
|
strict_ordering: inputs.strict_ordering,
|
|
239
339
|
mapped_reactor_class: inputs.mapped_reactor_class,
|
|
240
|
-
|
|
340
|
+
atomic: atomic?
|
|
241
341
|
)
|
|
242
342
|
queue_collector(map_id, step_name, inputs.strict_ordering)
|
|
243
343
|
"map:#{map_id}"
|
|
244
344
|
end
|
|
245
345
|
|
|
346
|
+
# Stores the context the Dispatcher reads the source from, and — for a
|
|
347
|
+
# map inside a composed child — the owner's tree, which the map's
|
|
348
|
+
# completion resumes (009 R-03): a fast resume must find it consistent,
|
|
349
|
+
# as `handle_background_handoff` ensures for a `background` hand-off.
|
|
246
350
|
def prepare_async_execution(map_id, count)
|
|
247
351
|
storage = RubyReactor.configuration.storage_adapter
|
|
248
352
|
middlewares = context.middlewares || Executor.middlewares_for(context.reactor_class)
|
|
249
353
|
middlewares.on(:before_async_enqueue, context)
|
|
250
354
|
serialized_context = ContextSerializer.serialize(context)
|
|
251
355
|
storage.store_context(context.context_id, serialized_context, context.reactor_class.name)
|
|
356
|
+
unless owner_context.equal?(context)
|
|
357
|
+
storage.store_context(owner_context.context_id, ContextSerializer.serialize(owner_context),
|
|
358
|
+
RubyReactor.reactor_storage_name(owner_context.reactor_class))
|
|
359
|
+
end
|
|
252
360
|
storage.set_map_counter(map_id, count, context.reactor_class.name)
|
|
253
361
|
end
|
|
254
362
|
|
data/lib/ruby_reactor/step.rb
CHANGED
|
@@ -113,6 +113,13 @@ module RubyReactor
|
|
|
113
113
|
catch(StepSignals::TAG) { new(with_defaults(arguments), context, reason: reason).compensate }
|
|
114
114
|
end
|
|
115
115
|
|
|
116
|
+
# Whether `undo` also reverts a run cut short by an interruption. A
|
|
117
|
+
# plain step's body stopped mid-way, so no: only a construct whose undo
|
|
118
|
+
# replays the child work it completed (compose, map) says yes (008 R-16).
|
|
119
|
+
def undoes_partial_run?
|
|
120
|
+
false
|
|
121
|
+
end
|
|
122
|
+
|
|
116
123
|
def input(...)
|
|
117
124
|
own_input_contract.input(...)
|
|
118
125
|
@input_contract = nil
|