ruby_reactor 0.8.5 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/.specify/feature.json +1 -1
- data/.specify/memory/constitution.md +17 -14
- data/CHANGELOG.md +82 -0
- data/CLAUDE.md +1 -1
- data/README.md +27 -8
- data/lib/ruby_reactor/adapters/active_job/map_element_rollback_worker.rb +20 -0
- data/lib/ruby_reactor/adapters/active_job/router.rb +32 -4
- data/lib/ruby_reactor/adapters/sidekiq/map_element_rollback_worker.rb +16 -0
- data/lib/ruby_reactor/adapters/sidekiq/router.rb +32 -4
- data/lib/ruby_reactor/context.rb +14 -1
- data/lib/ruby_reactor/dsl/definition_warnings.rb +30 -0
- data/lib/ruby_reactor/dsl/map_builder.rb +27 -7
- data/lib/ruby_reactor/dsl/step_builder.rb +8 -21
- data/lib/ruby_reactor/error/rescuable.rb +6 -1
- data/lib/ruby_reactor/error/rollback_handed_off.rb +27 -0
- data/lib/ruby_reactor/executor/compensation_manager.rb +39 -9
- data/lib/ruby_reactor/executor/result_handler.rb +88 -52
- data/lib/ruby_reactor/executor/retry_manager.rb +6 -5
- data/lib/ruby_reactor/executor/step_executor.rb +23 -1
- data/lib/ruby_reactor/executor.rb +186 -6
- data/lib/ruby_reactor/map/collector.rb +25 -93
- data/lib/ruby_reactor/map/dispatcher.rb +38 -7
- data/lib/ruby_reactor/map/element_executor.rb +14 -10
- data/lib/ruby_reactor/map/element_rollback.rb +158 -0
- data/lib/ruby_reactor/map/helpers.rb +11 -155
- data/lib/ruby_reactor/map/step_rollback.rb +136 -0
- data/lib/ruby_reactor/map/sweeper.rb +66 -8
- data/lib/ruby_reactor/map.rb +17 -0
- data/lib/ruby_reactor/max_retries_exhausted_failure.rb +4 -2
- data/lib/ruby_reactor/reactor.rb +113 -24
- data/lib/ruby_reactor/rspec/active_job_helpers.rb +13 -2
- data/lib/ruby_reactor/rspec/matchers.rb +55 -37
- data/lib/ruby_reactor/rspec/sidekiq_helpers.rb +7 -3
- data/lib/ruby_reactor/rspec/test_subject.rb +21 -21
- data/lib/ruby_reactor/rspec.rb +3 -6
- data/lib/ruby_reactor/step/compose_step.rb +13 -0
- data/lib/ruby_reactor/step/map_step.rb +117 -114
- data/lib/ruby_reactor/storage/adapter.rb +14 -0
- data/lib/ruby_reactor/storage/redis_adapter.rb +18 -2
- data/lib/ruby_reactor/storage/redis_map_rollback.rb +153 -0
- data/lib/ruby_reactor/storage/redis_reactor_scan.rb +1 -1
- data/lib/ruby_reactor/sweeper.rb +3 -1
- data/lib/ruby_reactor/version.rb +1 -1
- data/lib/ruby_reactor/web/api.rb +11 -1
- data/lib/ruby_reactor/web/public/assets/{index-CQbgHtd0.js → index-C6ejszCQ.js} +12 -12
- data/lib/ruby_reactor/web/public/assets/index-DM5darhy.css +1 -0
- data/lib/ruby_reactor/web/public/index.html +2 -2
- data/lib/ruby_reactor/worker.rb +4 -2
- data/specs/007-execution-flow-analysis/evidence/harness.rb +2 -0
- data/specs/007-execution-flow-analysis/evidence/probes/03_map.rb +4 -1
- data/specs/009-distributed-map-undo/checklists/requirements.md +53 -0
- data/specs/009-distributed-map-undo/contracts/api-surface.md +125 -0
- data/specs/009-distributed-map-undo/contracts/rollback-protocol.md +157 -0
- data/specs/009-distributed-map-undo/data-model.md +151 -0
- data/specs/009-distributed-map-undo/plan.md +323 -0
- data/specs/009-distributed-map-undo/pr-description.md +78 -0
- data/specs/009-distributed-map-undo/quickstart.md +154 -0
- data/specs/009-distributed-map-undo/research.md +585 -0
- data/specs/009-distributed-map-undo/spec.md +360 -0
- data/specs/009-distributed-map-undo/tasks.md +1060 -0
- data/specs/010-interrupt-in-composed-child/checklists/requirements.md +43 -0
- data/specs/010-interrupt-in-composed-child/contracts/api-surface.md +75 -0
- data/specs/010-interrupt-in-composed-child/data-model.md +40 -0
- data/specs/010-interrupt-in-composed-child/plan.md +149 -0
- data/specs/010-interrupt-in-composed-child/quickstart.md +64 -0
- data/specs/010-interrupt-in-composed-child/research.md +207 -0
- data/specs/010-interrupt-in-composed-child/spec.md +267 -0
- data/specs/010-interrupt-in-composed-child/tasks.md +315 -0
- data/specs/future_improvements.md +9 -33
- metadata +29 -3
- data/lib/ruby_reactor/web/public/assets/index-BQvIWPdx.css +0 -1
data/lib/ruby_reactor/reactor.rb
CHANGED
|
@@ -6,6 +6,9 @@ module RubyReactor
|
|
|
6
6
|
include RubyReactor::Dsl::Reactor
|
|
7
7
|
include RubyReactor::Dsl::Lockable
|
|
8
8
|
|
|
9
|
+
# Seconds a manual undo waits for the run's context lock (009 R-12).
|
|
10
|
+
UNDO_LOCK_WAIT = 5
|
|
11
|
+
|
|
9
12
|
attr_reader :context, :result, :undo_trace, :execution_trace
|
|
10
13
|
|
|
11
14
|
def self.find(id)
|
|
@@ -35,6 +38,14 @@ module RubyReactor
|
|
|
35
38
|
find(context_id)
|
|
36
39
|
end
|
|
37
40
|
|
|
41
|
+
# How a resume names an interrupt: a Symbol for one of the reactor's own,
|
|
42
|
+
# or its step path from the root (an Array: the compose step names, then
|
|
43
|
+
# the interrupt) for one inside a composed child (010).
|
|
44
|
+
def self.interrupt_key(step_name)
|
|
45
|
+
path = Array(step_name).map(&:to_sym)
|
|
46
|
+
path.one? ? path.first : path
|
|
47
|
+
end
|
|
48
|
+
|
|
38
49
|
def self.continue(id:, payload:, step_name:, idempotency_key: nil)
|
|
39
50
|
reactor = find(id)
|
|
40
51
|
result = reactor.continue(payload: payload, step_name: step_name, idempotency_key: idempotency_key)
|
|
@@ -60,9 +71,12 @@ module RubyReactor
|
|
|
60
71
|
reactor.cancel(reason)
|
|
61
72
|
end
|
|
62
73
|
|
|
74
|
+
# A rollback that hands off at a fan-out map finishes in a worker, which
|
|
75
|
+
# applies `cancelled` itself (009 R-12).
|
|
63
76
|
def self.undo(id)
|
|
64
77
|
reactor = find(id)
|
|
65
|
-
reactor.undo
|
|
78
|
+
return if reactor.undo == :handed_off
|
|
79
|
+
|
|
66
80
|
cancel(id: id, reason: "Undo triggered")
|
|
67
81
|
end
|
|
68
82
|
|
|
@@ -148,6 +162,11 @@ module RubyReactor
|
|
|
148
162
|
raise Error::ValidationError, "Cannot resume: context does not have a current step (was it interrupted?)"
|
|
149
163
|
end
|
|
150
164
|
|
|
165
|
+
# A composed child pauses only inside its root, which owns the run (010 R-07).
|
|
166
|
+
if @context.private_data[:composed] || @context.private_data["composed"]
|
|
167
|
+
raise Error::ValidationError, "Cannot resume: #{self.class.name} is a composed child; continue its root run"
|
|
168
|
+
end
|
|
169
|
+
|
|
151
170
|
if @context.cancelled
|
|
152
171
|
raise Error::ValidationError,
|
|
153
172
|
"Cannot resume: reactor has been cancelled (Reason: #{@context.cancellation_reason})"
|
|
@@ -161,19 +180,17 @@ module RubyReactor
|
|
|
161
180
|
"Cannot resume: the reactor is #{@context.status}, not paused at an interrupt"
|
|
162
181
|
end
|
|
163
182
|
|
|
164
|
-
|
|
183
|
+
path = Array(self.class.interrupt_key(step_name))
|
|
184
|
+
target_context, step_config = resolve_interrupt_target!(path)
|
|
165
185
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
end
|
|
186
|
+
failure = validate_continue_payload(payload, step_config, path)
|
|
187
|
+
return failure if failure
|
|
169
188
|
|
|
170
|
-
|
|
171
|
-
@context.set_result(target_step, payload)
|
|
189
|
+
target_context.set_result(path.last, payload) # the context that paused (010 R-06)
|
|
172
190
|
|
|
173
191
|
# `interrupt :x, resume: :background` — payload is validated and stored
|
|
174
192
|
# (above, in this process); the remaining work goes to a worker instead
|
|
175
193
|
# of running inline in the delivering process.
|
|
176
|
-
step_config = self.class.steps[step_name.to_sym]
|
|
177
194
|
if step_config.respond_to?(:background_resume?) && step_config.background_resume?
|
|
178
195
|
return @result = enqueue_background_resume
|
|
179
196
|
end
|
|
@@ -204,19 +221,49 @@ module RubyReactor
|
|
|
204
221
|
RubyReactor::Failure(e.message, invalid_payload: true)
|
|
205
222
|
end
|
|
206
223
|
|
|
224
|
+
# Undoes every completed step, under the run's context lock so no worker
|
|
225
|
+
# resumes it meanwhile (009 R-12). Returns `:handed_off` when a fan-out
|
|
226
|
+
# map's rollback continues in its element jobs: the run is then
|
|
227
|
+
# `rolling_back`, and a worker finishes it as `cancelled`.
|
|
207
228
|
def undo
|
|
229
|
+
raise Error::ValidationError, "rollback already in progress" if @context.rolling_back?
|
|
230
|
+
|
|
231
|
+
lock = acquire_undo_lock
|
|
208
232
|
executor = Executor.new(self.class, {}, @context)
|
|
209
|
-
|
|
233
|
+
@context.rollback = { "trigger" => "undo", "compensated" => true, "failures" => [] }
|
|
234
|
+
begin
|
|
235
|
+
executor.undo_all
|
|
236
|
+
rescue Error::RollbackHandedOff => e
|
|
237
|
+
@context.status = :rolling_back
|
|
238
|
+
executor.save_context
|
|
239
|
+
executor.hand_off_rollback!(e)
|
|
240
|
+
return :handed_off
|
|
241
|
+
end
|
|
242
|
+
@context.rollback = nil
|
|
210
243
|
executor.save_context
|
|
244
|
+
ensure
|
|
245
|
+
lock&.release
|
|
211
246
|
end
|
|
212
247
|
|
|
213
248
|
def cancel(reason)
|
|
249
|
+
# `cancelled` is terminal: the steps before the map would never be undone.
|
|
250
|
+
raise Error::ValidationError, "rollback in progress; cannot cancel" if @context.rolling_back?
|
|
251
|
+
|
|
214
252
|
@context.cancelled = true
|
|
215
253
|
@context.cancellation_reason = reason
|
|
216
254
|
@context.status = "cancelled"
|
|
217
255
|
save_context
|
|
218
256
|
end
|
|
219
257
|
|
|
258
|
+
# The interrupts this paused run can be resumed at (010 R-05): a Symbol
|
|
259
|
+
# for one of this reactor's own, and the step path from here (the compose
|
|
260
|
+
# step names, then the interrupt) for one inside a composed child.
|
|
261
|
+
def ready_interrupt_steps
|
|
262
|
+
return [] unless @context.status.to_s == "paused"
|
|
263
|
+
|
|
264
|
+
pending_interrupts(self.class, @context, [])
|
|
265
|
+
end
|
|
266
|
+
|
|
220
267
|
def validate!
|
|
221
268
|
# Validate reactor configuration
|
|
222
269
|
validate_steps!
|
|
@@ -235,6 +282,18 @@ module RubyReactor
|
|
|
235
282
|
save_context
|
|
236
283
|
end
|
|
237
284
|
|
|
285
|
+
# Raises `Lock::AcquisitionError` while a live run or rollback holds it.
|
|
286
|
+
# Skipped in inline job-testing mode, as `Executor#acquire_context_lock`.
|
|
287
|
+
def acquire_undo_lock
|
|
288
|
+
return if defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
|
|
289
|
+
|
|
290
|
+
lock = RubyReactor::Lock.new("async:#{@context.context_id}", owner: SecureRandom.uuid,
|
|
291
|
+
ttl: configuration.context_lock_ttl,
|
|
292
|
+
wait: UNDO_LOCK_WAIT)
|
|
293
|
+
lock.acquire
|
|
294
|
+
lock
|
|
295
|
+
end
|
|
296
|
+
|
|
238
297
|
def validate_steps!
|
|
239
298
|
return unless self.class.steps.empty?
|
|
240
299
|
|
|
@@ -395,39 +454,69 @@ module RubyReactor
|
|
|
395
454
|
@undo_trace = executor.undo_trace
|
|
396
455
|
end
|
|
397
456
|
|
|
398
|
-
|
|
399
|
-
|
|
457
|
+
# The context that paused at the interrupt `path` names, and that
|
|
458
|
+
# interrupt's config. A one-name path is a step of this reactor; a longer
|
|
459
|
+
# one names compose steps from here down, then the interrupt (010 R-05).
|
|
460
|
+
def resolve_interrupt_target!(path)
|
|
461
|
+
pending = ready_interrupt_steps
|
|
462
|
+
key = path.one? ? path.first : path
|
|
463
|
+
unless pending.include?(key)
|
|
464
|
+
raise Error::ValidationError,
|
|
465
|
+
"Cannot resume: expected step '#{@context.current_step}' " \
|
|
466
|
+
"or ready steps #{pending.inspect} but got '#{key}'"
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
target = path[0...-1].reduce(@context) { |context, name| paused_child(context, name) }
|
|
470
|
+
[target, target.reactor_class.steps[path.last]]
|
|
471
|
+
end
|
|
400
472
|
|
|
401
|
-
|
|
402
|
-
graph_manager = Executor::GraphManager.new(
|
|
473
|
+
def pending_interrupts(reactor_class, context, prefix)
|
|
474
|
+
graph_manager = Executor::GraphManager.new(reactor_class, DependencyGraph.new, context)
|
|
403
475
|
graph_manager.build_and_validate!
|
|
404
476
|
graph_manager.mark_completed_steps_from_context
|
|
405
|
-
ready_steps = graph_manager.dependency_graph.ready_steps.map(&:name).map(&:to_s)
|
|
406
477
|
|
|
407
|
-
|
|
478
|
+
# The interrupt a context paused at stays pending even once a resume
|
|
479
|
+
# stored its payload: that resume was contended and never ran.
|
|
480
|
+
ready = graph_manager.dependency_graph.ready_steps
|
|
481
|
+
paused_at = reactor_class.steps[context.current_step&.to_sym]
|
|
482
|
+
ready += [paused_at] if paused_at&.interrupt? && ready.none? { |step| step.name == paused_at.name }
|
|
483
|
+
|
|
484
|
+
ready.flat_map do |step_config|
|
|
485
|
+
name = step_config.name.to_sym
|
|
486
|
+
if step_config.respond_to?(:interrupt?) && step_config.interrupt?
|
|
487
|
+
[prefix.empty? ? name : prefix + [name]]
|
|
488
|
+
elsif (child = paused_child(context, name))
|
|
489
|
+
pending_interrupts(child.reactor_class, child, prefix + [name])
|
|
490
|
+
else
|
|
491
|
+
[]
|
|
492
|
+
end
|
|
493
|
+
end
|
|
494
|
+
end
|
|
408
495
|
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
496
|
+
def paused_child(context, step_name)
|
|
497
|
+
entry = context.composed_contexts[step_name] || context.composed_contexts[step_name.to_s]
|
|
498
|
+
child = entry.is_a?(Hash) && (entry[:context] || entry["context"])
|
|
499
|
+
child if child.is_a?(Context) && child.status.to_s == "paused"
|
|
412
500
|
end
|
|
413
501
|
|
|
414
|
-
def validate_continue_payload(payload,
|
|
415
|
-
step_config = self.class.steps[step_name.to_sym]
|
|
502
|
+
def validate_continue_payload(payload, step_config, path)
|
|
416
503
|
return unless step_config&.validation_schema
|
|
417
504
|
|
|
505
|
+
# A nested interrupt counts apart from a root step of the same name.
|
|
506
|
+
step_name = path.one? ? path.first : path.join(".").to_sym
|
|
507
|
+
|
|
418
508
|
validation = step_config.validation_schema.call(payload)
|
|
419
509
|
|
|
420
510
|
return unless validation.failure?
|
|
421
511
|
|
|
422
512
|
# Track attempts
|
|
423
|
-
step_key = step_name.to_sym
|
|
424
513
|
@context.private_data[:interrupt_attempts] ||= {}
|
|
425
|
-
@context.private_data[:interrupt_attempts][
|
|
426
|
-
@context.private_data[:interrupt_attempts][
|
|
514
|
+
@context.private_data[:interrupt_attempts][step_name] ||= 0
|
|
515
|
+
@context.private_data[:interrupt_attempts][step_name] += 1
|
|
427
516
|
|
|
428
517
|
save_context # Persist the attempt count
|
|
429
518
|
|
|
430
|
-
current_attempts = @context.private_data[:interrupt_attempts][
|
|
519
|
+
current_attempts = @context.private_data[:interrupt_attempts][step_name]
|
|
431
520
|
max_attempts = step_config.max_attempts
|
|
432
521
|
|
|
433
522
|
if max_attempts != :infinity && current_attempts >= max_attempts
|
|
@@ -11,12 +11,16 @@ module RubyReactor
|
|
|
11
11
|
PendingJob = Struct.new(:job_class, :raw) do
|
|
12
12
|
def perform!
|
|
13
13
|
::ActiveJob::Base.queue_adapter.enqueued_jobs.delete(raw)
|
|
14
|
-
job_class.new(*raw
|
|
14
|
+
job_class.new(*ActiveJobHelpers.arguments(raw)).perform_now
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def args
|
|
18
18
|
raw[:args]
|
|
19
19
|
end
|
|
20
|
+
|
|
21
|
+
# The Sidekiq `PendingJob`'s name for it, so step-wise drains read the
|
|
22
|
+
# same on both backends (009 U4).
|
|
23
|
+
alias_method :worker_class, :job_class
|
|
20
24
|
end
|
|
21
25
|
|
|
22
26
|
def self.test_adapter?
|
|
@@ -37,11 +41,18 @@ module RubyReactor
|
|
|
37
41
|
|
|
38
42
|
queue.dup.each do |job|
|
|
39
43
|
queue.delete(job)
|
|
40
|
-
job[:job].new(*job
|
|
44
|
+
job[:job].new(*arguments(job)).perform_now
|
|
41
45
|
end
|
|
42
46
|
end
|
|
43
47
|
end
|
|
44
48
|
|
|
49
|
+
# The test adapter keeps a job's arguments serialized; a real backend
|
|
50
|
+
# hands `perform` the deserialized ones (a job requeueing its own
|
|
51
|
+
# arguments would otherwise re-serialize ActiveJob's markers).
|
|
52
|
+
def self.arguments(raw)
|
|
53
|
+
::ActiveJob::Arguments.deserialize(raw[:args])
|
|
54
|
+
end
|
|
55
|
+
|
|
45
56
|
def self.pending_async_jobs
|
|
46
57
|
return [] unless test_adapter?
|
|
47
58
|
|
|
@@ -4,8 +4,10 @@ module RubyReactor
|
|
|
4
4
|
module RSpec
|
|
5
5
|
# rubocop:disable Metrics/ModuleLength
|
|
6
6
|
module Matchers
|
|
7
|
+
extend ::RSpec::Matchers::DSL
|
|
8
|
+
|
|
7
9
|
# rubocop:disable Metrics/BlockLength
|
|
8
|
-
|
|
10
|
+
matcher :be_success do
|
|
9
11
|
match do |subject|
|
|
10
12
|
subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
|
|
11
13
|
subject.success?
|
|
@@ -63,7 +65,7 @@ module RubyReactor
|
|
|
63
65
|
end
|
|
64
66
|
end
|
|
65
67
|
|
|
66
|
-
|
|
68
|
+
matcher :be_failure do
|
|
67
69
|
match do |subject|
|
|
68
70
|
subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
|
|
69
71
|
subject.failure?
|
|
@@ -74,7 +76,7 @@ module RubyReactor
|
|
|
74
76
|
end
|
|
75
77
|
end
|
|
76
78
|
|
|
77
|
-
|
|
79
|
+
matcher :have_run_step do |step_name|
|
|
78
80
|
match do |subject|
|
|
79
81
|
subject.ensure_executed!
|
|
80
82
|
@trace = subject.reactor_instance.context.execution_trace
|
|
@@ -127,7 +129,7 @@ module RubyReactor
|
|
|
127
129
|
end
|
|
128
130
|
end
|
|
129
131
|
|
|
130
|
-
|
|
132
|
+
matcher :have_retried_step do |step_name|
|
|
131
133
|
match do |subject|
|
|
132
134
|
subject.ensure_executed!
|
|
133
135
|
attempts = subject.reactor_instance.context.retry_context.attempts_for_step(step_name)
|
|
@@ -151,7 +153,7 @@ module RubyReactor
|
|
|
151
153
|
end
|
|
152
154
|
end
|
|
153
155
|
|
|
154
|
-
|
|
156
|
+
matcher :have_validation_error do |field|
|
|
155
157
|
match do |subject|
|
|
156
158
|
subject.ensure_executed!
|
|
157
159
|
return false unless subject.failure?
|
|
@@ -173,7 +175,7 @@ module RubyReactor
|
|
|
173
175
|
end
|
|
174
176
|
|
|
175
177
|
# Matcher to check if reactor is paused at an interrupt
|
|
176
|
-
|
|
178
|
+
matcher :be_paused do
|
|
177
179
|
match do |subject|
|
|
178
180
|
subject.ensure_executed!
|
|
179
181
|
subject.paused?
|
|
@@ -188,53 +190,69 @@ module RubyReactor
|
|
|
188
190
|
end
|
|
189
191
|
end
|
|
190
192
|
|
|
193
|
+
# A run whose rollback handed off at a fan-out map and has not finished
|
|
194
|
+
# (009): its element rollback jobs are still pending.
|
|
195
|
+
matcher :be_rolling_back do
|
|
196
|
+
match do |subject|
|
|
197
|
+
subject.ensure_executed!
|
|
198
|
+
subject.rolling_back?
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
failure_message do |subject|
|
|
202
|
+
"expected reactor to be rolling back, but status was #{subject.reactor_instance.context.status}"
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
failure_message_when_negated do |_subject|
|
|
206
|
+
"expected reactor not to be rolling back, but it is"
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
191
210
|
# Matcher to check if reactor is paused at a specific interrupt step
|
|
192
211
|
# Works with both single and multiple concurrent interrupts
|
|
193
|
-
|
|
212
|
+
matcher :be_paused_at do |*step_names|
|
|
213
|
+
# A step inside a composed child is named by its path, e.g. `[:fulfil, :approve]`.
|
|
214
|
+
expected = -> { step_names.map { |name| RubyReactor::Reactor.interrupt_key(name).inspect }.join(", ") }
|
|
215
|
+
|
|
194
216
|
match do |subject|
|
|
195
217
|
subject.ensure_executed!
|
|
196
218
|
return false unless subject.paused?
|
|
197
219
|
|
|
198
220
|
ready_steps = subject.ready_interrupt_steps
|
|
199
|
-
step_names.all? { |name| ready_steps.include?(name
|
|
221
|
+
step_names.all? { |name| ready_steps.include?(RubyReactor::Reactor.interrupt_key(name)) }
|
|
200
222
|
end
|
|
201
223
|
|
|
202
224
|
failure_message do |subject|
|
|
203
225
|
if subject.paused?
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
"expected reactor to be paused at :#{step_names.first}, " \
|
|
207
|
-
"but ready interrupt steps are: #{ready_steps.inspect}"
|
|
208
|
-
else
|
|
209
|
-
"expected reactor to be paused at #{step_names.map { |s| ":#{s}" }.join(", ")}, " \
|
|
210
|
-
"but ready interrupt steps are: #{ready_steps.inspect}"
|
|
211
|
-
end
|
|
226
|
+
"expected reactor to be paused at #{expected.call}, " \
|
|
227
|
+
"but ready interrupt steps are: #{subject.ready_interrupt_steps.inspect}"
|
|
212
228
|
else
|
|
213
|
-
"expected reactor to be paused at #{
|
|
229
|
+
"expected reactor to be paused at #{expected.call}, " \
|
|
214
230
|
"but status was #{subject.reactor_instance.context.status}"
|
|
215
231
|
end
|
|
216
232
|
end
|
|
217
233
|
|
|
218
234
|
failure_message_when_negated do |_subject|
|
|
219
|
-
"expected reactor not to be paused at #{
|
|
235
|
+
"expected reactor not to be paused at #{expected.call}, but it is"
|
|
220
236
|
end
|
|
221
237
|
end
|
|
222
238
|
|
|
223
239
|
# Matcher to check the exact set of ready interrupt steps
|
|
224
|
-
|
|
240
|
+
matcher :have_ready_interrupts do |*expected_steps|
|
|
241
|
+
# A step inside a composed child is named by its path, e.g. `[:fulfil, :approve]`.
|
|
242
|
+
expected = -> { expected_steps.map { |name| RubyReactor::Reactor.interrupt_key(name) } }
|
|
243
|
+
|
|
225
244
|
match do |subject|
|
|
226
245
|
subject.ensure_executed!
|
|
227
246
|
return false unless subject.paused?
|
|
228
247
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
actual_steps == expected
|
|
248
|
+
# Symbols and paths (Arrays) do not compare: sort by their inspect.
|
|
249
|
+
subject.ready_interrupt_steps.sort_by(&:inspect) == expected.call.sort_by(&:inspect)
|
|
232
250
|
end
|
|
233
251
|
|
|
234
252
|
failure_message do |subject|
|
|
235
253
|
if subject.paused?
|
|
236
254
|
actual_steps = subject.ready_interrupt_steps
|
|
237
|
-
"expected ready interrupt steps to be #{
|
|
255
|
+
"expected ready interrupt steps to be #{expected.call}, " \
|
|
238
256
|
"but got #{actual_steps.inspect}"
|
|
239
257
|
else
|
|
240
258
|
"expected reactor to be paused with ready interrupt steps, " \
|
|
@@ -244,7 +262,7 @@ module RubyReactor
|
|
|
244
262
|
|
|
245
263
|
failure_message_when_negated do |subject|
|
|
246
264
|
actual_steps = subject.ready_interrupt_steps
|
|
247
|
-
"expected ready interrupt steps not to be #{
|
|
265
|
+
"expected ready interrupt steps not to be #{expected.call}, " \
|
|
248
266
|
"but it was #{actual_steps.inspect}"
|
|
249
267
|
end
|
|
250
268
|
end
|
|
@@ -277,7 +295,7 @@ module RubyReactor
|
|
|
277
295
|
# expect(result).to be_halted
|
|
278
296
|
# expect(result).to be_halted.because(:period)
|
|
279
297
|
# expect(result).to be_halted.at_step(:second)
|
|
280
|
-
|
|
298
|
+
matcher :be_halted do
|
|
281
299
|
match do |subject|
|
|
282
300
|
subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
|
|
283
301
|
actual = subject.respond_to?(:result) ? subject.result : subject
|
|
@@ -320,7 +338,7 @@ module RubyReactor
|
|
|
320
338
|
# Examples:
|
|
321
339
|
# expect(result).to be_skipped
|
|
322
340
|
# expect(subject).to be_skipped.at_step(:maybe_sync)
|
|
323
|
-
|
|
341
|
+
matcher :be_skipped do
|
|
324
342
|
match do |subject|
|
|
325
343
|
subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
|
|
326
344
|
actual = subject.respond_to?(:result) ? subject.result : subject
|
|
@@ -367,7 +385,7 @@ module RubyReactor
|
|
|
367
385
|
#
|
|
368
386
|
# expect("order:42").to be_locked
|
|
369
387
|
# expect("order:42").to be_locked.by("ctx-abc")
|
|
370
|
-
|
|
388
|
+
matcher :be_locked do
|
|
371
389
|
match do |key|
|
|
372
390
|
info = Matchers.coordination_adapter.lock_info("lock:#{key}")
|
|
373
391
|
next false unless info
|
|
@@ -400,7 +418,7 @@ module RubyReactor
|
|
|
400
418
|
# user-provided semaphore name (without the "semaphore:" prefix).
|
|
401
419
|
#
|
|
402
420
|
# expect("api_limit").to have_available_tokens(3)
|
|
403
|
-
|
|
421
|
+
matcher :have_available_tokens do |expected|
|
|
404
422
|
match do |name|
|
|
405
423
|
Matchers.coordination_adapter.semaphore_state(name)[:available] == expected
|
|
406
424
|
end
|
|
@@ -415,7 +433,7 @@ module RubyReactor
|
|
|
415
433
|
# Asserts the number of currently-checked-out semaphore tokens.
|
|
416
434
|
#
|
|
417
435
|
# expect("api_limit").to have_held_tokens(2)
|
|
418
|
-
|
|
436
|
+
matcher :have_held_tokens do |expected|
|
|
419
437
|
match do |name|
|
|
420
438
|
Matchers.coordination_adapter.semaphore_state(name)[:held] == expected
|
|
421
439
|
end
|
|
@@ -434,7 +452,7 @@ module RubyReactor
|
|
|
434
452
|
#
|
|
435
453
|
# expect("stripe:42").to have_rate_limit_count(3).for(:second)
|
|
436
454
|
# expect("stripe:42").to have_rate_limit_count(1).for(:minute).since(started_at)
|
|
437
|
-
|
|
455
|
+
matcher :have_rate_limit_count do |expected|
|
|
438
456
|
match do |key_base|
|
|
439
457
|
raise ArgumentError, "have_rate_limit_count requires .for(period)" unless @period
|
|
440
458
|
|
|
@@ -458,7 +476,7 @@ module RubyReactor
|
|
|
458
476
|
# Asserts that a `with_period` bucket has been marked. Use `.for(period)`.
|
|
459
477
|
#
|
|
460
478
|
# expect("daily_report:7").to be_period_marked.for(:day)
|
|
461
|
-
|
|
479
|
+
matcher :be_period_marked do
|
|
462
480
|
match do |key_base|
|
|
463
481
|
raise ArgumentError, "be_period_marked requires .for(period)" unless @period
|
|
464
482
|
|
|
@@ -482,7 +500,7 @@ module RubyReactor
|
|
|
482
500
|
# the user-provided ordered_lock key (without the `ordered_lock:` prefix).
|
|
483
501
|
#
|
|
484
502
|
# expect("orders:42").to have_ordered_lock_next(3)
|
|
485
|
-
|
|
503
|
+
matcher :have_ordered_lock_next do |expected|
|
|
486
504
|
match { |key| Matchers.coordination_adapter.ordered_lock_peek(key)[:next] == expected }
|
|
487
505
|
|
|
488
506
|
failure_message do |key|
|
|
@@ -495,7 +513,7 @@ module RubyReactor
|
|
|
495
513
|
# Asserts the last-advanced cursor for an ordered_lock key.
|
|
496
514
|
#
|
|
497
515
|
# expect("orders:42").to have_ordered_lock_last_completed(2)
|
|
498
|
-
|
|
516
|
+
matcher :have_ordered_lock_last_completed do |expected|
|
|
499
517
|
match do |key|
|
|
500
518
|
Matchers.coordination_adapter.ordered_lock_peek(key)[:last_completed] == expected
|
|
501
519
|
end
|
|
@@ -511,7 +529,7 @@ module RubyReactor
|
|
|
511
529
|
# Order-insensitive — the matcher sorts both sides.
|
|
512
530
|
#
|
|
513
531
|
# expect("orders:42").to have_ordered_lock_in_flight(2, 3)
|
|
514
|
-
|
|
532
|
+
matcher :have_ordered_lock_in_flight do |*expected|
|
|
515
533
|
match do |key|
|
|
516
534
|
actual = Matchers.coordination_adapter.ordered_lock_peek(key)[:in_flight].sort
|
|
517
535
|
actual == expected.flatten.map(&:to_i).sort
|
|
@@ -528,7 +546,7 @@ module RubyReactor
|
|
|
528
546
|
# in-flight nonces. After a clean drain `peek` returns all zeros.
|
|
529
547
|
#
|
|
530
548
|
# expect("orders:42").to be_ordered_lock_drained
|
|
531
|
-
|
|
549
|
+
matcher :be_ordered_lock_drained do
|
|
532
550
|
match do |key|
|
|
533
551
|
state = Matchers.coordination_adapter.ordered_lock_peek(key)
|
|
534
552
|
state[:next].zero? && state[:last_completed].zero? && state[:in_flight].empty?
|
|
@@ -551,7 +569,7 @@ module RubyReactor
|
|
|
551
569
|
#
|
|
552
570
|
# expect(result).to have_contended_at(:charge)
|
|
553
571
|
# expect(result).to have_contended_at(:charge).on("acct:1")
|
|
554
|
-
|
|
572
|
+
matcher :have_contended_at do |step_name|
|
|
555
573
|
match do |subject|
|
|
556
574
|
trace = execution_trace_for(subject)
|
|
557
575
|
@entry = trace.find do |t|
|
|
@@ -596,7 +614,7 @@ module RubyReactor
|
|
|
596
614
|
# expect(result).to have_rollback_failure(:charge)
|
|
597
615
|
# expect(subject).to have_rollback_failure(:charge).for_key("acct:1").because(:coordination_unavailable)
|
|
598
616
|
# expect(subject).not_to have_rollback_failure(:charge)
|
|
599
|
-
|
|
617
|
+
matcher :have_rollback_failure do |step_name|
|
|
600
618
|
match do |subject|
|
|
601
619
|
@entries = rollback_failures_for(subject)
|
|
602
620
|
@entries.any? do |e|
|
|
@@ -12,15 +12,18 @@ module RubyReactor
|
|
|
12
12
|
# until the queues are empty. Recursive — handles jobs that re-enqueue
|
|
13
13
|
# themselves (e.g. ordered_lock snoozes) and worker chains that queue
|
|
14
14
|
# additional jobs.
|
|
15
|
+
# Whichever test backend is active (Sidekiq fake mode or the ActiveJob
|
|
16
|
+
# `:test` adapter), via `AsyncTestHelpers`.
|
|
15
17
|
def drain_async_jobs(max_iterations: 100)
|
|
16
|
-
|
|
18
|
+
AsyncTestHelpers.drain_async_jobs(max_iterations: max_iterations)
|
|
17
19
|
end
|
|
18
20
|
|
|
19
21
|
# All currently-pending async jobs, wrapped in `PendingJob` so callers
|
|
20
22
|
# can perform individual jobs out-of-order (e.g. to assert
|
|
21
|
-
# ordered_lock's snooze behavior) without touching
|
|
23
|
+
# ordered_lock's snooze behavior) without touching backend internals.
|
|
24
|
+
# `worker_class` names the job's class on either backend.
|
|
22
25
|
def pending_async_jobs
|
|
23
|
-
|
|
26
|
+
AsyncTestHelpers.pending_async_jobs
|
|
24
27
|
end
|
|
25
28
|
|
|
26
29
|
PendingJob = Struct.new(:worker_class, :raw) do
|
|
@@ -39,6 +42,7 @@ module RubyReactor
|
|
|
39
42
|
RubyReactor::Adapters::Sidekiq::Worker,
|
|
40
43
|
RubyReactor::Adapters::Sidekiq::MapElementWorker,
|
|
41
44
|
RubyReactor::Adapters::Sidekiq::MapCollectorWorker,
|
|
45
|
+
RubyReactor::Adapters::Sidekiq::MapElementRollbackWorker,
|
|
42
46
|
RubyReactor::Adapters::Sidekiq::StepWorker
|
|
43
47
|
]
|
|
44
48
|
end
|
|
@@ -351,14 +351,14 @@ module RubyReactor
|
|
|
351
351
|
RubyReactor::Success.new(val)
|
|
352
352
|
when "halted", "skipped" # "skipped" is the legacy name for a halt
|
|
353
353
|
halted_result(ctx)
|
|
354
|
-
when "running"
|
|
354
|
+
when "running", "rolling_back"
|
|
355
355
|
# Try to determine if it is truly running or if we just missed the completion
|
|
356
356
|
if @process_jobs && AsyncTestHelpers.active?
|
|
357
357
|
# Force one more check
|
|
358
358
|
process_pending_jobs
|
|
359
359
|
# Reload status
|
|
360
360
|
@reactor_instance = @reactor_class.find(@reactor_instance.context.context_id)
|
|
361
|
-
return result unless @reactor_instance.context.status.to_s
|
|
361
|
+
return result unless %w[running rolling_back].include?(@reactor_instance.context.status.to_s)
|
|
362
362
|
end
|
|
363
363
|
|
|
364
364
|
# If still running, return a Pending/Running result instead of nil
|
|
@@ -405,6 +405,13 @@ module RubyReactor
|
|
|
405
405
|
@reactor_instance.context.status.to_s == "paused"
|
|
406
406
|
end
|
|
407
407
|
|
|
408
|
+
# Whether the run's rollback handed off at a fan-out map and has not
|
|
409
|
+
# finished (009). Reloads, so it reflects jobs performed since.
|
|
410
|
+
def rolling_back?
|
|
411
|
+
ensure_executed!
|
|
412
|
+
@reactor_instance.context.status.to_s == "rolling_back"
|
|
413
|
+
end
|
|
414
|
+
|
|
408
415
|
# Get the current step where the reactor is paused (interrupt step)
|
|
409
416
|
# Note: When multiple interrupts are ready, this returns just one of them.
|
|
410
417
|
# Use `ready_interrupt_steps` to get all ready interrupt steps.
|
|
@@ -419,29 +426,19 @@ module RubyReactor
|
|
|
419
426
|
# Get all ready interrupt steps (steps that can be resumed)
|
|
420
427
|
# This is useful when multiple interrupts are waiting concurrently.
|
|
421
428
|
#
|
|
422
|
-
# @return [Array<Symbol
|
|
429
|
+
# @return [Array<Symbol, Array<Symbol>>] the reactor's own interrupts by
|
|
430
|
+
# name, and one inside a composed child by its step path
|
|
431
|
+
# (e.g. `[:fulfil, :approve]`)
|
|
423
432
|
def ready_interrupt_steps
|
|
424
433
|
ensure_executed!
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
# Build the dependency graph and get ready steps
|
|
428
|
-
graph = RubyReactor::DependencyGraph.new
|
|
429
|
-
graph_manager = RubyReactor::Executor::GraphManager.new(
|
|
430
|
-
@reactor_class, graph, @reactor_instance.context
|
|
431
|
-
)
|
|
432
|
-
graph_manager.build_and_validate!
|
|
433
|
-
graph_manager.mark_completed_steps_from_context
|
|
434
|
-
|
|
435
|
-
# Filter to only interrupt steps (using interrupt? predicate method)
|
|
436
|
-
ready = graph_manager.dependency_graph.ready_steps
|
|
437
|
-
ready.select { |step_config| step_config.respond_to?(:interrupt?) && step_config.interrupt? }
|
|
438
|
-
.map { |step_config| step_config.name.to_sym }
|
|
434
|
+
@reactor_instance.ready_interrupt_steps
|
|
439
435
|
end
|
|
440
436
|
|
|
441
437
|
# Resume a paused reactor with the given payload
|
|
442
438
|
#
|
|
443
439
|
# @param payload [Hash] The data to provide to the interrupt step
|
|
444
|
-
# @param step [Symbol, String, nil] The specific interrupt step to resume
|
|
440
|
+
# @param step [Symbol, String, Array, nil] The specific interrupt step to resume,
|
|
441
|
+
# or the step path of one inside a composed child (e.g. `[:fulfil, :approve]`).
|
|
445
442
|
# Required when multiple interrupts are ready. If not provided and only
|
|
446
443
|
# one interrupt is ready, that step will be used.
|
|
447
444
|
# @return [TestSubject] self for chaining and introspection
|
|
@@ -476,10 +473,10 @@ module RubyReactor
|
|
|
476
473
|
|
|
477
474
|
if step
|
|
478
475
|
# User explicitly specified a step
|
|
479
|
-
step_sym = step
|
|
476
|
+
step_sym = RubyReactor::Reactor.interrupt_key(step)
|
|
480
477
|
unless ready_steps.include?(step_sym)
|
|
481
478
|
raise RubyReactor::Error::ValidationError,
|
|
482
|
-
"Cannot resume: step
|
|
479
|
+
"Cannot resume: step #{step_sym.inspect} is not ready. Ready steps: #{ready_steps.inspect}"
|
|
483
480
|
end
|
|
484
481
|
step_sym
|
|
485
482
|
elsif ready_steps.size == 1
|
|
@@ -523,9 +520,12 @@ module RubyReactor
|
|
|
523
520
|
|
|
524
521
|
def ensure_executed!
|
|
525
522
|
run unless @executed
|
|
523
|
+
return unless %w[running rolling_back].include?(@reactor_instance.context.status.to_s)
|
|
526
524
|
|
|
525
|
+
# Not finished: jobs the spec performed itself may have moved it on.
|
|
526
|
+
@reactor_instance = @reactor_class.find(@reactor_instance.context.context_id)
|
|
527
527
|
# Process jobs if status is running and processing is enabled
|
|
528
|
-
return unless @process_jobs && @reactor_instance.context.status.to_s
|
|
528
|
+
return unless @process_jobs && %w[running rolling_back].include?(@reactor_instance.context.status.to_s)
|
|
529
529
|
|
|
530
530
|
process_pending_jobs
|
|
531
531
|
end
|
data/lib/ruby_reactor/rspec.rb
CHANGED
|
@@ -21,15 +21,12 @@ module RubyReactor
|
|
|
21
21
|
def self.configure(config)
|
|
22
22
|
require_relative "rspec/step_executor_patch"
|
|
23
23
|
|
|
24
|
-
#
|
|
24
|
+
# Helpers, matchers and async-job helpers are only auto-included into
|
|
25
25
|
# examples tagged `type: :reactor`, so a host app's other specs keep a
|
|
26
26
|
# clean namespace. Specs that want them without the tag can
|
|
27
|
-
# `include RubyReactor::RSpec::Helpers` explicitly.
|
|
28
|
-
#
|
|
29
|
-
# Matchers stay unconditional: `::RSpec::Matchers.define` registers them
|
|
30
|
-
# globally at load time, so gating this include would not scope anything.
|
|
27
|
+
# `include RubyReactor::RSpec::Helpers` / `Matchers` explicitly.
|
|
31
28
|
config.include RubyReactor::RSpec::Helpers, REACTOR_METADATA
|
|
32
|
-
config.include RubyReactor::RSpec::Matchers
|
|
29
|
+
config.include RubyReactor::RSpec::Matchers, REACTOR_METADATA
|
|
33
30
|
config.include RubyReactor::RSpec::SidekiqHelpers, REACTOR_METADATA
|
|
34
31
|
|
|
35
32
|
::RubyReactor::Executor::StepExecutor.prepend(RubyReactor::RSpec::StepExecutorPatch)
|