ruby_reactor 0.8.5 → 0.8.6

data/lib/ruby_reactor/map/step_rollback.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    # How a map step rolls back (009 R-01): a fan-out map distributed, one job
+    # per started element (S-2); an inline map in process, in bounded reads
+    # (S-4). Both report the same failures (I-7). Mixed into `Step::MapStep`.
+    module StepRollback
+      private
+
+      # S-2: start the map's rollback once (its records, then the first
+      # throw), then either aggregate a settled rollback or hand the run off
+      # until the element jobs report (R-04, R-05). Re-entered by the owner's
+      # resume, it finds the records and only checks for the settle.
+      def distributed_rollback(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        created, rollback = storage.start_map_rollback(map_id, map_class, **rollback_metadata(map_id, step_name))
+        if created
+          log_rollback("started", step_name, total: rollback["total"], batch_size: rollback["batch_size"])
+          RubyReactor::Map::Dispatcher.dispatch_rollback_batch(map_id: map_id, parent_reactor_class_name: map_class)
+        end
+        total = rollback["total"].to_i
+        if storage.count_map_rollback_outcomes(map_id, map_class) >= total
+          return finish_rollback(step_name, total, distributed_failures(map_id, step_name))
+        end
+
+        raise RubyReactor::Error::RollbackHandedOff.new(map_id: map_id, reactor_class_name: map_class)
+      end
+
+      def rollback_metadata(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        forward = storage.retrieve_map_metadata(map_id, map_class) || {}
+        {
+          total: storage.count_map_element_context_ids(map_id, map_class),
+          batch_size: forward["batch_size"] || RubyReactor::Map::DEFAULT_BATCH_SIZE,
+          step_name: step_name.to_s,
+          owner_context_id: owner_context.context_id,
+          owner_reactor_class_name: RubyReactor.reactor_storage_name(owner_context.reactor_class),
+          reactor_class_info: forward["reactor_class_info"] || build_reactor_class_info(element_class, step_name)
+        }
+      end
+
+      # The failures the element jobs reported, in position order (newest
+      # first, as inline), read in bounded chunks (R-09).
+      def distributed_failures(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        reported = []
+        unnamed = 0
+        storage.each_map_rollback_outcome(map_id, map_class) do |position, outcome|
+          unnamed += 1 if outcome["outcome"] == "context_unavailable" && outcome["index"].nil?
+          next unless %w[failed element_in_flight].include?(outcome["outcome"])
+
+          reported << [position, ContextSerializer.deserialize_value(outcome["failures"])]
+        end
+        unseen = lambda do |indexes|
+          indexes.each_slice(1000).flat_map do |slice|
+            slice.zip(storage.map_rollback_indexes_seen(map_id, map_class, slice)).reject(&:last).map(&:first)
+          end
+        end
+        unavailable_entries(step_name, unnamed, unseen) + reported.sort_by(&:first).flat_map(&:last)
+      end
+
+      # S-4: in process, `Map::ROLLBACK_CHUNK` elements per read from the tail
+      # of the element index, never all at once.
+      def inline_rollback(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        total = storage.count_map_element_context_ids(map_id, map_class)
+        reported = []
+        seen = Set.new
+        unnamed = 0
+        log_rollback("started", step_name, total: total, batch_size: RubyReactor::Map::ROLLBACK_CHUNK)
+        (0...total).step(RubyReactor::Map::ROLLBACK_CHUNK) do |position|
+          ids = storage.retrieve_map_element_context_ids_from_tail(map_id, map_class, position,
+                                                                   RubyReactor::Map::ROLLBACK_CHUNK, total: total)
+          ids.each do |id|
+            outcome = inline_element_rollback(map_id, id, step_name)
+            outcome["index"] ? seen << outcome["index"] : unnamed += 1
+            reported.concat(ContextSerializer.deserialize_value(outcome["failures"])) if outcome["index"]
+          end
+        end
+        unseen = ->(indexes) { indexes.reject { |index| seen.include?(index) } }
+        finish_rollback(step_name, total, unavailable_entries(step_name, unnamed, unseen) + reported)
+      end
+
+      # Today's inline behavior: one wait for the element's lock, then report
+      # it in flight.
+      def inline_element_rollback(map_id, element_context_id, step_name)
+        outcome = RubyReactor::Map::ElementRollback.call(map_id: map_id, element_context_id: element_context_id,
+                                                         element_class: element_class, step_name: step_name)
+        if outcome["outcome"] == "contended"
+          outcome = { "index" => outcome["index"], "outcome" => "element_in_flight",
+                      "failures" => ContextSerializer.serialize_value(
+                        [rollback_entry(outcome["index"], :element_in_flight, "was still running at rollback time")]
+                      ) }
+        end
+        log_rollback("element", step_name, index: outcome["index"], outcome: outcome["outcome"],
+                                           failures: outcome["failures"].size)
+        outcome
+      end
+
+      # Each element whose context expired is reported once (R-09, I2). The
+      # parent's map reference counts the elements that started: set per
+      # element inline, and to the total when a fan-out map completes. With
+      # it, every started index no rollback saw is named, even when the index
+      # list itself expired; without it (a failed fan-out map skipped some
+      # indices), one unnamed entry per element whose row is gone.
+      def unavailable_entries(step_name, unnamed, unseen)
+        ref = Utils::FetchIndifferent.call(context.composed_contexts, step_name)
+        started = ref && Utils::FetchIndifferent.call(ref, :started)
+        missing = started ? unseen.call((0...started.to_i).to_a) : [nil] * unnamed
+        missing.map { |index| rollback_entry(index, :context_unavailable, "context expired") }
+      end
+
+      def finish_rollback(step_name, total, failures)
+        log_rollback("completed", step_name, total: total, failed: failures.size)
+        return RubyReactor.Success() if failures.empty?
+
+        RubyReactor.Failure("map :#{step_name} rollback incomplete", rollback_failures: failures)
+      end
+
+      def log_rollback(event, step_name, **fields)
+        RubyReactor::Map.log("ruby_reactor.map.rollback.#{event}",
+                             reactor: RubyReactor.reactor_storage_name(owner_context.reactor_class),
+                             context_id: owner_context.context_id, map_step: step_name.to_s, **fields)
+      end
+
+      def rollback_entry(index, reason, message)
+        RubyReactor::Map::ElementRollback.entry(context.current_step, index, reason, message)
+      end
+    end
+  end
+end

data/lib/ruby_reactor/map/sweeper.rb

@@ -16,6 +16,12 @@ module RubyReactor
     #     re-triggered (M2) — gated so it never fires while a collector or the
     #     parent is alive, or after the parent already collected.
     #
+    # For each distributed map rollback (009 S-5), the same signal on the
+    # rollback's own records: a claimed position with no outcome and no live
+    # element lock lost its job and is re-dispatched; with every claimed
+    # position settled but positions left, the throw's trigger was lost and
+    # the next throw is claimed. A lost owner resume is `RubyReactor::Sweeper`'s.
+    #
     # `run_once` is pure and idempotent; the host wires the cadence (same contract
     # as RubyReactor::Sweeper).
     class Sweeper
@@ -29,10 +35,11 @@ module RubyReactor
         @logger = logger || RubyReactor.configuration.logger
       end
 
-      # Returns { redispatched:, recollected: } counts.
+      # Returns { redispatched:, recollected:, rollback_redispatched: } counts.
       def run_once(limit: 1000)
         redispatched = 0
         recollected = 0
+        rollback_redispatched = 0
 
         @storage.scan_maps(count: limit).each do |meta|
           missing = missing_indices(meta)
@@ -46,11 +53,53 @@ module RubyReactor
           @logger.warn("RubyReactor::Map::Sweeper failed on map #{meta["map_id"]}: #{e.class}: #{e.message}")
         end
 
-        { redispatched: redispatched, recollected: recollected }
+        @storage.scan_map_rollbacks(count: limit).each do |meta|
+          rollback_redispatched += sweep_rollback(meta)
+        rescue StandardError => e
+          @logger.warn("RubyReactor::Map::Sweeper failed on map rollback #{meta["map_id"]}: #{e.class}: #{e.message}")
+        end
+
+        { redispatched: redispatched, recollected: recollected, rollback_redispatched: rollback_redispatched }
       end
 
       private
 
+      def sweep_rollback(meta)
+        map_id = meta["map_id"]
+        map_class = meta["parent_reactor_class_name"]
+        total = meta["total"].to_i
+        claimed = [@storage.retrieve_map_rollback_offset(map_id, map_class), total].min
+        missing = (0...claimed).to_a - @storage.stored_map_rollback_positions(map_id, map_class)
+        return redispatch_rollback(meta, missing) if missing.any?
+        return 0 if claimed >= total
+
+        RubyReactor::Map::Dispatcher.dispatch_rollback_batch(map_id: map_id, parent_reactor_class_name: map_class)
+      end
+
+      def redispatch_rollback(meta, positions)
+        map_id = meta["map_id"]
+        map_class = meta["parent_reactor_class_name"]
+        positions.count do |position|
+          id = @storage.retrieve_map_element_context_ids_from_tail(map_id, map_class, position, 1,
+                                                                   total: meta["total"].to_i).first
+          index = element_index(id, meta)
+          next false if index && @storage.lock_held?("map_element:#{map_id}:#{index}") # element alive
+
+          RubyReactor::Map::Dispatcher.queue_rollback_job(meta, map_id, map_class, position, id)
+          true
+        end
+      end
+
+      # From the element's row; nil once the row is gone (the job reports it).
+      def element_index(element_context_id, meta)
+        return nil unless element_context_id
+
+        element_class = RubyReactor::Map::Helpers.resolve_reactor_class(meta["reactor_class_info"])
+        data = @storage.retrieve_context(element_context_id, RubyReactor.reactor_storage_name(element_class))
+        metadata = data && ContextSerializer.deserialize_value(data["map_metadata"])
+        metadata && Utils::FetchIndifferent.call(metadata, :index)
+      end
+
       def missing_indices(meta)
         @storage.missing_map_indices(meta["map_id"], meta["count"].to_i, meta["parent_reactor_class_name"])
       end
@@ -77,22 +126,31 @@ module RubyReactor
       end
 
       # N1: a nested map's parent is a map element running under a `map_element:`
-      # lock, not an `async:` lock. Derive the right key from metadata.
+      # lock, not an `async:` lock. Derive the right key from metadata. Any
+      # other map's completion resumes its OWNER run (009 R-03), so that run's
+      # lock is the one to check; metadata from before 009 names no owner.
       def parent_live_lock?(meta)
         if meta["parent_is_map_element"]
           @storage.lock_held?("map_element:#{meta["outer_map_id"]}:#{meta["outer_index"]}")
         else
-          @storage.lock_held?("async:#{meta["parent_context_id"]}")
+          @storage.lock_held?("async:#{meta["owner_context_id"] || meta["parent_context_id"]}")
         end
       end
 
+      # The map's own context already recorded the step, or the owner run is
+      # past it: finished, or rolling back (it adopted the map).
       def parent_already_collected?(meta)
         data = @storage.retrieve_context(meta["parent_context_id"], meta["parent_reactor_class_name"])
-        return false unless data
+        return true if data && (data["intermediate_results"] || {}).key?(meta["step_name"].to_s)
+
+        owner = owner_context_data(meta) || data
+        owner && %w[completed failed halted skipped cancelled rolling_back].include?(owner["status"].to_s)
+      end
+
+      def owner_context_data(meta)
+        return nil unless meta["owner_context_id"]
 
-        results = data["intermediate_results"] || {}
-        status = data["status"].to_s
-        results.key?(meta["step_name"].to_s) || %w[completed failed halted skipped].include?(status)
+        @storage.retrieve_context(meta["owner_context_id"], meta["owner_reactor_class_name"])
       end
 
       def retrigger_collector(meta)

data/lib/ruby_reactor/map.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    # Element jobs one throw enqueues when a fan-out map declares no
+    # `batch_size` (009 R-10), forward and rollback alike.
+    DEFAULT_BATCH_SIZE = 50
+    # Element contexts an inline map's rollback loads per read (009 R-01).
+    ROLLBACK_CHUNK = 100
+
+    # One structured key=value line (009 FR-011), as `Executor#log_completion`.
+    def self.log(event, **fields)
+      line = { event: event, **fields }.map { |key, value| "#{key}=#{value.inspect}" }.join(" ")
+      RubyReactor.configuration.logger.info(line)
+    end
+  end
+end

data/lib/ruby_reactor/max_retries_exhausted_failure.rb

@@ -7,12 +7,14 @@ module RubyReactor
     # rubocop:disable Metrics/ParameterLists
     def initialize(message, step:, attempts:, original_error: nil,
                    inputs: {}, backtrace: nil, redact_inputs: [],
-                   reactor_name: nil, step_arguments: {}, validation_errors: nil, rollback_failures: nil)
+                   reactor_name: nil, step_arguments: {}, validation_errors: nil, rollback_failures: nil,
+                   exception_class: nil)
       # rubocop:enable Metrics/ParameterLists
       super(message,
             step_name: step, inputs: inputs, backtrace: backtrace,
             redact_inputs: redact_inputs, reactor_name: reactor_name, step_arguments: step_arguments,
-            validation_errors: validation_errors, rollback_failures: rollback_failures)
+            validation_errors: validation_errors, rollback_failures: rollback_failures,
+            exception_class: exception_class)
       @attempts = attempts
       @original_error = original_error
     end

data/lib/ruby_reactor/reactor.rb

@@ -6,6 +6,9 @@ module RubyReactor
     include RubyReactor::Dsl::Reactor
     include RubyReactor::Dsl::Lockable
 
+    # Seconds a manual undo waits for the run's context lock (009 R-12).
+    UNDO_LOCK_WAIT = 5
+
     attr_reader :context, :result, :undo_trace, :execution_trace
 
     def self.find(id)
@@ -60,9 +63,12 @@ module RubyReactor
       reactor.cancel(reason)
     end
 
+    # A rollback that hands off at a fan-out map finishes in a worker, which
+    # applies `cancelled` itself (009 R-12).
     def self.undo(id)
       reactor = find(id)
-      reactor.undo
+      return if reactor.undo == :handed_off
+
       cancel(id: id, reason: "Undo triggered")
     end
 
@@ -204,13 +210,34 @@ module RubyReactor
       RubyReactor::Failure(e.message, invalid_payload: true)
     end
 
+    # Undoes every completed step, under the run's context lock so no worker
+    # resumes it meanwhile (009 R-12). Returns `:handed_off` when a fan-out
+    # map's rollback continues in its element jobs: the run is then
+    # `rolling_back`, and a worker finishes it as `cancelled`.
     def undo
+      raise Error::ValidationError, "rollback already in progress" if @context.rolling_back?
+
+      lock = acquire_undo_lock
       executor = Executor.new(self.class, {}, @context)
-      executor.undo_all
+      @context.rollback = { "trigger" => "undo", "compensated" => true, "failures" => [] }
+      begin
+        executor.undo_all
+      rescue Error::RollbackHandedOff => e
+        @context.status = :rolling_back
+        executor.save_context
+        executor.hand_off_rollback!(e)
+        return :handed_off
+      end
+      @context.rollback = nil
       executor.save_context
+    ensure
+      lock&.release
     end
 
     def cancel(reason)
+      # `cancelled` is terminal: the steps before the map would never be undone.
+      raise Error::ValidationError, "rollback in progress; cannot cancel" if @context.rolling_back?
+
       @context.cancelled = true
       @context.cancellation_reason = reason
       @context.status = "cancelled"
@@ -235,6 +262,18 @@ module RubyReactor
       save_context
     end
 
+    # Raises `Lock::AcquisitionError` while a live run or rollback holds it.
+    # Skipped in inline job-testing mode, as `Executor#acquire_context_lock`.
+    def acquire_undo_lock
+      return if defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
+
+      lock = RubyReactor::Lock.new("async:#{@context.context_id}", owner: SecureRandom.uuid,
+                                                                   ttl: configuration.context_lock_ttl,
+                                                                   wait: UNDO_LOCK_WAIT)
+      lock.acquire
+      lock
+    end
+
     def validate_steps!
       return unless self.class.steps.empty?
 

data/lib/ruby_reactor/rspec/active_job_helpers.rb

@@ -11,12 +11,16 @@ module RubyReactor
       PendingJob = Struct.new(:job_class, :raw) do
         def perform!
           ::ActiveJob::Base.queue_adapter.enqueued_jobs.delete(raw)
-          job_class.new(*raw[:args]).perform_now
+          job_class.new(*ActiveJobHelpers.arguments(raw)).perform_now
         end
 
         def args
           raw[:args]
         end
+
+        # The Sidekiq `PendingJob`'s name for it, so step-wise drains read the
+        # same on both backends (009 U4).
+        alias_method :worker_class, :job_class
       end
 
       def self.test_adapter?
@@ -37,11 +41,18 @@ module RubyReactor
 
           queue.dup.each do |job|
             queue.delete(job)
-            job[:job].new(*job[:args]).perform_now
+            job[:job].new(*arguments(job)).perform_now
           end
         end
       end
 
+      # The test adapter keeps a job's arguments serialized; a real backend
+      # hands `perform` the deserialized ones (a job requeueing its own
+      # arguments would otherwise re-serialize ActiveJob's markers).
+      def self.arguments(raw)
+        ::ActiveJob::Arguments.deserialize(raw[:args])
+      end
+
       def self.pending_async_jobs
         return [] unless test_adapter?
 

data/lib/ruby_reactor/rspec/matchers.rb

@@ -4,8 +4,10 @@ module RubyReactor
   module RSpec
     # rubocop:disable Metrics/ModuleLength
     module Matchers
+      extend ::RSpec::Matchers::DSL
+
       # rubocop:disable Metrics/BlockLength
-      ::RSpec::Matchers.define :be_success do
+      matcher :be_success do
         match do |subject|
           subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
           subject.success?
@@ -63,7 +65,7 @@ module RubyReactor
         end
       end
 
-      ::RSpec::Matchers.define :be_failure do
+      matcher :be_failure do
         match do |subject|
           subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
           subject.failure?
@@ -74,7 +76,7 @@ module RubyReactor
         end
       end
 
-      ::RSpec::Matchers.define :have_run_step do |step_name|
+      matcher :have_run_step do |step_name|
         match do |subject|
           subject.ensure_executed!
           @trace = subject.reactor_instance.context.execution_trace
@@ -127,7 +129,7 @@ module RubyReactor
         end
       end
 
-      ::RSpec::Matchers.define :have_retried_step do |step_name|
+      matcher :have_retried_step do |step_name|
         match do |subject|
           subject.ensure_executed!
           attempts = subject.reactor_instance.context.retry_context.attempts_for_step(step_name)
@@ -151,7 +153,7 @@ module RubyReactor
         end
       end
 
-      ::RSpec::Matchers.define :have_validation_error do |field|
+      matcher :have_validation_error do |field|
         match do |subject|
           subject.ensure_executed!
           return false unless subject.failure?
@@ -173,7 +175,7 @@ module RubyReactor
       end
 
       # Matcher to check if reactor is paused at an interrupt
-      ::RSpec::Matchers.define :be_paused do
+      matcher :be_paused do
         match do |subject|
           subject.ensure_executed!
           subject.paused?
@@ -188,9 +190,26 @@ module RubyReactor
         end
       end
 
+      # A run whose rollback handed off at a fan-out map and has not finished
+      # (009): its element rollback jobs are still pending.
+      matcher :be_rolling_back do
+        match do |subject|
+          subject.ensure_executed!
+          subject.rolling_back?
+        end
+
+        failure_message do |subject|
+          "expected reactor to be rolling back, but status was #{subject.reactor_instance.context.status}"
+        end
+
+        failure_message_when_negated do |_subject|
+          "expected reactor not to be rolling back, but it is"
+        end
+      end
+
       # Matcher to check if reactor is paused at a specific interrupt step
       # Works with both single and multiple concurrent interrupts
-      ::RSpec::Matchers.define :be_paused_at do |*step_names|
+      matcher :be_paused_at do |*step_names|
         match do |subject|
           subject.ensure_executed!
           return false unless subject.paused?
@@ -221,7 +240,7 @@ module RubyReactor
       end
 
       # Matcher to check the exact set of ready interrupt steps
-      ::RSpec::Matchers.define :have_ready_interrupts do |*expected_steps|
+      matcher :have_ready_interrupts do |*expected_steps|
         match do |subject|
           subject.ensure_executed!
           return false unless subject.paused?
@@ -277,7 +296,7 @@ module RubyReactor
       #   expect(result).to be_halted
       #   expect(result).to be_halted.because(:period)
       #   expect(result).to be_halted.at_step(:second)
-      ::RSpec::Matchers.define :be_halted do
+      matcher :be_halted do
         match do |subject|
           subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
           actual = subject.respond_to?(:result) ? subject.result : subject
@@ -320,7 +339,7 @@ module RubyReactor
       # Examples:
       #   expect(result).to be_skipped
       #   expect(subject).to be_skipped.at_step(:maybe_sync)
-      ::RSpec::Matchers.define :be_skipped do
+      matcher :be_skipped do
         match do |subject|
           subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
           actual = subject.respond_to?(:result) ? subject.result : subject
@@ -367,7 +386,7 @@ module RubyReactor
       #
       #   expect("order:42").to be_locked
       #   expect("order:42").to be_locked.by("ctx-abc")
-      ::RSpec::Matchers.define :be_locked do
+      matcher :be_locked do
         match do |key|
           info = Matchers.coordination_adapter.lock_info("lock:#{key}")
           next false unless info
@@ -400,7 +419,7 @@ module RubyReactor
       # user-provided semaphore name (without the "semaphore:" prefix).
       #
       #   expect("api_limit").to have_available_tokens(3)
-      ::RSpec::Matchers.define :have_available_tokens do |expected|
+      matcher :have_available_tokens do |expected|
         match do |name|
           Matchers.coordination_adapter.semaphore_state(name)[:available] == expected
         end
@@ -415,7 +434,7 @@ module RubyReactor
       # Asserts the number of currently-checked-out semaphore tokens.
       #
       #   expect("api_limit").to have_held_tokens(2)
-      ::RSpec::Matchers.define :have_held_tokens do |expected|
+      matcher :have_held_tokens do |expected|
         match do |name|
           Matchers.coordination_adapter.semaphore_state(name)[:held] == expected
         end
@@ -434,7 +453,7 @@ module RubyReactor
       #
       #   expect("stripe:42").to have_rate_limit_count(3).for(:second)
       #   expect("stripe:42").to have_rate_limit_count(1).for(:minute).since(started_at)
-      ::RSpec::Matchers.define :have_rate_limit_count do |expected|
+      matcher :have_rate_limit_count do |expected|
         match do |key_base|
           raise ArgumentError, "have_rate_limit_count requires .for(period)" unless @period
 
@@ -458,7 +477,7 @@ module RubyReactor
       # Asserts that a `with_period` bucket has been marked. Use `.for(period)`.
       #
       #   expect("daily_report:7").to be_period_marked.for(:day)
-      ::RSpec::Matchers.define :be_period_marked do
+      matcher :be_period_marked do
         match do |key_base|
           raise ArgumentError, "be_period_marked requires .for(period)" unless @period
 
@@ -482,7 +501,7 @@ module RubyReactor
       # the user-provided ordered_lock key (without the `ordered_lock:` prefix).
       #
       #   expect("orders:42").to have_ordered_lock_next(3)
-      ::RSpec::Matchers.define :have_ordered_lock_next do |expected|
+      matcher :have_ordered_lock_next do |expected|
         match { |key| Matchers.coordination_adapter.ordered_lock_peek(key)[:next] == expected }
 
         failure_message do |key|
@@ -495,7 +514,7 @@ module RubyReactor
       # Asserts the last-advanced cursor for an ordered_lock key.
       #
       #   expect("orders:42").to have_ordered_lock_last_completed(2)
-      ::RSpec::Matchers.define :have_ordered_lock_last_completed do |expected|
+      matcher :have_ordered_lock_last_completed do |expected|
         match do |key|
           Matchers.coordination_adapter.ordered_lock_peek(key)[:last_completed] == expected
         end
@@ -511,7 +530,7 @@ module RubyReactor
       # Order-insensitive — the matcher sorts both sides.
       #
       #   expect("orders:42").to have_ordered_lock_in_flight(2, 3)
-      ::RSpec::Matchers.define :have_ordered_lock_in_flight do |*expected|
+      matcher :have_ordered_lock_in_flight do |*expected|
         match do |key|
           actual = Matchers.coordination_adapter.ordered_lock_peek(key)[:in_flight].sort
           actual == expected.flatten.map(&:to_i).sort
@@ -528,7 +547,7 @@ module RubyReactor
       # in-flight nonces. After a clean drain `peek` returns all zeros.
       #
       #   expect("orders:42").to be_ordered_lock_drained
-      ::RSpec::Matchers.define :be_ordered_lock_drained do
+      matcher :be_ordered_lock_drained do
         match do |key|
           state = Matchers.coordination_adapter.ordered_lock_peek(key)
           state[:next].zero? && state[:last_completed].zero? && state[:in_flight].empty?
@@ -551,7 +570,7 @@ module RubyReactor
       #
       #   expect(result).to have_contended_at(:charge)
       #   expect(result).to have_contended_at(:charge).on("acct:1")
-      ::RSpec::Matchers.define :have_contended_at do |step_name|
+      matcher :have_contended_at do |step_name|
         match do |subject|
           trace = execution_trace_for(subject)
           @entry = trace.find do |t|
@@ -596,7 +615,7 @@ module RubyReactor
       #   expect(result).to have_rollback_failure(:charge)
       #   expect(subject).to have_rollback_failure(:charge).for_key("acct:1").because(:coordination_unavailable)
       #   expect(subject).not_to have_rollback_failure(:charge)
-      ::RSpec::Matchers.define :have_rollback_failure do |step_name|
+      matcher :have_rollback_failure do |step_name|
         match do |subject|
           @entries = rollback_failures_for(subject)
           @entries.any? do |e|

data/lib/ruby_reactor/rspec/sidekiq_helpers.rb

@@ -12,15 +12,18 @@ module RubyReactor
       # until the queues are empty. Recursive — handles jobs that re-enqueue
       # themselves (e.g. ordered_lock snoozes) and worker chains that queue
       # additional jobs.
+      # Whichever test backend is active (Sidekiq fake mode or the ActiveJob
+      # `:test` adapter), via `AsyncTestHelpers`.
       def drain_async_jobs(max_iterations: 100)
-        SidekiqHelpers.drain_async_jobs(max_iterations: max_iterations)
+        AsyncTestHelpers.drain_async_jobs(max_iterations: max_iterations)
       end
 
       # All currently-pending async jobs, wrapped in `PendingJob` so callers
       # can perform individual jobs out-of-order (e.g. to assert
-      # ordered_lock's snooze behavior) without touching Sidekiq internals.
+      # ordered_lock's snooze behavior) without touching backend internals.
+      # `worker_class` names the job's class on either backend.
       def pending_async_jobs
-        SidekiqHelpers.pending_async_jobs
+        AsyncTestHelpers.pending_async_jobs
       end
 
       PendingJob = Struct.new(:worker_class, :raw) do
@@ -39,6 +42,7 @@ module RubyReactor
           RubyReactor::Adapters::Sidekiq::Worker,
           RubyReactor::Adapters::Sidekiq::MapElementWorker,
           RubyReactor::Adapters::Sidekiq::MapCollectorWorker,
+          RubyReactor::Adapters::Sidekiq::MapElementRollbackWorker,
           RubyReactor::Adapters::Sidekiq::StepWorker
         ]
       end