ruby_reactor 0.8.5 → 0.8.7

data/lib/ruby_reactor/map/element_rollback.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    # Rolls back ONE element of a map (009 R-07): the element's own undo
+    # stack, replayed under its liveness lock and saved after every undone
+    # entry, so a redelivery resumes after the last one (only the entry that
+    # was cut off may run again). The inline path calls `.call` in process; a
+    # fan-out map's `MapElementRollbackWorker` calls `.perform`.
+    class ElementRollback
+      extend Helpers
+
+      # The element context row is gone: its index went with it.
+      def self.unavailable(step_name)
+        { "index" => nil, "outcome" => "context_unavailable",
+          "failures" => ContextSerializer.serialize_value([entry(step_name, nil, :context_unavailable,
+                                                                 "context expired")]) }
+      end
+
+      # `{ "index", "outcome", "failures" }` (DM §5), with `failures`
+      # serialized. Outcome `contended`, never stored, means the element's lock
+      # was still held after `wait`.
+      def self.call(map_id:, element_context_id:, element_class:, step_name:, wait: Step::MapStep::ELEMENT_LOCK_WAIT)
+        storage_name = RubyReactor.reactor_storage_name(element_class)
+        data = element_context_id && RubyReactor.configuration.storage_adapter.retrieve_context(element_context_id,
+                                                                                                storage_name)
+        return unavailable(step_name) unless data
+
+        element = Context.deserialize_from_retry(data)
+        index = Utils::FetchIndifferent.call(element.map_metadata || {}, :index).to_i
+        lock = acquire_element_lock(map_id, index, wait)
+        return outcome(index, "contended") if lock == :held
+
+        begin
+          roll_back(element, element_class, index, step_name)
+        ensure
+          lock.release if lock.respond_to?(:release)
+        end
+      end
+
+      # A completed element, or an aborted one (an inline run interrupted in
+      # it, which kept the undo entries of the steps it completed), undoes;
+      # a failed one already rolled itself back, and a halted or superseded
+      # running one did nothing to undo.
+      def self.roll_back(element, element_class, index, step_name)
+        return outcome(index, "not_needed") unless %w[completed aborted].include?(element.status.to_s)
+
+        executor = Executor.new(element.reactor_class || element_class, {}, element)
+        executor.undo_all { executor.checkpoint! }
+        executor.checkpoint!
+        tag = { map_step: step_name.to_sym, element_index: index }
+        failures = executor.compensation_manager.rollback_failures.map { |failure| failure.merge(tag) }
+        outcome(index, failures.empty? ? "undone" : "failed", failures)
+      end
+
+      # The job's body. A contended lock requeues the job, storing nothing,
+      # up to `lock_snooze_max_attempts` times before reporting the element
+      # in flight: two contexts of one index in the same batch then serialize
+      # instead of one reporting a false failure (R-07).
+      def self.perform(arguments)
+        args = Helpers.normalize_arguments(arguments)
+        map_id = args[:map_id]
+        map_class = args[:parent_reactor_class_name]
+        storage = RubyReactor.configuration.storage_adapter
+        return if storage.map_rollback_outcome_stored?(map_id, map_class, args[:position])
+
+        attempt = args[:attempt].to_i
+        result = call(map_id: map_id, element_context_id: args[:element_context_id],
+                      element_class: resolve_reactor_class(args[:reactor_class_info]), step_name: args[:step_name],
+                      wait: attempt.zero? ? Step::MapStep::ELEMENT_LOCK_WAIT : 0)
+        if result["outcome"] == "contended"
+          return requeue(args, attempt) unless snooze_exhausted?(attempt)
+
+          result = in_flight(result["index"], args[:step_name])
+        end
+        return unless storage.store_map_rollback_outcome(map_id, map_class, args[:position], result)
+
+        log_outcome(args, result)
+        Dispatcher.dispatch_rollback_batch(map_id: map_id, parent_reactor_class_name: map_class) if batch_end?(args)
+        signal_owner_if_settled(args)
+      end
+
+      def self.in_flight(index, step_name)
+        outcome(index, "element_in_flight",
+                [entry(step_name, index, :element_in_flight, "was still running at rollback time")])
+      end
+
+      def self.snooze_exhausted?(attempt)
+        max = RubyReactor.configuration.lock_snooze_max_attempts
+        max != :infinity && attempt >= max
+      end
+
+      def self.requeue(args, attempt)
+        config = RubyReactor.configuration
+        config.async_router.perform_map_element_rollback_in(
+          Worker.snooze_delay(config, nil), **args.slice(*ROLLBACK_JOB_ARGS), attempt: attempt + 1
+        )
+      end
+
+      # The position that ends a throw claims the next one (R-02).
+      def self.batch_end?(args)
+        batch_size = args[:batch_size].to_i
+        batch_size.positive? && ((args[:position].to_i + 1) % batch_size).zero?
+      end
+
+      # The handshake's job side (R-05): only once the hand-off is saved, so
+      # the owner never resumes from a stale blob — and never in inline job
+      # mode, where the map settles before any hand-off exists.
+      def self.signal_owner_if_settled(args)
+        storage = RubyReactor.configuration.storage_adapter
+        map_id = args[:map_id]
+        map_class = args[:parent_reactor_class_name]
+        meta = storage.retrieve_map_rollback_metadata(map_id, map_class)
+        return unless meta && storage.count_map_rollback_outcomes(map_id, map_class) >= meta["total"].to_i
+        return unless storage.map_rollback_handed_off?(map_id, map_class)
+        return unless storage.claim_map_rollback_signal(map_id, map_class)
+
+        RubyReactor.configuration.async_router.perform_async(args[:owner_context_id], args[:owner_reactor_class_name])
+      end
+
+      def self.log_outcome(args, result)
+        Map.log("ruby_reactor.map.rollback.element",
+                reactor: args[:owner_reactor_class_name], context_id: args[:owner_context_id],
+                map_step: args[:step_name], index: result["index"], outcome: result["outcome"],
+                failures: Array(result["failures"]).size)
+      end
+
+      def self.outcome(index, outcome, failures = [])
+        { "index" => index, "outcome" => outcome, "failures" => ContextSerializer.serialize_value(failures) }
+      end
+
+      # A rollback failure entry for the element as a whole (008 shape).
+      def self.entry(step_name, index, reason, message)
+        step = step_name.to_sym
+        { step: step, kind: :undo, key: nil, reason: reason, map_step: step, element_index: index,
+          message: "map element #{index} #{message}" }
+      end
+
+      def self.acquire_element_lock(map_id, index, wait)
+        return nil if ElementExecutor.inline_testing_mode?
+
+        lock = RubyReactor::Lock.new("map_element:#{map_id}:#{index}", owner: SecureRandom.uuid,
+                                                                       ttl: RubyReactor.configuration.context_lock_ttl,
+                                                                       wait: wait)
+        lock.acquire
+        lock
+      rescue RubyReactor::Lock::AcquisitionError
+        :held
+      end
+
+      ROLLBACK_JOB_ARGS = %i[map_id position element_context_id reactor_class_info parent_reactor_class_name step_name
+                             batch_size owner_context_id owner_reactor_class_name].freeze
+
+      private_class_method :roll_back, :in_flight, :snooze_exhausted?, :requeue, :batch_end?,
+                           :signal_owner_if_settled, :log_outcome, :acquire_element_lock
+    end
+  end
+end

data/lib/ruby_reactor/map/helpers.rb

@@ -4,6 +4,16 @@ module RubyReactor
   module Map
     # Shared helper methods for Map executors
     module Helpers
+      # Job payloads enqueued before 009 carry `fail_fast` where they now carry
+      # `atomic` (R-11, FR-023): the one place the old key is read. Removed at
+      # the next MAJOR.
+      def self.normalize_arguments(arguments)
+        normalized = arguments.to_h.transform_keys(&:to_sym)
+        normalized[:atomic] = normalized[:fail_fast] if !normalized.key?(:atomic) && normalized.key?(:fail_fast)
+        normalized.delete(:fail_fast)
+        normalized
+      end
+
       # Resolves the reactor class from reactor_class_info
       def resolve_reactor_class(info)
         if info["type"] == "class"
@@ -20,6 +30,7 @@ module RubyReactor
           raise "Unknown reactor class info: #{info}"
         end
       end
+      module_function :resolve_reactor_class
 
       # Loads parent context from storage
       def load_parent_context_from_storage(parent_context_id, reactor_class_name, storage)
@@ -31,161 +42,6 @@ module RubyReactor
       def build_element_inputs(mappings, parent_context, element)
         RubyReactor::Step::MapStep.build_mapped_inputs(mappings, parent_context, element)
       end
-
-      # Applies collect block to results
-      def apply_collect_block(results, step_config)
-        collect_block = step_config.arguments[:collect_block][:source].value
-
-        if collect_block
-          # Pass all results (Success and Failure) to collect block
-          begin
-            collected = collect_block.call(results)
-            RubyReactor::Success(collected)
-          rescue RubyReactor::Error::Rescuable => e
-            RubyReactor::Failure(e)
-          end
-        else
-          # Default behavior: fail if any failure
-          first_failure = results.find(&:failure?)
-          first_failure || RubyReactor::Success(results.map(&:value))
-        end
-      end
-
-      # Resumes parent reactor execution after map completion
-      def resume_parent_execution(parent_context, step_name, final_result, storage) # rubocop:disable Metrics/MethodLength
-        executor = RubyReactor::Executor.new(parent_context.reactor_class, {}, parent_context)
-        step_name_sym = step_name.to_sym
-
-        if final_result.failure?
-          parent_context.current_step = step_name_sym
-
-          error = RubyReactor::Error::StepFailureError.new(
-            final_result.error,
-            step: step_name_sym,
-            context: parent_context,
-            original_error: final_result.error.is_a?(Exception) ? final_result.error : nil,
-            exception_class: final_result.respond_to?(:exception_class) ? final_result.exception_class : nil
-          )
-
-          # Pass backtrace if available
-          if final_result.respond_to?(:backtrace) && final_result.backtrace
-            error.set_backtrace(final_result.backtrace)
-          elsif final_result.error.respond_to?(:backtrace)
-            error.set_backtrace(final_result.error.backtrace)
-          end
-
-          # Bracket the rollback with reactor lifecycle events so that
-          # compensation/undo spans nest under a reactor span (and stay attached
-          # to the originating trace), mirroring the success path's
-          # resume_execution. Without this, rollback runs in the collector worker
-          # with no active reactor span and the undo/compensation spans orphan.
-          executor.middlewares.on(:start_reactor, parent_context.reactor_class.name, parent_context.inputs,
-                                  parent_context)
-          failure_response = nil
-          begin
-            # The map failed in its element jobs, so no executor compensated it.
-            # Fail it exactly as the inline path does: adopt the failing
-            # element's rollback failures, compensate the map (its completed
-            # elements; every index has settled, R-04), undo the completed
-            # steps, and fail with `CompensationError` if the map's rollback
-            # was incomplete.
-            manager = executor.compensation_manager
-            manager.rollback_failures.concat(final_result.rollback_failures)
-            failure_response = begin
-              manager.handle_step_failure(parent_context.reactor_class.steps[step_name_sym], final_result.error, {})
-              executor.result_handler.handle_execution_error(error)
-            rescue RubyReactor::Error::CompensationError => e
-              executor.result_handler.handle_execution_error(e)
-            end
-            # Manually update context status since we're not running executor loop
-            executor.send(:update_context_status, failure_response)
-          ensure
-            executor.middlewares.on(:failed_reactor, parent_context.reactor_class.name, failure_response,
-                                    parent_context)
-          end
-          # This branch runs no executor loop, so nothing else persists the
-          # failed status: store it here.
-          store_parent(parent_context, storage)
-        else
-          parent_context.set_result(step_name_sym, final_result.value)
-          # A completed fan-out map is tracked for undo like any step (R-03).
-          # The record stays empty: `MapStep#undo` finds its elements through
-          # the map's element index, so the parent does not grow per element.
-          parent_context.undo_stack << { step: parent_context.reactor_class.steps[step_name_sym], arguments: {},
-                                         result: RubyReactor.Success(nil) }
-          record_elements_started(parent_context, step_name, storage)
-
-          # Manually update execution trace to reflect completion
-          # This is necessary because resume_execution continues from the NEXT step
-          # and the async step (which returned DispatchResult) needs to be marked as done with actual value
-          parent_context.append_execution_trace(
-            {
-              type: :result,
-              step: step_name_sym,
-              timestamp: Time.now,
-              value: final_result.value,
-              status: :success
-            }
-          )
-
-          parent_context.current_step = nil
-
-          # Mark the context as a worker before resuming: resume_execution runs
-          # the collector's own worker (this call), not the reactor's normal
-          # caller. Without this, StepExecutor#handoff_at? (gated on
-          # `inline_async_execution`) re-evaluates any later `background`
-          # before:/after: cut point and enqueues a second hand-off, or runs an
-          # `after:` target here in the collector instead of the original
-          # dispatching worker.
-          parent_context.inline_async_execution = true
-          resume_parked_aware(executor, parent_context)
-        end
-      end
-
-      # `resume_execution` persists the parent itself — under the parent's
-      # context lock, and deliberately NOT when it lost that lock to a live
-      # duplicate or replayed an already-terminal run. Storing again here, after
-      # the lock is released, would overwrite whatever the lock's holder wrote
-      # (single writer), so the resume's own save is the only one.
-      #
-      # This collector is a worker running the parent's execution, so it is
-      # also a final handler for park signals (005 R-01), like `Worker` and
-      # `ElementExecutor`: the resume has already parked the parent's holds and
-      # saved; hand the rest back to the parent's own worker.
-      #
-      # The parent is loaded from its own blob, so it has no `root_context`:
-      # a map inside a composed child resumes and requeues that child as its
-      # own execution, never its root. ponytail: the root is never resumed
-      # after such a map (already so on main); see "Fan-out map inside a
-      # composed child" in specs/future_improvements.md.
-      def resume_parked_aware(executor, parent_context)
-        executor.resume_execution
-      rescue RubyReactor::Error::ExecutionParked => e
-        config = RubyReactor.configuration
-        config.async_router.perform_in(
-          RubyReactor::Worker.snooze_delay(config, e), parent_context.context_id,
-          RubyReactor.reactor_storage_name(parent_context.reactor_class)
-        )
-      end
-
-      # Every index of a completed map ran: record the count on the map's
-      # reference in the parent, which outlives the element index, so a late
-      # rollback names each element whose context expired.
-      def record_elements_started(parent_context, step_name, storage)
-        ref = Utils::FetchIndifferent.call(parent_context.composed_contexts, step_name)
-        return unless ref
-
-        map_id = Utils::FetchIndifferent.call(ref, :map_id)
-        ref[:started] = storage.retrieve_map_metadata(map_id, parent_context.reactor_class.name)&.fetch("count", 0).to_i
-      end
-
-      def store_parent(parent_context, storage)
-        storage.store_context(
-          parent_context.context_id,
-          ContextSerializer.serialize(parent_context),
-          RubyReactor.reactor_storage_name(parent_context.reactor_class)
-        )
-      end
     end
   end
 end

data/lib/ruby_reactor/map/step_rollback.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    # How a map step rolls back (009 R-01): a fan-out map distributed, one job
+    # per started element (S-2); an inline map in process, in bounded reads
+    # (S-4). Both report the same failures (I-7). Mixed into `Step::MapStep`.
+    module StepRollback
+      private
+
+      # S-2: start the map's rollback once (its records, then the first
+      # throw), then either aggregate a settled rollback or hand the run off
+      # until the element jobs report (R-04, R-05). Re-entered by the owner's
+      # resume, it finds the records and only checks for the settle.
+      def distributed_rollback(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        created, rollback = storage.start_map_rollback(map_id, map_class, **rollback_metadata(map_id, step_name))
+        if created
+          log_rollback("started", step_name, total: rollback["total"], batch_size: rollback["batch_size"])
+          RubyReactor::Map::Dispatcher.dispatch_rollback_batch(map_id: map_id, parent_reactor_class_name: map_class)
+        end
+        total = rollback["total"].to_i
+        if storage.count_map_rollback_outcomes(map_id, map_class) >= total
+          return finish_rollback(step_name, total, distributed_failures(map_id, step_name))
+        end
+
+        raise RubyReactor::Error::RollbackHandedOff.new(map_id: map_id, reactor_class_name: map_class)
+      end
+
+      def rollback_metadata(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        forward = storage.retrieve_map_metadata(map_id, map_class) || {}
+        {
+          total: storage.count_map_element_context_ids(map_id, map_class),
+          batch_size: forward["batch_size"] || RubyReactor::Map::DEFAULT_BATCH_SIZE,
+          step_name: step_name.to_s,
+          owner_context_id: owner_context.context_id,
+          owner_reactor_class_name: RubyReactor.reactor_storage_name(owner_context.reactor_class),
+          reactor_class_info: forward["reactor_class_info"] || build_reactor_class_info(element_class, step_name)
+        }
+      end
+
+      # The failures the element jobs reported, in position order (newest
+      # first, as inline), read in bounded chunks (R-09).
+      def distributed_failures(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        reported = []
+        unnamed = 0
+        storage.each_map_rollback_outcome(map_id, map_class) do |position, outcome|
+          unnamed += 1 if outcome["outcome"] == "context_unavailable" && outcome["index"].nil?
+          next unless %w[failed element_in_flight].include?(outcome["outcome"])
+
+          reported << [position, ContextSerializer.deserialize_value(outcome["failures"])]
+        end
+        unseen = lambda do |indexes|
+          indexes.each_slice(1000).flat_map do |slice|
+            slice.zip(storage.map_rollback_indexes_seen(map_id, map_class, slice)).reject(&:last).map(&:first)
+          end
+        end
+        unavailable_entries(step_name, unnamed, unseen) + reported.sort_by(&:first).flat_map(&:last)
+      end
+
+      # S-4: in process, `Map::ROLLBACK_CHUNK` elements per read from the tail
+      # of the element index, never all at once.
+      def inline_rollback(map_id, step_name)
+        storage = RubyReactor.configuration.storage_adapter
+        map_class = context.reactor_class.name
+        total = storage.count_map_element_context_ids(map_id, map_class)
+        reported = []
+        seen = Set.new
+        unnamed = 0
+        log_rollback("started", step_name, total: total, batch_size: RubyReactor::Map::ROLLBACK_CHUNK)
+        (0...total).step(RubyReactor::Map::ROLLBACK_CHUNK) do |position|
+          ids = storage.retrieve_map_element_context_ids_from_tail(map_id, map_class, position,
+                                                                   RubyReactor::Map::ROLLBACK_CHUNK, total: total)
+          ids.each do |id|
+            outcome = inline_element_rollback(map_id, id, step_name)
+            outcome["index"] ? seen << outcome["index"] : unnamed += 1
+            reported.concat(ContextSerializer.deserialize_value(outcome["failures"])) if outcome["index"]
+          end
+        end
+        unseen = ->(indexes) { indexes.reject { |index| seen.include?(index) } }
+        finish_rollback(step_name, total, unavailable_entries(step_name, unnamed, unseen) + reported)
+      end
+
+      # Today's inline behavior: one wait for the element's lock, then report
+      # it in flight.
+      def inline_element_rollback(map_id, element_context_id, step_name)
+        outcome = RubyReactor::Map::ElementRollback.call(map_id: map_id, element_context_id: element_context_id,
+                                                         element_class: element_class, step_name: step_name)
+        if outcome["outcome"] == "contended"
+          outcome = { "index" => outcome["index"], "outcome" => "element_in_flight",
+                      "failures" => ContextSerializer.serialize_value(
+                        [rollback_entry(outcome["index"], :element_in_flight, "was still running at rollback time")]
+                      ) }
+        end
+        log_rollback("element", step_name, index: outcome["index"], outcome: outcome["outcome"],
+                                           failures: outcome["failures"].size)
+        outcome
+      end
+
+      # Each element whose context expired is reported once (R-09, I2). The
+      # parent's map reference counts the elements that started: set per
+      # element inline, and to the total when a fan-out map completes. With
+      # it, every started index no rollback saw is named, even when the index
+      # list itself expired; without it (a failed fan-out map skipped some
+      # indices), one unnamed entry per element whose row is gone.
+      def unavailable_entries(step_name, unnamed, unseen)
+        ref = Utils::FetchIndifferent.call(context.composed_contexts, step_name)
+        started = ref && Utils::FetchIndifferent.call(ref, :started)
+        missing = started ? unseen.call((0...started.to_i).to_a) : [nil] * unnamed
+        missing.map { |index| rollback_entry(index, :context_unavailable, "context expired") }
+      end
+
+      def finish_rollback(step_name, total, failures)
+        log_rollback("completed", step_name, total: total, failed: failures.size)
+        return RubyReactor.Success() if failures.empty?
+
+        RubyReactor.Failure("map :#{step_name} rollback incomplete", rollback_failures: failures)
+      end
+
+      def log_rollback(event, step_name, **fields)
+        RubyReactor::Map.log("ruby_reactor.map.rollback.#{event}",
+                             reactor: RubyReactor.reactor_storage_name(owner_context.reactor_class),
+                             context_id: owner_context.context_id, map_step: step_name.to_s, **fields)
+      end
+
+      def rollback_entry(index, reason, message)
+        RubyReactor::Map::ElementRollback.entry(context.current_step, index, reason, message)
+      end
+    end
+  end
+end

data/lib/ruby_reactor/map/sweeper.rb

@@ -16,6 +16,12 @@ module RubyReactor
     #     re-triggered (M2) — gated so it never fires while a collector or the
     #     parent is alive, or after the parent already collected.
     #
+    # For each distributed map rollback (009 S-5), the same signal on the
+    # rollback's own records: a claimed position with no outcome and no live
+    # element lock lost its job and is re-dispatched; with every claimed
+    # position settled but positions left, the throw's trigger was lost and
+    # the next throw is claimed. A lost owner resume is `RubyReactor::Sweeper`'s.
+    #
     # `run_once` is pure and idempotent; the host wires the cadence (same contract
     # as RubyReactor::Sweeper).
     class Sweeper
@@ -29,10 +35,11 @@ module RubyReactor
         @logger = logger || RubyReactor.configuration.logger
       end
 
-      # Returns { redispatched:, recollected: } counts.
+      # Returns { redispatched:, recollected:, rollback_redispatched: } counts.
       def run_once(limit: 1000)
         redispatched = 0
         recollected = 0
+        rollback_redispatched = 0
 
         @storage.scan_maps(count: limit).each do |meta|
           missing = missing_indices(meta)
@@ -46,11 +53,53 @@ module RubyReactor
           @logger.warn("RubyReactor::Map::Sweeper failed on map #{meta["map_id"]}: #{e.class}: #{e.message}")
         end
 
-        { redispatched: redispatched, recollected: recollected }
+        @storage.scan_map_rollbacks(count: limit).each do |meta|
+          rollback_redispatched += sweep_rollback(meta)
+        rescue StandardError => e
+          @logger.warn("RubyReactor::Map::Sweeper failed on map rollback #{meta["map_id"]}: #{e.class}: #{e.message}")
+        end
+
+        { redispatched: redispatched, recollected: recollected, rollback_redispatched: rollback_redispatched }
       end
 
       private
 
+      def sweep_rollback(meta)
+        map_id = meta["map_id"]
+        map_class = meta["parent_reactor_class_name"]
+        total = meta["total"].to_i
+        claimed = [@storage.retrieve_map_rollback_offset(map_id, map_class), total].min
+        missing = (0...claimed).to_a - @storage.stored_map_rollback_positions(map_id, map_class)
+        return redispatch_rollback(meta, missing) if missing.any?
+        return 0 if claimed >= total
+
+        RubyReactor::Map::Dispatcher.dispatch_rollback_batch(map_id: map_id, parent_reactor_class_name: map_class)
+      end
+
+      def redispatch_rollback(meta, positions)
+        map_id = meta["map_id"]
+        map_class = meta["parent_reactor_class_name"]
+        positions.count do |position|
+          id = @storage.retrieve_map_element_context_ids_from_tail(map_id, map_class, position, 1,
+                                                                   total: meta["total"].to_i).first
+          index = element_index(id, meta)
+          next false if index && @storage.lock_held?("map_element:#{map_id}:#{index}") # element alive
+
+          RubyReactor::Map::Dispatcher.queue_rollback_job(meta, map_id, map_class, position, id)
+          true
+        end
+      end
+
+      # From the element's row; nil once the row is gone (the job reports it).
+      def element_index(element_context_id, meta)
+        return nil unless element_context_id
+
+        element_class = RubyReactor::Map::Helpers.resolve_reactor_class(meta["reactor_class_info"])
+        data = @storage.retrieve_context(element_context_id, RubyReactor.reactor_storage_name(element_class))
+        metadata = data && ContextSerializer.deserialize_value(data["map_metadata"])
+        metadata && Utils::FetchIndifferent.call(metadata, :index)
+      end
+
       def missing_indices(meta)
         @storage.missing_map_indices(meta["map_id"], meta["count"].to_i, meta["parent_reactor_class_name"])
       end
@@ -77,22 +126,31 @@ module RubyReactor
       end
 
       # N1: a nested map's parent is a map element running under a `map_element:`
-      # lock, not an `async:` lock. Derive the right key from metadata.
+      # lock, not an `async:` lock. Derive the right key from metadata. Any
+      # other map's completion resumes its OWNER run (009 R-03), so that run's
+      # lock is the one to check; metadata from before 009 names no owner.
       def parent_live_lock?(meta)
         if meta["parent_is_map_element"]
           @storage.lock_held?("map_element:#{meta["outer_map_id"]}:#{meta["outer_index"]}")
         else
-          @storage.lock_held?("async:#{meta["parent_context_id"]}")
+          @storage.lock_held?("async:#{meta["owner_context_id"] || meta["parent_context_id"]}")
         end
       end
 
+      # The map's own context already recorded the step, or the owner run is
+      # past it: finished, or rolling back (it adopted the map).
       def parent_already_collected?(meta)
         data = @storage.retrieve_context(meta["parent_context_id"], meta["parent_reactor_class_name"])
-        return false unless data
+        return true if data && (data["intermediate_results"] || {}).key?(meta["step_name"].to_s)
+
+        owner = owner_context_data(meta) || data
+        owner && %w[completed failed halted skipped cancelled rolling_back].include?(owner["status"].to_s)
+      end
+
+      def owner_context_data(meta)
+        return nil unless meta["owner_context_id"]
 
-        results = data["intermediate_results"] || {}
-        status = data["status"].to_s
-        results.key?(meta["step_name"].to_s) || %w[completed failed halted skipped].include?(status)
+        @storage.retrieve_context(meta["owner_context_id"], meta["owner_reactor_class_name"])
       end
 
       def retrigger_collector(meta)

data/lib/ruby_reactor/map.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    # Element jobs one throw enqueues when a fan-out map declares no
+    # `batch_size` (009 R-10), forward and rollback alike.
+    DEFAULT_BATCH_SIZE = 50
+    # Element contexts an inline map's rollback loads per read (009 R-01).
+    ROLLBACK_CHUNK = 100
+
+    # One structured key=value line (009 FR-011), as `Executor#log_completion`.
+    def self.log(event, **fields)
+      line = { event: event, **fields }.map { |key, value| "#{key}=#{value.inspect}" }.join(" ")
+      RubyReactor.configuration.logger.info(line)
+    end
+  end
+end

data/lib/ruby_reactor/max_retries_exhausted_failure.rb

@@ -7,12 +7,14 @@ module RubyReactor
     # rubocop:disable Metrics/ParameterLists
     def initialize(message, step:, attempts:, original_error: nil,
                    inputs: {}, backtrace: nil, redact_inputs: [],
-                   reactor_name: nil, step_arguments: {}, validation_errors: nil, rollback_failures: nil)
+                   reactor_name: nil, step_arguments: {}, validation_errors: nil, rollback_failures: nil,
+                   exception_class: nil)
       # rubocop:enable Metrics/ParameterLists
       super(message,
             step_name: step, inputs: inputs, backtrace: backtrace,
             redact_inputs: redact_inputs, reactor_name: reactor_name, step_arguments: step_arguments,
-            validation_errors: validation_errors, rollback_failures: rollback_failures)
+            validation_errors: validation_errors, rollback_failures: rollback_failures,
+            exception_class: exception_class)
       @attempts = attempts
       @original_error = original_error
     end