ruby_reactor 0.8.4 → 0.8.6

data/lib/ruby_reactor/map/dispatcher.rb

@@ -6,7 +6,7 @@ module RubyReactor
       extend Helpers
 
       def self.perform(arguments)
-        arguments = arguments.transform_keys(&:to_sym)
+        arguments = Helpers.normalize_arguments(arguments)
         parent_reactor_class_name = arguments[:parent_reactor_class_name]
 
         storage = RubyReactor.configuration.storage_adapter
@@ -64,13 +64,12 @@ module RubyReactor
         map_id = arguments[:map_id]
         reactor_class_name = arguments[:parent_reactor_class_name]
 
-        # Fail Fast Check
-        if arguments[:fail_fast]
-          failed_context_id = storage.retrieve_map_failed_context_id(map_id, reactor_class_name)
-          return if failed_context_id
+        # An atomic map stops dispatching at its first failure.
+        if arguments[:atomic] && storage.retrieve_map_failed_context_id(map_id, reactor_class_name)
+          return settle_undispatched(arguments, storage)
         end
 
-        batch_size = arguments[:batch_size] || source.size # Default to all if no batch_size (async=true only)
+        batch_size = arguments[:batch_size] || RubyReactor::Map::DEFAULT_BATCH_SIZE
 
         # Atomically reserve a batch
         new_offset = storage.increment_map_offset(map_id, batch_size, reactor_class_name)
@@ -104,6 +103,33 @@ module RubyReactor
         end
       end
 
+      # An atomic map stopped dispatching: claim every index not dispatched
+      # yet (one offset bump, so a later dispatcher claims none), settle each
+      # with a `_skipped` slot, count them down, and trigger the collector if
+      # that settled the map. Without this those indices never settle, so the
+      # failure would never be applied (R-04) and the map sweeper would keep
+      # re-dispatching them.
+      def self.settle_undispatched(arguments, storage)
+        map_id = arguments[:map_id]
+        reactor_class_name = arguments[:parent_reactor_class_name]
+        total = storage.retrieve_map_metadata(map_id, reactor_class_name)&.fetch("count", 0).to_i
+        new_offset = storage.increment_map_offset(map_id, total, reactor_class_name)
+        claimed = (new_offset - total)...[new_offset, total].min
+        return if claimed.none?
+
+        claimed.each do |index|
+          storage.store_map_result(map_id, index, { "_skipped" => true }, reactor_class_name,
+                                   strict_ordering: arguments[:strict_ordering])
+        end
+        return unless storage.decrement_map_counter_by(map_id, claimed.size, reactor_class_name) <= 0
+
+        RubyReactor.configuration.async_router.perform_map_collection_async(
+          parent_context_id: arguments[:parent_context_id], map_id: map_id,
+          parent_reactor_class_name: reactor_class_name, step_name: arguments[:step_name].to_s,
+          strict_ordering: arguments[:strict_ordering], timeout: 3600
+        )
+      end
+
       # Re-dispatch a SPECIFIC index whose result slot is missing (Phase 5c, used
       # by the map sweeper). Index-driven rather than offset-driven: resolve the
       # source from the stored parent context and pick source[index]. Idempotent
@@ -120,7 +146,7 @@ module RubyReactor
           strict_ordering: map_meta["strict_ordering"],
           parent_context_id: map_meta["parent_context_id"],
           parent_reactor_class_name: parent_class_name,
-          fail_fast: map_meta["fail_fast"],
+          atomic: map_meta["atomic"],
           batch_size: map_meta["batch_size"]
         }
 
@@ -185,9 +211,40 @@ module RubyReactor
           parent_reactor_class_name: context.reactor_class.name,
           step_name: options[:step_name].to_s,
           batch_size: arguments[:batch_size], # Passed to worker so it knows to trigger next batch?
-          fail_fast: arguments[:fail_fast]
+          atomic: arguments[:atomic]
+        )
+      end
+
+      # One throw of a map's distributed rollback (009 R-02, R-06): claim the
+      # next `batch_size` positions of the element-context index, counted from
+      # its tail, and enqueue one rollback job per position. The job at the
+      # last position of a throw claims the next one, as forward.
+      def self.dispatch_rollback_batch(map_id:, parent_reactor_class_name:)
+        storage = RubyReactor.configuration.storage_adapter
+        meta = storage.retrieve_map_rollback_metadata(map_id, parent_reactor_class_name)
+        return 0 unless meta
+
+        positions = storage.claim_map_rollback_positions(map_id, parent_reactor_class_name, meta["batch_size"].to_i)
+        return 0 if positions.none?
+
+        ids = storage.retrieve_map_element_context_ids_from_tail(map_id, parent_reactor_class_name, positions.first,
+                                                                 positions.size, total: meta["total"].to_i)
+        positions.each_with_index do |position, i|
+          queue_rollback_job(meta, map_id, parent_reactor_class_name, position, ids[i])
+        end
+        positions.size
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def self.queue_rollback_job(meta, map_id, parent_reactor_class_name, position, element_context_id, attempt: 0)
+        RubyReactor.configuration.async_router.perform_map_element_rollback_async(
+          map_id: map_id, position: position, element_context_id: element_context_id,
+          reactor_class_info: meta["reactor_class_info"], parent_reactor_class_name: parent_reactor_class_name,
+          step_name: meta["step_name"], batch_size: meta["batch_size"], owner_context_id: meta["owner_context_id"],
+          owner_reactor_class_name: meta["owner_reactor_class_name"], attempt: attempt
         )
       end
+      # rubocop:enable Metrics/ParameterLists
 
       def self.resolve_reactor_class_info(arguments, context)
         mapped_reactor_class = arguments[:mapped_reactor_class]

data/lib/ruby_reactor/map/element_executor.rb

@@ -6,7 +6,7 @@ module RubyReactor
       extend Helpers
 
       def self.perform(arguments)
-        arguments = arguments.transform_keys(&:to_sym)
+        arguments = Helpers.normalize_arguments(arguments)
 
         # Per-element liveness lock (Phase 5b): its presence is the map sweeper's
         # "element alive" signal, and it serializes duplicate deliveries so a
@@ -55,10 +55,16 @@ module RubyReactor
         context.inline_async_execution = true
 
         storage = RubyReactor.configuration.storage_adapter
-        storage.store_map_element_context_id(arguments[:map_id], context.context_id,
-                                             arguments[:parent_reactor_class_name])
-
-        return if check_fail_fast?(arguments, storage)
+        return if check_atomic?(arguments, storage)
+
+        # Indexed only once it runs: a skipped element saves no context, and
+        # the map's rollback reports an indexed element without one as expired.
+        # Only a fresh context: a parked or retried element re-enters with its
+        # own, already indexed (009 R-06).
+        unless arguments[:serialized_context]
+          storage.store_map_element_context_id(arguments[:map_id], context.context_id,
+                                               arguments[:parent_reactor_class_name])
+        end
 
         executor = Executor.new(context.reactor_class, {}, context)
         begin
@@ -93,7 +99,7 @@ module RubyReactor
           strict_ordering: arguments[:strict_ordering], parent_context_id: arguments[:parent_context_id],
           parent_reactor_class_name: arguments[:parent_reactor_class_name], step_name: arguments[:step_name],
           batch_size: arguments[:batch_size], serialized_context: ContextSerializer.serialize(context),
-          fail_fast: arguments[:fail_fast]
+          atomic: arguments[:atomic]
         )
       end
 
@@ -142,8 +148,8 @@ module RubyReactor
       end
       # rubocop:enable Style/IdenticalConditionalBranches
 
-      def self.check_fail_fast?(arguments, storage)
-        return false unless arguments[:fail_fast]
+      def self.check_atomic?(arguments, storage)
+        return false unless arguments[:atomic]
 
         map_id = arguments[:map_id]
         parent_reactor_class_name = arguments[:parent_reactor_class_name]
@@ -151,7 +157,11 @@ module RubyReactor
         failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name)
         return false unless failed_context_id
 
-        # Skip execution
+        # Skip execution, but settle the index: the collector applies a
+        # atomic map's failure only once every index has a result slot (R-04), and
+        # the map sweeper would otherwise keep re-dispatching this one.
+        storage.store_map_result(map_id, arguments[:index], { "_skipped" => true }, parent_reactor_class_name,
+                                 strict_ordering: arguments[:strict_ordering])
         finalize_execution(arguments, storage)
         true
       end
@@ -177,12 +187,12 @@ module RubyReactor
           # Store the whole serialized Failure, not just its message: step_name,
           # backtrace, file_path and code_snippet are the only record of why this
           # element failed once its context row expires, and the dashboard has
-          # nothing else to show for a non-fail_fast map.
+          # nothing else to show for a non-atomic map.
           storage.store_map_result(map_id, index,
                                    { _error: ContextSerializer.serialize_value(result) }, parent_class,
                                    strict_ordering: arguments[:strict_ordering])
 
-          if arguments[:fail_fast]
+          if arguments[:atomic]
             storage.store_map_failed_context_id(map_id, context.context_id, parent_class)
             # FAST FAIL: Trigger Collector immediately to cancel/fail the map execution
             RubyReactor.configuration.async_router.perform_map_collection_async(

data/lib/ruby_reactor/map/element_rollback.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    # Rolls back ONE element of a map (009 R-07): the element's own undo
+    # stack, replayed under its liveness lock and saved after every undone
+    # entry, so a redelivery resumes after the last one (only the entry that
+    # was cut off may run again). The inline path calls `.call` in process; a
+    # fan-out map's `MapElementRollbackWorker` calls `.perform`.
+    class ElementRollback
+      extend Helpers
+
+      # The element context row is gone: its index went with it.
+      def self.unavailable(step_name)
+        { "index" => nil, "outcome" => "context_unavailable",
+          "failures" => ContextSerializer.serialize_value([entry(step_name, nil, :context_unavailable,
+                                                                 "context expired")]) }
+      end
+
+      # `{ "index", "outcome", "failures" }` (DM §5), with `failures`
+      # serialized. Outcome `contended`, never stored, means the element's lock
+      # was still held after `wait`.
+      def self.call(map_id:, element_context_id:, element_class:, step_name:, wait: Step::MapStep::ELEMENT_LOCK_WAIT)
+        storage_name = RubyReactor.reactor_storage_name(element_class)
+        data = element_context_id && RubyReactor.configuration.storage_adapter.retrieve_context(element_context_id,
+                                                                                                storage_name)
+        return unavailable(step_name) unless data
+
+        element = Context.deserialize_from_retry(data)
+        index = Utils::FetchIndifferent.call(element.map_metadata || {}, :index).to_i
+        lock = acquire_element_lock(map_id, index, wait)
+        return outcome(index, "contended") if lock == :held
+
+        begin
+          roll_back(element, element_class, index, step_name)
+        ensure
+          lock.release if lock.respond_to?(:release)
+        end
+      end
+
+      # A completed element, or an aborted one (an inline run interrupted in
+      # it, which kept the undo entries of the steps it completed), undoes;
+      # a failed one already rolled itself back, and a halted or superseded
+      # running one did nothing to undo.
+      def self.roll_back(element, element_class, index, step_name)
+        return outcome(index, "not_needed") unless %w[completed aborted].include?(element.status.to_s)
+
+        executor = Executor.new(element.reactor_class || element_class, {}, element)
+        executor.undo_all { executor.checkpoint! }
+        executor.checkpoint!
+        tag = { map_step: step_name.to_sym, element_index: index }
+        failures = executor.compensation_manager.rollback_failures.map { |failure| failure.merge(tag) }
+        outcome(index, failures.empty? ? "undone" : "failed", failures)
+      end
+
+      # The job's body. A contended lock requeues the job, storing nothing,
+      # up to `lock_snooze_max_attempts` times before reporting the element
+      # in flight: two contexts of one index in the same batch then serialize
+      # instead of one reporting a false failure (R-07).
+      def self.perform(arguments)
+        args = Helpers.normalize_arguments(arguments)
+        map_id = args[:map_id]
+        map_class = args[:parent_reactor_class_name]
+        storage = RubyReactor.configuration.storage_adapter
+        return if storage.map_rollback_outcome_stored?(map_id, map_class, args[:position])
+
+        attempt = args[:attempt].to_i
+        result = call(map_id: map_id, element_context_id: args[:element_context_id],
+                      element_class: resolve_reactor_class(args[:reactor_class_info]), step_name: args[:step_name],
+                      wait: attempt.zero? ? Step::MapStep::ELEMENT_LOCK_WAIT : 0)
+        if result["outcome"] == "contended"
+          return requeue(args, attempt) unless snooze_exhausted?(attempt)
+
+          result = in_flight(result["index"], args[:step_name])
+        end
+        return unless storage.store_map_rollback_outcome(map_id, map_class, args[:position], result)
+
+        log_outcome(args, result)
+        Dispatcher.dispatch_rollback_batch(map_id: map_id, parent_reactor_class_name: map_class) if batch_end?(args)
+        signal_owner_if_settled(args)
+      end
+
+      def self.in_flight(index, step_name)
+        outcome(index, "element_in_flight",
+                [entry(step_name, index, :element_in_flight, "was still running at rollback time")])
+      end
+
+      def self.snooze_exhausted?(attempt)
+        max = RubyReactor.configuration.lock_snooze_max_attempts
+        max != :infinity && attempt >= max
+      end
+
+      def self.requeue(args, attempt)
+        config = RubyReactor.configuration
+        config.async_router.perform_map_element_rollback_in(
+          Worker.snooze_delay(config, nil), **args.slice(*ROLLBACK_JOB_ARGS), attempt: attempt + 1
+        )
+      end
+
+      # The position that ends a throw claims the next one (R-02).
+      def self.batch_end?(args)
+        batch_size = args[:batch_size].to_i
+        batch_size.positive? && ((args[:position].to_i + 1) % batch_size).zero?
+      end
+
+      # The handshake's job side (R-05): only once the hand-off is saved, so
+      # the owner never resumes from a stale blob — and never in inline job
+      # mode, where the map settles before any hand-off exists.
+      def self.signal_owner_if_settled(args)
+        storage = RubyReactor.configuration.storage_adapter
+        map_id = args[:map_id]
+        map_class = args[:parent_reactor_class_name]
+        meta = storage.retrieve_map_rollback_metadata(map_id, map_class)
+        return unless meta && storage.count_map_rollback_outcomes(map_id, map_class) >= meta["total"].to_i
+        return unless storage.map_rollback_handed_off?(map_id, map_class)
+        return unless storage.claim_map_rollback_signal(map_id, map_class)
+
+        RubyReactor.configuration.async_router.perform_async(args[:owner_context_id], args[:owner_reactor_class_name])
+      end
+
+      def self.log_outcome(args, result)
+        Map.log("ruby_reactor.map.rollback.element",
+                reactor: args[:owner_reactor_class_name], context_id: args[:owner_context_id],
+                map_step: args[:step_name], index: result["index"], outcome: result["outcome"],
+                failures: Array(result["failures"]).size)
+      end
+
+      def self.outcome(index, outcome, failures = [])
+        { "index" => index, "outcome" => outcome, "failures" => ContextSerializer.serialize_value(failures) }
+      end
+
+      # A rollback failure entry for the element as a whole (008 shape).
+      def self.entry(step_name, index, reason, message)
+        step = step_name.to_sym
+        { step: step, kind: :undo, key: nil, reason: reason, map_step: step, element_index: index,
+          message: "map element #{index} #{message}" }
+      end
+
+      def self.acquire_element_lock(map_id, index, wait)
+        return nil if ElementExecutor.inline_testing_mode?
+
+        lock = RubyReactor::Lock.new("map_element:#{map_id}:#{index}", owner: SecureRandom.uuid,
+                                                                       ttl: RubyReactor.configuration.context_lock_ttl,
+                                                                       wait: wait)
+        lock.acquire
+        lock
+      rescue RubyReactor::Lock::AcquisitionError
+        :held
+      end
+
+      ROLLBACK_JOB_ARGS = %i[map_id position element_context_id reactor_class_info parent_reactor_class_name step_name
+                             batch_size owner_context_id owner_reactor_class_name].freeze
+
+      private_class_method :roll_back, :in_flight, :snooze_exhausted?, :requeue, :batch_end?,
+                           :signal_owner_if_settled, :log_outcome, :acquire_element_lock
+    end
+  end
+end

data/lib/ruby_reactor/map/helpers.rb

@@ -4,6 +4,16 @@ module RubyReactor
   module Map
     # Shared helper methods for Map executors
     module Helpers
+      # Job payloads enqueued before 009 carry `fail_fast` where they now carry
+      # `atomic` (R-11, FR-023): the one place the old key is read. Removed at
+      # the next MAJOR.
+      def self.normalize_arguments(arguments)
+        normalized = arguments.to_h.transform_keys(&:to_sym)
+        normalized[:atomic] = normalized[:fail_fast] if !normalized.key?(:atomic) && normalized.key?(:fail_fast)
+        normalized.delete(:fail_fast)
+        normalized
+      end
+
       # Resolves the reactor class from reactor_class_info
       def resolve_reactor_class(info)
         if info["type"] == "class"
@@ -20,6 +30,7 @@ module RubyReactor
           raise "Unknown reactor class info: #{info}"
         end
       end
+      module_function :resolve_reactor_class
 
       # Loads parent context from storage
       def load_parent_context_from_storage(parent_context_id, reactor_class_name, storage)
@@ -31,131 +42,6 @@ module RubyReactor
       def build_element_inputs(mappings, parent_context, element)
         RubyReactor::Step::MapStep.build_mapped_inputs(mappings, parent_context, element)
       end
-
-      # Applies collect block to results
-      def apply_collect_block(results, step_config)
-        collect_block = step_config.arguments[:collect_block][:source].value
-
-        if collect_block
-          # Pass all results (Success and Failure) to collect block
-          begin
-            collected = collect_block.call(results)
-            RubyReactor::Success(collected)
-          rescue StandardError => e
-            RubyReactor::Failure(e)
-          end
-        else
-          # Default behavior: fail if any failure
-          first_failure = results.find(&:failure?)
-          first_failure || RubyReactor::Success(results.map(&:value))
-        end
-      end
-
-      # Resumes parent reactor execution after map completion
-      def resume_parent_execution(parent_context, step_name, final_result, storage) # rubocop:disable Metrics/MethodLength
-        executor = RubyReactor::Executor.new(parent_context.reactor_class, {}, parent_context)
-        step_name_sym = step_name.to_sym
-
-        if final_result.failure?
-          parent_context.current_step = step_name_sym
-
-          error = RubyReactor::Error::StepFailureError.new(
-            final_result.error,
-            step: step_name_sym,
-            context: parent_context,
-            original_error: final_result.error.is_a?(Exception) ? final_result.error : nil,
-            exception_class: final_result.respond_to?(:exception_class) ? final_result.exception_class : nil
-          )
-
-          # Pass backtrace if available
-          if final_result.respond_to?(:backtrace) && final_result.backtrace
-            error.set_backtrace(final_result.backtrace)
-          elsif final_result.error.respond_to?(:backtrace)
-            error.set_backtrace(final_result.error.backtrace)
-          end
-
-          # Bracket the rollback with reactor lifecycle events so that
-          # compensation/undo spans nest under a reactor span (and stay attached
-          # to the originating trace), mirroring the success path's
-          # resume_execution. Without this, rollback runs in the collector worker
-          # with no active reactor span and the undo/compensation spans orphan.
-          executor.middlewares.on(:start_reactor, parent_context.reactor_class.name, parent_context.inputs,
-                                  parent_context)
-          failure_response = nil
-          begin
-            failure_response = executor.result_handler.handle_execution_error(error)
-            # Manually update context status since we're not running executor loop
-            executor.send(:update_context_status, failure_response)
-          ensure
-            executor.middlewares.on(:failed_reactor, parent_context.reactor_class.name, failure_response,
-                                    parent_context)
-          end
-          # This branch runs no executor loop, so nothing else persists the
-          # failed status: store it here.
-          store_parent(parent_context, storage)
-        else
-          parent_context.set_result(step_name_sym, final_result.value)
-
-          # Manually update execution trace to reflect completion
-          # This is necessary because resume_execution continues from the NEXT step
-          # and the async step (which returned DispatchResult) needs to be marked as done with actual value
-          parent_context.append_execution_trace(
-            {
-              type: :result,
-              step: step_name_sym,
-              timestamp: Time.now,
-              value: final_result.value,
-              status: :success
-            }
-          )
-
-          parent_context.current_step = nil
-
-          # Mark the context as a worker before resuming: resume_execution runs
-          # the collector's own worker (this call), not the reactor's normal
-          # caller. Without this, StepExecutor#handoff_at? (gated on
-          # `inline_async_execution`) re-evaluates any later `background`
-          # before:/after: cut point and enqueues a second hand-off, or runs an
-          # `after:` target here in the collector instead of the original
-          # dispatching worker.
-          parent_context.inline_async_execution = true
-          resume_parked_aware(executor, parent_context)
-        end
-      end
-
-      # `resume_execution` persists the parent itself — under the parent's
-      # context lock, and deliberately NOT when it lost that lock to a live
-      # duplicate or replayed an already-terminal run. Storing again here, after
-      # the lock is released, would overwrite whatever the lock's holder wrote
-      # (single writer), so the resume's own save is the only one.
-      #
-      # This collector is a worker running the parent's execution, so it is
-      # also a final handler for park signals (005 R-01), like `Worker` and
-      # `ElementExecutor`: the resume has already parked the parent's holds and
-      # saved; hand the rest back to the parent's own worker.
-      #
-      # The parent is loaded from its own blob, so it has no `root_context`:
-      # a map inside a composed child resumes and requeues that child as its
-      # own execution, never its root. ponytail: the root is never resumed
-      # after such a map (already so on main); see "Fan-out map inside a
-      # composed child" in specs/future_improvements.md.
-      def resume_parked_aware(executor, parent_context)
-        executor.resume_execution
-      rescue RubyReactor::Error::ExecutionParked => e
-        config = RubyReactor.configuration
-        config.async_router.perform_in(
-          RubyReactor::Worker.snooze_delay(config, e), parent_context.context_id,
-          RubyReactor.reactor_storage_name(parent_context.reactor_class)
-        )
-      end
-
-      def store_parent(parent_context, storage)
-        storage.store_context(
-          parent_context.context_id,
-          ContextSerializer.serialize(parent_context),
-          RubyReactor.reactor_storage_name(parent_context.reactor_class)
-        )
-      end
     end
   end
 end

data/lib/ruby_reactor/map/result_enumerator.rb

@@ -22,7 +22,8 @@ module RubyReactor
 
         if @strict_ordering
           count.times do |i|
-            yield self[i]
+            raw = raw_at(i)
+            yield raw && wrap_result(raw) unless skipped?(raw)
           end
         else
           offset = 0
@@ -37,7 +38,7 @@ module RubyReactor
 
             break if results.empty?
 
-            results.each { |result| yield wrap_result(result) }
+            results.each { |result| yield wrap_result(result) unless skipped?(result) }
 
             offset += results.size
             break if results.size < @batch_size
@@ -59,21 +60,15 @@ module RubyReactor
         !empty?
       end
 
+      # nil for an index a fail-fast map skipped (never started).
       def [](index)
         index += count if index.negative?
         return nil if index.negative? || index >= count
 
-        results = @storage.retrieve_map_results_batch(
-          @map_id,
-          @reactor_class_name,
-          offset: index,
-          limit: 1,
-          strict_ordering: @strict_ordering
-        )
+        raw = raw_at(index)
+        return nil if raw.nil? || skipped?(raw)
 
-        return nil if results.empty?
-
-        wrap_result(results.first)
+        wrap_result(raw)
       end
 
       def first
@@ -94,6 +89,17 @@ module RubyReactor
 
       private
 
+      def raw_at(index)
+        @storage.retrieve_map_results_batch(@map_id, @reactor_class_name, offset: index, limit: 1,
+                                                                          strict_ordering: @strict_ordering).first
+      end
+
+      # A `_skipped` slot settles an index a fail-fast map never ran (R-04).
+      # Such a map is never collected as a success, so no consumer sees one.
+      def skipped?(result)
+        result.is_a?(Hash) && result.key?("_skipped")
+      end
+
       def wrap_result(result)
         if result.is_a?(Hash) && result.key?("_error")
           # `backtrace: []` is load-bearing: without it Failure falls back to