ruby_reactor 0.8.5 → 0.8.7

data/lib/ruby_reactor/executor.rb

@@ -135,6 +135,10 @@ module RubyReactor
       handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
       completed = true
       @result
+    rescue Error::RollbackHandedOff => e
+      record_rollback_handoff(e)
+      completed = true
+      @result
     rescue RubyReactor::Lock::AcquisitionError,
            RubyReactor::Semaphore::AcquisitionError,
            RubyReactor::RateLimit::ExceededError,
@@ -153,7 +157,7 @@ module RubyReactor
       @contention_snooze = true
       raise
     rescue Error::Rescuable => e
-      @result = aborting_on_interruption { @result_handler.handle_execution_error(e) }
+      @result = handing_off { aborting_on_interruption { @result_handler.handle_execution_error(e) } }
       update_context_status(@result)
       completed = true
       @result
@@ -184,6 +188,9 @@ module RubyReactor
     end
 
     def resume_execution # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity
+      # A composed child re-entered while its own rollback is handed off.
+      return resume_rollback if @context.rolling_back?
+
       middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
       completed = false
 
@@ -256,6 +263,10 @@ module RubyReactor
       handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
       completed = true
       @result
+    rescue Error::RollbackHandedOff => e
+      record_rollback_handoff(e)
+      completed = true
+      @result
     rescue RubyReactor::Lock::AcquisitionError,
            RubyReactor::Semaphore::AcquisitionError,
            RubyReactor::RateLimit::ExceededError,
@@ -274,7 +285,7 @@ module RubyReactor
       @contention_snooze = true
       raise e
     rescue Error::Rescuable => e
-      aborting_on_interruption { handle_resume_error(e) }
+      handing_off { aborting_on_interruption { handle_resume_error(e) } }
       update_context_status(@result)
       completed = true
       @result
@@ -283,16 +294,100 @@ module RubyReactor
       raise
     ensure
       release_locks unless @parked
+      # Saved while the context lock is still held (009 R-13): released first,
+      # a worker resuming this run could load the pre-save blob, and this save
+      # would then overwrite its progress.
+      save_context unless skip_context_persist?
       @acquired_context_lock&.release
       @acquired_context_lock = nil
       leave_ordered_lock_scope
+
+      emit_lifecycle_completion(completed)
+    end
+
+    # Undoes every completed step (manual undo, a composed child's undo, a
+    # map element's rollback). A rollback that hands off here (009 R-04)
+    # leaves this level's rollback failures on its context, and the next
+    # `undo_all` of the same context picks them up, so none is lost though
+    # the executor is rebuilt on resume (G2).
+    def undo_all(&after_pop)
+      saved = @context.rollback && @context.rollback["failures"]
+      @compensation_manager.restore_rollback_failures(saved) if saved
+      @compensation_manager.rollback_completed_steps(&after_pop)
+      clear_saved_rollback_failures
+    rescue Error::RollbackHandedOff
+      (@context.rollback ||= {})["failures"] =
+        ContextSerializer.serialize_value(@compensation_manager.rollback_failures)
+      raise
+    end
+
+    # Finishes a rollback that handed off at a fan-out map (009 R-04), once
+    # the map's element rollbacks have all reported: the Worker of a
+    # `rolling_back` run, or a composed child re-entered by its root. Under
+    # the run's context lock, as `resume_execution`.
+    def resume_rollback
+      state = @context.rollback || {}
+      # The hand-off came out of a running step (a composed child rolling
+      # back inside `ComposeStep#run`): resume forward, and that step
+      # finishes its child's rollback.
+      if state["trigger"] == "step" || state.empty?
+        @context.rollback = nil
+        @context.status = :running
+        return resume_execution
+      end
+
+      middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
+      completed = false
+      enter_ordered_lock_scope
+      acquire_context_lock
+      @result = finish_rollback(state)
+      @context.rollback = nil
+      update_context_status(@result)
+      completed = true
+      @result
+    rescue Error::RollbackHandedOff => e
+      record_rollback_handoff(e)
+      completed = true
+      @result
+    rescue RubyReactor::Lock::AcquisitionError => e
+      @contention_snooze = true
+      raise composed_contention_park(e) || e
+    rescue Error::Rescuable => e
+      handing_off { aborting_on_interruption { handle_resume_error(e) } }
+      update_context_status(@result)
+      completed = true
+      @result
+    rescue Exception # rubocop:disable Lint/RescueException
+      mark_aborted
+      raise
+    ensure
       save_context unless skip_context_persist?
+      @acquired_context_lock&.release
+      @acquired_context_lock = nil
+      leave_ordered_lock_scope
 
       emit_lifecycle_completion(completed)
     end
 
-    def undo_all
-      @compensation_manager.rollback_completed_steps
+    # The rollback's other half (009 R-05): marks the hand-off saved, then
+    # enqueues the owner's Worker if the map rollback already settled and this
+    # side claims the signal. Whichever of this and the last element rollback
+    # job sees both second enqueues the owner, once. Called right after the
+    # `rolling_back` save, while the context lock is still held.
+    def hand_off_rollback!(handed_off)
+      storage = RubyReactor.configuration.storage_adapter
+      map_id = handed_off.map_id
+      map_class = handed_off.reactor_class_name
+      storage.mark_map_rollback_handed_off(map_id, map_class)
+      RubyReactor.configuration.logger.info(
+        "event=ruby_reactor.rollback.handed_off context_id=#{@context.context_id.inspect} map_id=#{map_id.inspect}"
+      )
+      meta = storage.retrieve_map_rollback_metadata(map_id, map_class)
+      return unless meta && storage.count_map_rollback_outcomes(map_id, map_class) >= meta["total"].to_i
+      return unless storage.claim_map_rollback_signal(map_id, map_class)
+
+      RubyReactor.configuration.async_router.perform_async(@context.context_id,
+                                                           RubyReactor.reactor_storage_name(@reactor_class))
     end
 
     # True once `resume_execution` is past its reactor-level gates (context
@@ -318,10 +413,18 @@ module RubyReactor
     def aborting_on_interruption
       yield
     rescue Exception => e # rubocop:disable Lint/RescueException
-      mark_aborted unless Error::Rescuable === e # rubocop:disable Style/CaseEquality
+      mark_aborted unless Error::Rescuable === e || e.is_a?(Error::RollbackHandedOff) # rubocop:disable Style/CaseEquality
       raise
     end
 
+    # A rollback run from a `rescue Error::Rescuable` body (the error's own
+    # rollback) is outside the sibling `rescue Error::RollbackHandedOff`.
+    def handing_off
+      yield
+    rescue Error::RollbackHandedOff => e
+      record_rollback_handoff(e)
+    end
+
     def undo_stack
       @compensation_manager.undo_stack
     end
@@ -768,7 +871,8 @@ module RubyReactor
 
       case result
       when RubyReactor::DispatchResult
-        @context.status = :running
+        # A rollback hand-off returns one too, and stays `rolling_back`.
+        @context.status = :running unless @context.rolling_back?
       when RubyReactor::Halt
         @context.status = :halted
       when RubyReactor::Success
@@ -833,6 +937,82 @@ module RubyReactor
       @result
     end
 
+    # Where this level's rollback stands when it hands off (009 DM §1). A
+    # composed child keeps it on its own context, inside the root's blob, and
+    # re-raises; the top-level run saves it as `rolling_back` while it still
+    # holds the context lock (I-6), runs the handshake, and returns a
+    # `DispatchResult`, as a forward hand-off does.
+    def record_rollback_handoff(handed_off)
+      failure = handed_off.failure
+      handed_off.failure = nil # this level's Failure; the level above builds its own
+      @context.rollback = rollback_state(failure)
+      @context.status = :rolling_back
+      raise handed_off if @context.root_context
+
+      save_context
+      hand_off_rollback!(handed_off)
+      @result = RubyReactor::DispatchResult.new(job_id: "map_rollback:#{handed_off.map_id}",
+                                                intermediate_results: @context.intermediate_results,
+                                                execution_id: @context.context_id)
+    end
+
+    # `trigger`: `failure` (a step failure is rolling back), `undo` (set by
+    # `Reactor#undo`), or `step` (a running step's child handed off, and no
+    # rollback started at this level). A later hand-off of the same rollback
+    # keeps what an earlier one recorded.
+    def rollback_state(failure)
+      state = (@context.rollback || {}).dup
+      pending = @compensation_manager.pending
+      state["trigger"] ||= failure || pending ? "failure" : "step"
+      if pending
+        state["step"] = pending[:step].to_s
+        state["compensated"] = pending[:compensated]
+        state["compensation_error"] = pending[:compensation_error]
+      end
+      state["failure"] = ContextSerializer.serialize_value(failure) if failure
+      state["failures"] = ContextSerializer.serialize_value(@compensation_manager.rollback_failures)
+      state
+    end
+
+    # The rest of a handed-off rollback: the failing step's own compensate if
+    # it had not finished (a map or compose adopts its settled state), the
+    # steps still on the undo stack, then the outcome the inline rollback
+    # gives (I-7).
+    def finish_rollback(state)
+      @compensation_manager.restore_rollback_failures(state["failures"])
+      return finish_undo if state["trigger"] == "undo"
+
+      failure = ContextSerializer.deserialize_value(state["failure"])
+      step_config = state["step"] && @reactor_class.steps[state["step"].to_sym]
+      if step_config && !state["compensated"]
+        @compensation_manager.handle_step_failure(step_config, failure.error, {})
+      else
+        @compensation_manager.rollback_completed_steps
+        if state["compensation_error"]
+          raise Error::CompensationError.new(state["compensation_error"], step: step_config&.name, context: @context)
+        end
+      end
+      @context.current_step = failure.step_name&.to_sym
+      @result_handler.handed_off_failure(failure)
+    rescue Error::CompensationError => e
+      @result_handler.handle_execution_error(e)
+    end
+
+    def finish_undo
+      @compensation_manager.rollback_completed_steps
+      @context.cancelled = true
+      @context.cancellation_reason = "Undo triggered"
+      @context.status = "cancelled"
+      nil
+    end
+
+    def clear_saved_rollback_failures
+      return unless @context.rollback
+
+      @context.rollback.delete("failures")
+      @context.rollback = nil if @context.rollback.empty?
+    end
+
     def handle_interrupt(interrupt_result)
       save_context
 

data/lib/ruby_reactor/map/collector.rb

@@ -3,8 +3,6 @@
 module RubyReactor
   module Map
     class Collector
-      extend Helpers
-
       # Seconds a collector waits for another collector of the same map. One
       # that found the map unsettled releases within milliseconds, and may be
       # the only trigger left for a failure it just deferred (R-04); a holder
@@ -12,7 +10,7 @@ module RubyReactor
       COLLECT_LOCK_WAIT = 2
 
       def self.perform(arguments)
-        arguments = arguments.transform_keys(&:to_sym)
+        arguments = Helpers.normalize_arguments(arguments)
         map_id = arguments[:map_id]
 
         # Serialize concurrent collector deliveries for the SAME map (eager queue +
@@ -48,105 +46,39 @@ module RubyReactor
         defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
       end
 
+      # Once every index has a result slot (a value, `_error`, `_halt` or
+      # `_skipped`), signal the map's OWNER run — the top-level context —
+      # once, and write nothing else (009 R-03, I-1). The owner's Worker
+      # resumes, and `MapStep#run` adopts the settled outcome at any
+      # composition depth. An atomic map's failure is applied the same way
+      # only once every index has settled, so its compensate sees every
+      # element that completed, including ones still in flight when it
+      # failed. Until then the last element to settle, or the map sweeper,
+      # re-triggers this collector.
       def self.perform_collection(arguments)
         map_id = arguments[:map_id]
-        parent_context_id = arguments[:parent_context_id]
         parent_reactor_class_name = arguments[:parent_reactor_class_name]
-        step_name = arguments[:step_name]
-        strict_ordering = arguments[:strict_ordering]
-        # timeout = arguments[:timeout]
-
         storage = RubyReactor.configuration.storage_adapter
-        parent_context_data = storage.retrieve_context(parent_context_id, parent_reactor_class_name)
-        parent_context = RubyReactor::Context.deserialize_from_retry(parent_context_data)
-
-        # Idempotency: if the parent already recorded this map step's result, a
-        # prior collector already resumed it. Re-resuming would double-execute the
-        # steps after the map. Skip. A parent already finished (a prior
-        # collector applied this map's failure) must not be rolled back twice.
-        return if parent_context.intermediate_results.key?(step_name.to_sym) || parent_context.finished?
 
-        # Check if all tasks are completed
         metadata = storage.retrieve_map_metadata(map_id, parent_reactor_class_name)
-        total_count = metadata ? metadata["count"].to_i : 0
+        return unless metadata
+        # Judged against the element count, not map_offset (which batching
+        # reservation can push past it).
+        return if storage.count_map_results(map_id, parent_reactor_class_name) < metadata["count"].to_i
+        return unless storage.claim_map_owner_signal(map_id, parent_reactor_class_name)
 
-        results_count = storage.count_map_results(map_id, parent_reactor_class_name)
-
-        # Completion is judged against the total count of elements, not
-        # map_offset (which batching reservation can push past it).
-        #
-        # A fail-fast failure is applied only once every index has settled
-        # (a result, `_error`, `_halt` or `_skipped` slot), so the map's
-        # compensate sees every element that completed — including ones still
-        # in flight when the failure happened (R-04). Until then the last
-        # element to settle, or the map sweeper, re-triggers this collector.
-        return if results_count < total_count
-
-        if (failed_context_id = storage.retrieve_map_failed_context_id(map_id, parent_reactor_class_name))
-          handle_failure(failed_context_id, metadata, storage, parent_context, step_name)
-          return
-        end
-
-        # Retrieve results lazily
-        results = RubyReactor::Map::ResultEnumerator.new(
-          map_id,
-          parent_reactor_class_name,
-          strict_ordering: strict_ordering
-        )
-
-        # Apply collect block (or default collection)
-        step_config = parent_context.reactor_class.steps[step_name.to_sym]
-
-        begin
-          final_result = apply_collect_block(results, step_config)
-
-          if final_result.failure?
-            # Optionally log failure internally or just rely on context status update
-          end
-        rescue StandardError => e
-          final_result = RubyReactor::Failure(e)
-        end
-
-        # Resume parent execution
-        resume_parent_execution(parent_context, step_name, final_result, storage)
-      rescue StandardError => e
-        RubyReactor.configuration.logger.error("Map collector crashed: #{e.message}")
-        RubyReactor.configuration.logger.error(e.backtrace.join("\n")) if e.backtrace
-        raise e
-      end
-
-      def self.apply_collect_block(results, step_config)
-        collect_block = step_config.arguments[:collect_block][:source].value if step_config.arguments[:collect_block]
-        # TODO: Check allow_partial_failure option
-
-        if collect_block
-          begin
-            # Pass Enumerator to collect block
-            collected = collect_block.call(results)
-            RubyReactor::Success(collected)
-          rescue RubyReactor::Error::Rescuable => e
-            RubyReactor.configuration.logger.error("Map collect block raised: #{e.message}")
-            RubyReactor.configuration.logger.error(e.backtrace.join("\n")) if e.backtrace
-            RubyReactor::Failure(e)
-          end
-        else
-          # Default behavior: Return Success(Enumerator).
-          # Logic for checking failures is deferred to the consumer of the enumerator.
-          RubyReactor::Success(results)
-        end
+        signal_owner(metadata, arguments)
       end
 
-      def self.handle_failure(failed_context_id, metadata, storage, parent_context, step_name)
-        # Resolve the class of the mapped reactor to retrieve its context
-        reactor_class = resolve_reactor_class(metadata["reactor_class_info"])
-        failed_context_data = storage.retrieve_context(failed_context_id, reactor_class.name)
-
-        return unless failed_context_data
-
-        failed_context = RubyReactor::Context.deserialize_from_retry(failed_context_data)
-        reason = failed_context.failure_reason
-        result = reason.is_a?(RubyReactor::Failure) ? reason : RubyReactor::Failure(reason)
-        resume_parent_execution(parent_context, step_name, result, storage)
+      # Metadata written before the upgrade names no owner (S-6): the parent is
+      # the owner of a root-level map.
+      def self.signal_owner(metadata, arguments)
+        owner_id = metadata["owner_context_id"] || metadata["parent_context_id"] || arguments[:parent_context_id]
+        owner_class = metadata["owner_reactor_class_name"] || metadata["parent_reactor_class_name"]
+        RubyReactor.configuration.logger.info(
+          "event=ruby_reactor.map.settled map_id=#{arguments[:map_id].inspect} owner=#{owner_id.inspect}"
+        )
+        RubyReactor.configuration.async_router.perform_async(owner_id, owner_class)
       end
     end
   end

data/lib/ruby_reactor/map/dispatcher.rb

@@ -6,7 +6,7 @@ module RubyReactor
       extend Helpers
 
       def self.perform(arguments)
-        arguments = arguments.transform_keys(&:to_sym)
+        arguments = Helpers.normalize_arguments(arguments)
         parent_reactor_class_name = arguments[:parent_reactor_class_name]
 
         storage = RubyReactor.configuration.storage_adapter
@@ -64,12 +64,12 @@ module RubyReactor
         map_id = arguments[:map_id]
         reactor_class_name = arguments[:parent_reactor_class_name]
 
-        # Fail Fast Check
-        if arguments[:fail_fast] && storage.retrieve_map_failed_context_id(map_id, reactor_class_name)
+        # An atomic map stops dispatching at its first failure.
+        if arguments[:atomic] && storage.retrieve_map_failed_context_id(map_id, reactor_class_name)
           return settle_undispatched(arguments, storage)
         end
 
-        batch_size = arguments[:batch_size] || source.size # Default to all if no batch_size (async=true only)
+        batch_size = arguments[:batch_size] || RubyReactor::Map::DEFAULT_BATCH_SIZE
 
         # Atomically reserve a batch
         new_offset = storage.increment_map_offset(map_id, batch_size, reactor_class_name)
@@ -103,7 +103,7 @@ module RubyReactor
         end
       end
 
-      # A fail-fast map stopped dispatching: claim every index not dispatched
+      # An atomic map stopped dispatching: claim every index not dispatched
       # yet (one offset bump, so a later dispatcher claims none), settle each
       # with a `_skipped` slot, count them down, and trigger the collector if
       # that settled the map. Without this those indices never settle, so the
@@ -146,7 +146,7 @@ module RubyReactor
           strict_ordering: map_meta["strict_ordering"],
           parent_context_id: map_meta["parent_context_id"],
           parent_reactor_class_name: parent_class_name,
-          fail_fast: map_meta["fail_fast"],
+          atomic: map_meta["atomic"],
           batch_size: map_meta["batch_size"]
         }
 
@@ -211,10 +211,41 @@ module RubyReactor
           parent_reactor_class_name: context.reactor_class.name,
           step_name: options[:step_name].to_s,
           batch_size: arguments[:batch_size], # Passed to worker so it knows to trigger next batch?
-          fail_fast: arguments[:fail_fast]
+          atomic: arguments[:atomic]
         )
       end
 
+      # One throw of a map's distributed rollback (009 R-02, R-06): claim the
+      # next `batch_size` positions of the element-context index, counted from
+      # its tail, and enqueue one rollback job per position. The job at the
+      # last position of a throw claims the next one, as forward.
+      def self.dispatch_rollback_batch(map_id:, parent_reactor_class_name:)
+        storage = RubyReactor.configuration.storage_adapter
+        meta = storage.retrieve_map_rollback_metadata(map_id, parent_reactor_class_name)
+        return 0 unless meta
+
+        positions = storage.claim_map_rollback_positions(map_id, parent_reactor_class_name, meta["batch_size"].to_i)
+        return 0 if positions.none?
+
+        ids = storage.retrieve_map_element_context_ids_from_tail(map_id, parent_reactor_class_name, positions.first,
+                                                                 positions.size, total: meta["total"].to_i)
+        positions.each_with_index do |position, i|
+          queue_rollback_job(meta, map_id, parent_reactor_class_name, position, ids[i])
+        end
+        positions.size
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def self.queue_rollback_job(meta, map_id, parent_reactor_class_name, position, element_context_id, attempt: 0)
+        RubyReactor.configuration.async_router.perform_map_element_rollback_async(
+          map_id: map_id, position: position, element_context_id: element_context_id,
+          reactor_class_info: meta["reactor_class_info"], parent_reactor_class_name: parent_reactor_class_name,
+          step_name: meta["step_name"], batch_size: meta["batch_size"], owner_context_id: meta["owner_context_id"],
+          owner_reactor_class_name: meta["owner_reactor_class_name"], attempt: attempt
+        )
+      end
+      # rubocop:enable Metrics/ParameterLists
+
       def self.resolve_reactor_class_info(arguments, context)
         mapped_reactor_class = arguments[:mapped_reactor_class]
         step_name = arguments[:step_name]

data/lib/ruby_reactor/map/element_executor.rb

@@ -6,7 +6,7 @@ module RubyReactor
       extend Helpers
 
       def self.perform(arguments)
-        arguments = arguments.transform_keys(&:to_sym)
+        arguments = Helpers.normalize_arguments(arguments)
 
         # Per-element liveness lock (Phase 5b): its presence is the map sweeper's
         # "element alive" signal, and it serializes duplicate deliveries so a
@@ -55,12 +55,16 @@ module RubyReactor
         context.inline_async_execution = true
 
         storage = RubyReactor.configuration.storage_adapter
-        return if check_fail_fast?(arguments, storage)
+        return if check_atomic?(arguments, storage)
 
         # Indexed only once it runs: a skipped element saves no context, and
         # the map's rollback reports an indexed element without one as expired.
-        storage.store_map_element_context_id(arguments[:map_id], context.context_id,
-                                             arguments[:parent_reactor_class_name])
+        # Only a fresh context: a parked or retried element re-enters with its
+        # own, already indexed (009 R-06).
+        unless arguments[:serialized_context]
+          storage.store_map_element_context_id(arguments[:map_id], context.context_id,
+                                               arguments[:parent_reactor_class_name])
+        end
 
         executor = Executor.new(context.reactor_class, {}, context)
         begin
@@ -95,7 +99,7 @@ module RubyReactor
           strict_ordering: arguments[:strict_ordering], parent_context_id: arguments[:parent_context_id],
           parent_reactor_class_name: arguments[:parent_reactor_class_name], step_name: arguments[:step_name],
           batch_size: arguments[:batch_size], serialized_context: ContextSerializer.serialize(context),
-          fail_fast: arguments[:fail_fast]
+          atomic: arguments[:atomic]
         )
       end
 
@@ -144,8 +148,8 @@ module RubyReactor
       end
       # rubocop:enable Style/IdenticalConditionalBranches
 
-      def self.check_fail_fast?(arguments, storage)
-        return false unless arguments[:fail_fast]
+      def self.check_atomic?(arguments, storage)
+        return false unless arguments[:atomic]
 
         map_id = arguments[:map_id]
         parent_reactor_class_name = arguments[:parent_reactor_class_name]
@@ -154,7 +158,7 @@ module RubyReactor
         return false unless failed_context_id
 
         # Skip execution, but settle the index: the collector applies a
-        # fail-fast failure only once every index has a result slot (R-04), and
+        # atomic map's failure only once every index has a result slot (R-04), and
         # the map sweeper would otherwise keep re-dispatching this one.
         storage.store_map_result(map_id, arguments[:index], { "_skipped" => true }, parent_reactor_class_name,
                                  strict_ordering: arguments[:strict_ordering])
@@ -183,12 +187,12 @@ module RubyReactor
           # Store the whole serialized Failure, not just its message: step_name,
           # backtrace, file_path and code_snippet are the only record of why this
           # element failed once its context row expires, and the dashboard has
-          # nothing else to show for a non-fail_fast map.
+          # nothing else to show for a non-atomic map.
           storage.store_map_result(map_id, index,
                                    { _error: ContextSerializer.serialize_value(result) }, parent_class,
                                    strict_ordering: arguments[:strict_ordering])
 
-          if arguments[:fail_fast]
+          if arguments[:atomic]
             storage.store_map_failed_context_id(map_id, context.context_id, parent_class)
             # FAST FAIL: Trigger Collector immediately to cancel/fail the map execution
             RubyReactor.configuration.async_router.perform_map_collection_async(