ruby_reactor 0.8.2 → 0.8.3

data/lib/ruby_reactor/executor/compensation_manager.rb

@@ -3,22 +3,49 @@
 module RubyReactor
   class Executor
     class CompensationManager
+      # Raised ONLY by the step's own coordination, before its body: a
+      # coordination error raised from inside a body (a nested direct
+      # `Step.run`) arrives as `StepCoordination::NestedCoordinationError`, and
+      # a bare `Lock::AcquisitionError` etc. may come from a nested
+      # `Reactor.run` — both mean the body ran, so neither is listed here.
+      NEVER_STARTED_ERROR_CLASSES = [
+        RubyReactor::Executor::StepCoordination::Contended,
+        RubyReactor::Executor::StepCoordination::KeyError,
+        RubyReactor::Executor::StepCoordination::DispatchRefused
+      ].freeze
+
       def initialize(context)
         @context = context
         @undo_trace = []
+        @rollback_failures = []
       end
 
       def undo_stack
         @context.undo_stack
       end
 
-      attr_reader :undo_trace
+      # Every undo/compensation that did not complete, in rollback order
+      # (005 FR-004). `ResultHandler` attaches it to the final Failure.
+      attr_reader :undo_trace, :rollback_failures
 
       def add_to_undo_stack(step_info)
         @context.undo_stack << step_info
       end
 
       def handle_step_failure(step_config, error, arguments)
+        # A step whose OWN coordination acquisition failed (contention, or a
+        # bad key proc) never ran its body — "no step compensates, the
+        # contended step's work has not been attempted" (US3-1/T018), which
+        # applies here exactly as it does to a worker park: compensating a
+        # step that never started is meaningless, and attempting one would
+        # try to re-acquire the very key that is (usually) still contended,
+        # turning a plain contention failure into a confusing
+        # CompensationError. Prior steps still roll back normally.
+        if step_never_started?(error)
+          rollback_completed_steps
+          return RubyReactor.Failure("Step '#{step_config.name}' failed: #{error}")
+        end
+
         # Try compensation
         compensation_result = compensate_step(step_config, error, arguments)
         case compensation_result
@@ -70,16 +97,38 @@ module RubyReactor
         result.respond_to?(:skipped?) && result.skipped?
       end
 
+      def step_never_started?(error)
+        NEVER_STARTED_ERROR_CLASSES.any? { |klass| error.is_a?(klass) }
+      end
+
+      # US6/T047: re-take a step's own lock/semaphore around its compensate
+      # or undo body, so a concurrent forward execution cannot enter the
+      # step's critical section while rollback is undoing what it protected.
+      # A no-op when the step declares no coordination.
+      def coordinated_rollback(step_config, arguments, &block)
+        return block.call if Executor::StepCoordination.none?(step_config)
+
+        # The undo stack stores the RESOLVED arguments; the forward key was
+        # computed from `coordination_arguments` of them, so rollback does the
+        # same and re-takes the very key the forward run held.
+        Executor::StepCoordination.new(
+          step_config: step_config, arguments: step_config.coordination_arguments(arguments, @context.inputs),
+          context: @context, reactor_class: @context.reactor_class, middlewares: middlewares
+        ).around_rollback(&block)
+      end
+
       def compensate_step(step_config, error, arguments)
         middlewares.on(:start_compensation, step_config.name, error, arguments, @context)
         begin
-          compensate_result = catch(StepSignals::TAG) do
-            if step_config.compensate_block
-              step_config.compensate_block.call(error, arguments, @context)
-            elsif step_config.has_impl?
-              step_config.impl.compensate(error, arguments, @context)
-            else
-              RubyReactor.Skipped() # Default: nothing defined, rollback continues
+          compensate_result = coordinated_rollback(step_config, arguments) do
+            catch(StepSignals::TAG) do
+              if step_config.compensate_block
+                step_config.compensate_block.call(error, arguments, @context)
+              elsif step_config.has_impl?
+                step_config.impl.compensate(error, arguments, @context)
+              else
+                RubyReactor.Skipped() # Default: nothing defined, rollback continues
+              end
             end
           end
 
@@ -96,6 +145,7 @@ module RubyReactor
           @undo_trace << { type: :compensation, step: step_config.name, error: error, arguments: arguments }
 
           if compensate_result.is_a?(RubyReactor::Failure)
+            record_rollback_failure(step_config.name, :compensate, compensate_result)
             middlewares.on(:failed_compensation, step_config.name, compensate_result, @context)
           else
             middlewares.on(:complete_compensation, step_config.name, compensate_result, @context)
@@ -103,21 +153,27 @@ module RubyReactor
 
           compensate_result
         rescue StandardError => e
+          record_rollback_failure(step_config.name, :compensate, e)
           middlewares.on(:failed_compensation, step_config.name, e, @context)
-          raise e
+          # A raise is a compensation failure like a returned Failure: the
+          # caller still rolls back the completed steps, then raises
+          # `CompensationError`. Re-raising here skipped both.
+          RubyReactor.Failure(e)
         end
       end
 
-      def undo_step(step_config, result, arguments)
+      def undo_step(step_config, result, arguments) # rubocop:disable Metrics/MethodLength
         middlewares.on(:start_undo, step_config.name, result, arguments, @context)
         begin
-          undo_result = catch(StepSignals::TAG) do
-            if step_config.undo_block
-              step_config.undo_block.call(result.value, arguments, @context)
-            elsif step_config.has_impl?
-              step_config.impl.undo(result.value, arguments, @context)
-            else
-              RubyReactor.Skipped() # Default: nothing defined, rollback continues
+          undo_result = coordinated_rollback(step_config, arguments) do
+            catch(StepSignals::TAG) do
+              if step_config.undo_block
+                step_config.undo_block.call(result.value, arguments, @context)
+              elsif step_config.has_impl?
+                step_config.impl.undo(result.value, arguments, @context)
+              else
+                RubyReactor.Skipped() # Default: nothing defined, rollback continues
+              end
             end
           end
 
@@ -133,6 +189,7 @@ module RubyReactor
           )
 
           if undo_result.is_a?(RubyReactor::Failure)
+            record_rollback_failure(step_config.name, :undo, undo_result)
             middlewares.on(:failed_undo, step_config.name, undo_result, @context)
           else
             middlewares.on(:complete_undo, step_config.name, undo_result, @context)
@@ -140,6 +197,7 @@ module RubyReactor
 
           undo_result
         rescue StandardError => e
+          record_rollback_failure(step_config.name, :undo, e)
           middlewares.on(:failed_undo, step_config.name, e, @context)
           # Log undo failure but don't halt the rollback process
           @context.append_execution_trace(
@@ -148,6 +206,30 @@ module RubyReactor
           RubyReactor.Failure(e)
         end
       end
+
+      # `outcome` is the Failure an undo/compensate returned, or the exception
+      # it raised. A composed child's Failure already carries its own list —
+      # flatten it instead of adding one opaque entry for the compose step.
+      def record_rollback_failure(step_name, kind, outcome)
+        if outcome.is_a?(RubyReactor::Failure) && outcome.rollback_failures.any?
+          @rollback_failures.concat(outcome.rollback_failures)
+          return
+        end
+
+        error = outcome.is_a?(RubyReactor::Failure) ? outcome.error : outcome
+        contended = error.is_a?(StepCoordination::Contended)
+        reason = if contended
+                   :coordination_unavailable
+                 elsif outcome.is_a?(Exception)
+                   :raised
+                 else
+                   :returned_failure
+                 end
+        @rollback_failures << {
+          step: step_name.to_sym, kind: kind, key: (error.key if contended), reason: reason,
+          message: error.respond_to?(:message) ? error.message : error.to_s
+        }
+      end
     end
   end
 end

data/lib/ruby_reactor/executor/ordered_lock_support.rb

@@ -45,27 +45,39 @@ module RubyReactor
         }
       end
 
-      # Strict-ordering gate. Runs BEFORE rate-limit / lock / semaphore so a
-      # waiting nonce never holds any other primitive — preventing
-      # hold-and-wait deadlocks when `with_lock` and `with_ordered_lock`
-      # share inputs. Raises {OrderedLock::WaitError}; the Sidekiq worker
-      # rescues and snoozes.
-      def check_ordered_lock_gate
-        info = ordered_lock_info
-        return :go unless info
-
-        OrderedLock.new(
-          info.fetch(:key),
-          nonce: info.fetch(:nonce),
-          epoch: info.fetch(:epoch),
-          poison_pill_timeout: info.fetch(:poison_pill_timeout),
-          strict: info.fetch(:strict)
+      # THE strict-ordering gate classifier, shared by the reactor level
+      # (`enter_ordered_lock_scope`) and the step level
+      # (`StepCoordination#ordered_lock_gate`), so a gate state one level
+      # handles cannot fall through to "run" at the other (005 R-06, F7).
+      # Exhaustive: a state nobody mapped raises instead of running.
+      #
+      #   :go         — proceed (includes a poison advance past a dead blocker)
+      #   :skip_chain — strict chain already failed, and this run is `fresh`
+      #                 (an in-flight run that paused completes regardless)
+      #   :stale      — the position belongs to a drained, reused generation
+      #   :drained    — the batch drained and GC'd while this caller slept
+      #
+      # Raises {OrderedLock::WaitError} when it is not this nonce's turn.
+      def self.gate(info, fresh:)
+        state = OrderedLock.new(
+          info.fetch(:key), nonce: info.fetch(:nonce), epoch: info.fetch(:epoch),
+                            poison_pill_timeout: info.fetch(:poison_pill_timeout), strict: info.fetch(:strict)
         ).check!
+
+        case state
+        when :go then :go
+        when :skip_chain_failed then fresh ? :skip_chain : :go
+        when :stale_batch then :stale
+        when :drained_go then :drained
+        else raise ArgumentError, "unhandled ordered-lock gate state #{state.inspect}"
+        end
       end
 
       # Combined gate-check + thread-local push. Call at the top of
       # `execute` / `resume_execution`. Pair with `leave_ordered_lock_scope`
-      # in `ensure`.
+      # in `ensure`. The gate runs BEFORE rate-limit / lock / semaphore so a
+      # waiting nonce never holds any other primitive; a `WaitError` goes to
+      # the worker, which snoozes.
       #
       # The strict-mode chain-skip only fires on a *fresh* start (no step
       # has run yet on this context). This lets an in-flight run that paused
@@ -74,20 +86,19 @@ module RubyReactor
       # strict to a fresh Sidekiq job (which enters via `resume_execution`
       # but has no prior step state).
       def enter_ordered_lock_scope
-        gate = check_ordered_lock_gate
+        info = ordered_lock_info
+        outcome = info ? OrderedLockSupport.gate(info, fresh: fresh_ordered_lock_start?) : :go
         # A stale-batch run never participates regardless of fresh/resume state —
-        # its numbering belongs to a drained generation. Chain-skip stays gated
-        # on a fresh start so an in-flight paused run still completes on resume.
-        @ordered_lock_stale_batch = gate == :stale_batch
-        @ordered_lock_chain_skip = fresh_ordered_lock_start? && gate == :skip_chain_failed
+        # its numbering belongs to a drained generation.
+        @ordered_lock_stale_batch = outcome == :stale
+        @ordered_lock_chain_skip = outcome == :skip_chain
 
         # Drained-batch gate: the batch GC'd while this caller slept. A genuine
         # late straggler runs (poison semantics); a Sidekiq redelivery of an
         # ALREADY-terminal context must not re-execute its steps. Only the
         # latter — confirmed by a terminal stored status — is short-circuited.
-        @ordered_lock_drained_replay = gate == :drained_go && stored_status_terminal?
+        @ordered_lock_drained_replay = outcome == :drained && stored_status_terminal?
 
-        info = ordered_lock_info
         return unless info
 
         OrderedLockSupport.active_keys << info[:key]
@@ -101,8 +112,9 @@ module RubyReactor
         start_ordered_lock_heartbeat(info)
       end
 
+      # Same predicate as `Executor#first_execution?` (005 R-03).
       def fresh_ordered_lock_start?
-        @context.intermediate_results.empty? && @context.current_step.nil?
+        first_execution?
       end
 
       def ordered_lock_chain_skip?
@@ -215,17 +227,49 @@ module RubyReactor
       def start_ordered_lock_heartbeat(info)
         return if @ordered_lock_heartbeat_running
 
+        @ordered_lock_heartbeat_running = true
+        @ordered_lock_heartbeat = OrderedLockSupport.start_heartbeat(info)
+      end
+
+      def stop_ordered_lock_heartbeat
+        return unless @ordered_lock_heartbeat_running
+
+        @ordered_lock_heartbeat_running = false
+        @ordered_lock_heartbeat&.stop
+        @ordered_lock_heartbeat = nil
+      end
+
+      # Value object so a caller can `heartbeat.stop` without reaching into
+      # the thread/flag it wraps. Returned by `.start_heartbeat`.
+      class Heartbeat
+        def initialize(thread, running)
+          @thread = thread
+          @running = running
+        end
+
+        def stop
+          @running[0] = false
+          @thread.wakeup if @thread.alive?
+          @thread.join(0.1)
+        rescue StandardError
+          # Best-effort shutdown; never let heartbeat teardown break the caller's ensure chain.
+        end
+      end
+
+      # Shared by the reactor-level heartbeat above (`#start_ordered_lock_heartbeat`)
+      # and the step-level gate (`StepCoordination#ordered_lock_gate`, T061) — the
+      # restamp logic and interval math are identical, only the caller's own
+      # on/off bookkeeping differs.
+      def self.start_heartbeat(info)
         pp = info[:poison_pill_timeout].to_f
         interval = [pp / 3.0, HEARTBEAT_MIN_INTERVAL].max
-        @ordered_lock_heartbeat_running = true
-        lock = OrderedLock.new(
-          info.fetch(:key), nonce: info.fetch(:nonce), epoch: info.fetch(:epoch)
-        )
+        running = [true]
+        lock = OrderedLock.new(info.fetch(:key), nonce: info.fetch(:nonce), epoch: info.fetch(:epoch))
 
-        @ordered_lock_heartbeat = Thread.new do
-          while @ordered_lock_heartbeat_running
+        thread = Thread.new do
+          while running[0]
             sleep interval
-            break unless @ordered_lock_heartbeat_running
+            break unless running[0]
 
             begin
               lock.heartbeat!
@@ -238,20 +282,8 @@ module RubyReactor
             end
           end
         end
-      end
 
-      def stop_ordered_lock_heartbeat
-        return unless @ordered_lock_heartbeat_running
-
-        @ordered_lock_heartbeat_running = false
-        thread = @ordered_lock_heartbeat
-        @ordered_lock_heartbeat = nil
-        return unless thread
-
-        thread.wakeup if thread.alive?
-        thread.join(0.1)
-      rescue StandardError
-        # Best-effort shutdown; never let heartbeat teardown break the ensure chain.
+        Heartbeat.new(thread, running)
       end
 
       # Advance the cursor when this run reached a *terminal* status.

data/lib/ruby_reactor/executor/result_handler.rb

@@ -32,7 +32,26 @@ module RubyReactor
         end
       end
 
+      # Every reactor-level Failure that follows a rollback is built here, so
+      # this is the one place `rollback_failures` is attached (005 R-08).
       def handle_execution_error(error)
+        failure = build_execution_failure(error)
+        failure.rollback_failures.concat(@compensation_manager.rollback_failures) if failure.is_a?(RubyReactor::Failure)
+        failure
+      end
+
+      def final_result(reactor_class)
+        if reactor_class.return_step
+          result_value = @context.get_result(reactor_class.return_step)
+          RubyReactor.Success(result_value)
+        else
+          RubyReactor.Success(@context.intermediate_results)
+        end
+      end
+
+      private
+
+      def build_execution_failure(error)
         case error
         when Error::StepFailureError
           handle_step_failure_error(error)
@@ -52,17 +71,6 @@ module RubyReactor
         end
       end
 
-      def final_result(reactor_class)
-        if reactor_class.return_step
-          result_value = @context.get_result(reactor_class.return_step)
-          RubyReactor.Success(result_value)
-        else
-          RubyReactor.Success(@context.intermediate_results)
-        end
-      end
-
-      private
-
       # Failure for a validation error (reactor inputs, step arguments, or
       # step output), carrying both the structured field errors and the step/
       # reactor attribution stamped at the raise site (nil step_name for
@@ -138,7 +146,16 @@ module RubyReactor
         step_config.respond_to?(:async_dispatch?) && step_config.async_dispatch?
       end
 
+      # A composed child's Failure carries the child's own rollback failures;
+      # fold them in BEFORE this level rolls back, so they come first.
+      def adopt_rollback_failures(result)
+        return unless result.respond_to?(:rollback_failures)
+
+        @compensation_manager.rollback_failures.concat(result.rollback_failures)
+      end
+
       def handle_retries_exhausted(step_config, result, resolved_arguments)
+        adopt_rollback_failures(result)
         @compensation_manager.handle_step_failure(step_config, result.original_error, resolved_arguments)
         orig_err = result.original_error.is_a?(Exception) ? result.original_error : nil
         error = Error::StepFailureError.new(result.error, step: step_config.name, context: @context,
@@ -154,6 +171,7 @@ module RubyReactor
       end
 
       def handle_failure(step_config, result, resolved_arguments)
+        adopt_rollback_failures(result)
         failure_result = @compensation_manager.handle_step_failure(step_config, result.error, resolved_arguments)
         orig_err = result.error.is_a?(Exception) ? result.error : nil
         # A step that propagates another unit's validation failure (an
@@ -233,6 +251,8 @@ module RubyReactor
       end
 
       def resolve_exception_class(original_error, error)
+        # A step's own contention is reported by its cause (Lock::AcquisitionError, ...).
+        original_error = original_error.original if original_error.is_a?(StepCoordination::Contended)
         return original_error.class.name if original_error
 
         error.respond_to?(:exception_class) ? error.exception_class : nil

data/lib/ruby_reactor/executor/retry_manager.rb

@@ -38,6 +38,13 @@ module RubyReactor
         @context.current_step = step_config.name
         delay = calculate_backoff_delay(step_config, error, reactor_class)
 
+        requeue_job(step_config, delay)
+      end
+
+      # The requeue for a failure retry, given an already-decided backoff
+      # delay. (A contention park never comes through here: it raises
+      # `Error::StepContentionPark` and is requeued by the worker, 005 R-01.)
+      def requeue_job(_step_config, delay)
         # Serialize context and requeue the job
         # Use root context if available to ensure we serialize the full tree
         # BUT for map elements (which have map_metadata), we must serialize the element context itself
@@ -186,7 +193,8 @@ module RubyReactor
                          end,
           reactor_name: reactor_class.name,
           step_arguments: result.respond_to?(:step_arguments) ? result.step_arguments : {},
-          validation_errors: result.validation_errors
+          validation_errors: result.validation_errors,
+          rollback_failures: result.rollback_failures
         )
       end