ruby_reactor 0.8.2 → 0.8.3

data/lib/ruby_reactor/retry_context.rb

@@ -3,13 +3,14 @@
 module RubyReactor
   # Tracks retry attempts and state for steps in async execution
   class RetryContext
-    attr_accessor :step_attempts, :current_step, :failure_reason, :next_retry_at
+    attr_accessor :step_attempts, :current_step, :failure_reason, :next_retry_at, :contention_attempts
 
     def initialize
       @step_attempts = {}
       @current_step = nil
       @failure_reason = nil
       @next_retry_at = nil
+      @contention_attempts = {}
     end
 
     def increment_attempt_for_step(step_name)
@@ -18,6 +19,16 @@ module RubyReactor
       @step_attempts[step_name] += 1
     end
 
+    # Undoes one `increment_attempt_for_step` — called at the start of a
+    # contention park (Finding 2) so a busy key never eats the retry budget
+    # meant for genuine failures. Floored at 0.
+    def decrement_attempt_for_step(step_name)
+      step_name = step_name.to_s
+      return unless @step_attempts[step_name]
+
+      @step_attempts[step_name] = [@step_attempts[step_name] - 1, 0].max
+    end
+
     def attempts_for_step(step_name)
       step_name = step_name.to_s
       @step_attempts[step_name] || 0
@@ -28,11 +39,27 @@ module RubyReactor
       attempts < max_attempts
     end
 
+    def increment_contention_for_step(step_name)
+      step_name = step_name.to_s
+      @contention_attempts[step_name] ||= 0
+      @contention_attempts[step_name] += 1
+    end
+
+    def contention_attempts_for_step(step_name)
+      step_name = step_name.to_s
+      @contention_attempts[step_name] || 0
+    end
+
+    def clear_contention_for_step(step_name)
+      @contention_attempts.delete(step_name.to_s)
+    end
+
     def reset
       @step_attempts = {}
       @current_step = nil
       @failure_reason = nil
       @next_retry_at = nil
+      @contention_attempts = {}
     end
 
     def serialize_for_retry
@@ -40,7 +67,8 @@ module RubyReactor
         step_attempts: @step_attempts,
         current_step: @current_step,
         failure_reason: serialize_error(@failure_reason),
-        next_retry_at: @next_retry_at&.iso8601
+        next_retry_at: @next_retry_at&.iso8601,
+        contention_attempts: @contention_attempts
       }
     end
 
@@ -50,6 +78,7 @@ module RubyReactor
       context.current_step = data["current_step"]
       context.failure_reason = deserialize_error(data["failure_reason"])
       context.next_retry_at = data["next_retry_at"] ? Time.iso8601(data["next_retry_at"]) : nil
+      context.contention_attempts = data["contention_attempts"] || {}
       context
     end
 

data/lib/ruby_reactor/rspec/helpers.rb

@@ -19,6 +19,21 @@ module RubyReactor
           process_jobs: process_jobs
         )
       end
+
+      # Holds `key` as an EXTERNAL owner for the duration of the block, so a
+      # spec can assert against genuine lock contention (a step or reactor
+      # failing/parking because something else already holds its key)
+      # without reaching into `RubyReactor::Lock` or raw Redis calls itself.
+      # Released even if the block raises.
+      #
+      #   hold_lock("acct:1") { expect(ChargeStep.run({ account_id: 1 }, nil)).to be_a(RubyReactor::Failure) }
+      def hold_lock(key, owner: "spec", ttl: 30)
+        lock = RubyReactor::Lock.new(key, owner: owner, ttl: ttl, wait: 0, auto_extend: true)
+        lock.acquire
+        yield
+      ensure
+        lock&.release
+      end
     end
   end
 end

data/lib/ruby_reactor/rspec/matchers.rb

@@ -528,6 +528,98 @@ module RubyReactor
         end
       end
 
+      # Asserts a `:contention_park` trace entry exists for a step (US7-4,
+      # T054). Subject can be a `test_reactor` wrapper, a dispatch/find result
+      # (`.context`), or a raw `Context` (`.execution_trace`) — matching
+      # `have_run_step`'s subject flexibility. Use `.on(key)` to pin the key.
+      #
+      #   expect(result).to have_contended_at(:charge)
+      #   expect(result).to have_contended_at(:charge).on("acct:1")
+      ::RSpec::Matchers.define :have_contended_at do |step_name|
+        match do |subject|
+          trace = execution_trace_for(subject)
+          @entry = trace.find do |t|
+            t[:type].to_s == "contention_park" && t[:step].to_s == step_name.to_s &&
+              (@expected_key.nil? || t[:key].to_s == @expected_key.to_s)
+          end
+          !@entry.nil?
+        end
+
+        chain :on do |key|
+          @expected_key = key
+        end
+
+        def execution_trace_for(subject)
+          subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
+          if subject.respond_to?(:reactor_instance)
+            subject.reactor_instance.context.execution_trace
+          elsif subject.respond_to?(:context)
+            subject.context.execution_trace
+          else
+            subject.execution_trace
+          end
+        end
+
+        failure_message do |_subject|
+          msg = "expected execution trace to contain a contention_park entry for step :#{step_name}"
+          msg += " on key #{@expected_key.inspect}" if @expected_key
+          msg
+        end
+
+        failure_message_when_negated do |_subject|
+          msg = "expected execution trace not to contain a contention_park entry for step :#{step_name}"
+          msg += " on key #{@expected_key.inspect}" if @expected_key
+          msg
+        end
+      end
+
+      # Asserts that a failed run reports a rollback (undo or compensation)
+      # of `step_name` that did not complete, on `Failure#rollback_failures`.
+      # Subject is a `Failure` or a `test_reactor` wrapper (its `result`).
+      #
+      #   expect(result).to have_rollback_failure(:charge)
+      #   expect(subject).to have_rollback_failure(:charge).for_key("acct:1").because(:coordination_unavailable)
+      #   expect(subject).not_to have_rollback_failure(:charge)
+      ::RSpec::Matchers.define :have_rollback_failure do |step_name|
+        match do |subject|
+          @entries = rollback_failures_for(subject)
+          @entries.any? do |e|
+            e[:step].to_s == step_name.to_s &&
+              (@expected_key.nil? || e[:key].to_s == @expected_key.to_s) &&
+              (@expected_reason.nil? || e[:reason].to_s == @expected_reason.to_s)
+          end
+        end
+
+        chain :for_key do |key|
+          @expected_key = key
+        end
+
+        chain :because do |reason|
+          @expected_reason = reason
+        end
+
+        def rollback_failures_for(subject)
+          subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
+          actual = subject.respond_to?(:result) ? subject.result : subject
+          actual.respond_to?(:rollback_failures) ? actual.rollback_failures : []
+        end
+
+        def expectation_text(step_name)
+          text = "a rollback failure for :#{step_name}"
+          text += " on key #{@expected_key.inspect}" if @expected_key
+          text += " because #{@expected_reason.inspect}" if @expected_reason
+          text
+        end
+
+        failure_message do |_subject|
+          "expected #{expectation_text(step_name)}, got rollback_failures #{@entries.inspect}"
+        end
+
+        failure_message_when_negated do |_subject|
+          "expected no #{expectation_text(step_name)}, got rollback_failures #{@entries.inspect}"
+        end
+      end
+
       # Add more matchers as per plan
       # rubocop:enable Metrics/BlockLength
     end

data/lib/ruby_reactor/rspec/test_subject.rb

@@ -767,7 +767,13 @@ module RubyReactor
         if step_config_orig.has_run_block?
           step_config_orig.run_block
         elsif step_config_orig.has_impl?
-          ->(args, ctx) { step_config_orig.impl.run(args, ctx) }
+          impl = step_config_orig.impl
+
+          if impl.respond_to?(:run_without_coordination)
+            ->(args, ctx) { impl.run_without_coordination(args, ctx) }
+          else
+            ->(args, ctx) { impl.run(args, ctx) }
+          end
         else
           ->(_, _) { raise "No implementation found for #{target_step}" }
         end

data/lib/ruby_reactor/step/async_reactor_step.rb

@@ -15,6 +15,44 @@ module RubyReactor
     # ordered-lock nonce at ENQUEUE time (so ordering matches caller order), and
     # persist before enqueueing (F2).
     class AsyncReactorStep < RubyReactor::Step
+      class << self
+        # Exclusive keys this EXECUTION currently holds, read from the root
+        # context's registry. Shared by the `async_reactor` dispatch check
+        # below (via `detect_lock_deadlock`) and the `async_step` dispatch-time
+        # guard (`Executor::AsyncStepDispatch`, T034) — both hand off work to
+        # a process that will never share this execution's owner, so a key
+        # the execution currently holds would deadlock the dispatched unit.
+        def held_lock_keys(context)
+          root = context.root_context || context
+          Array(root.private_data[:held_lock_keys] || root.private_data["held_lock_keys"])
+        end
+
+        # `kind:` names what is being dispatched ("async_reactor" / "async_step")
+        # so the two guards share one message shape instead of drifting apart.
+        def deadlock_message(key, dispatched_name, context, kind: "async_reactor")
+          parent = context.reactor_class&.name || "the dispatching reactor"
+          <<~MSG.strip
+            #{kind} dispatch of #{dispatched_name || "<anonymous>"} would deadlock: it declares the lock key \
+            '#{key}', which #{parent} currently holds and will not release until it finishes.
+            The child would snooze forever, and if #{parent} later reads this child's result it would wait \
+            for work that can never start. Lock ownership is never shared across the async boundary — the \
+            two run concurrently, so sharing it would break mutual exclusion outright.
+            Fix, in order of preference:
+              1. Use `compose` instead of `async_reactor` if the child belongs inside #{parent}'s critical \
+            section and its result is needed — waiting for it means the work is sequential anyway.
+              2. Narrow the lock keys, if parent and child actually protect different resources.
+              3. Restructure so the locked reactor never reads the child's result — fire-and-forget, and \
+            verify in the child itself or in a successor reactor outside the lock window.#{async_step_remedy(kind)}
+          MSG
+        end
+
+        def async_step_remedy(kind)
+          return "" unless kind == "async_step"
+
+          "\n  4. Run the step inline (drop `async_step`) if it belongs inside the critical section."
+        end
+      end
+
       def run
         child_class = inputs[:async_reactor_class]
         child_inputs = build_child_inputs(inputs[:argument_mappings] || {})
@@ -60,13 +98,13 @@ module RubyReactor
       # loudly, at dispatch. Ordinary cross-execution contention on the same
       # key is unaffected and still snoozes normally.
       def detect_lock_deadlock(child_class, child_inputs)
-        held = held_lock_keys
+        held = self.class.held_lock_keys(context)
         return nil if held.empty?
 
         collision = child_lock_keys(child_class, child_inputs).find { |key| held.include?(key) }
         return nil unless collision
 
-        RubyReactor.Failure(deadlock_message(collision, child_class))
+        RubyReactor.Failure(self.class.deadlock_message(collision, child_class.name, context))
       end
 
       def child_lock_keys(child_class, child_inputs)
@@ -83,28 +121,6 @@ module RubyReactor
         keys.compact
       end
 
-      def held_lock_keys
-        root = context.root_context || context
-        Array(root.private_data[:held_lock_keys] || root.private_data["held_lock_keys"])
-      end
-
-      def deadlock_message(key, child_class)
-        parent = context.reactor_class&.name || "the dispatching reactor"
-        <<~MSG.strip
-          async_reactor dispatch of #{child_class.name || "<anonymous>"} would deadlock: it declares the \
-          lock key '#{key}', which #{parent} currently holds and will not release until it finishes.
-          The child would snooze forever, and if #{parent} later reads this child's result it would wait \
-          for work that can never start. Lock ownership is never shared across the async boundary — the \
-          two run concurrently, so sharing it would break mutual exclusion outright.
-          Fix, in order of preference:
-            1. Use `compose` instead of `async_reactor` if the child belongs inside #{parent}'s critical \
-          section and its result is needed — waiting for it means the work is sequential anyway.
-            2. Narrow the lock keys, if parent and child actually protect different resources.
-            3. Restructure so the locked reactor never reads the child's result — fire-and-forget, and \
-          verify in the child itself or in a successor reactor outside the lock window.
-        MSG
-      end
-
       # `RSpec::TestSubject`'s `async: false` clears the dispatch marker to run
       # the whole reactor in one process.
       def run_inline?

data/lib/ruby_reactor/step/compose_step.rb

@@ -21,7 +21,9 @@ module RubyReactor
       end
 
       # Compensating a failed compose and undoing a completed one are the same
-      # work: roll back whatever the child reactor completed.
+      # work: roll back whatever the child reactor completed. Any child undo
+      # that did not complete comes back on the Failure, which the parent's
+      # `CompensationManager` flattens into its own `rollback_failures`.
       def compensate
         step_name = context.current_step
         composed_data = context.composed_contexts[step_name]
@@ -32,7 +34,10 @@ module RubyReactor
         executor.undo_all
         executor.save_context
 
-        RubyReactor.Success()
+        failures = executor.compensation_manager.rollback_failures
+        return RubyReactor.Success() if failures.empty?
+
+        RubyReactor.Failure("composed :#{step_name} rollback incomplete", rollback_failures: failures)
       end
 
       alias undo compensate
@@ -79,7 +84,13 @@ module RubyReactor
       def execute_child_reactor(composed_reactor, child_context, composed_data)
         executor = RubyReactor::Executor.new(composed_reactor, {}, child_context)
 
-        if composed_data && child_context.current_step
+        # Resume once the child has been admitted, not by its `current_step`:
+        # a park two levels down unwinds the child's `with_step` and clears
+        # it, and `execute` would then re-charge the child's rate limit and
+        # fresh-acquire its lock instead of re-adopting the parked hold
+        # (005 R-02). `current_step` stays as the fallback for a child saved
+        # before `admitted` existed.
+        if composed_data && (child_context.admitted? || child_context.current_step)
           executor.resume_execution
         else
           executor.execute

data/lib/ruby_reactor/step.rb

@@ -13,9 +13,15 @@ module RubyReactor
   # 1. Resolve `inputs`: the given arguments with the contract's defaults
   #    applied. `run`, `undo`, and `compensate` all see the same values.
   # 2. `.run` ONLY: enforce the declared input contract, raising
-  #    `Error::InputValidationError` before any instance exists. `.undo` and
-  #    `.compensate` NEVER enforce it: rollback must not fail on the very
-  #    inputs that may have caused the failure.
+  #    `Error::InputValidationError` before any instance exists.
+  # 2b. `.run` ONLY: step-scoped coordination (`with_lock` etc, if declared)
+  #    is acquired around the rest of `.run`, keyed on the just-validated
+  #    inputs — after contract enforcement, so a step that will fail
+  #    validation never takes a hold (research Finding 7 / Finding 8). A
+  #    reactor never goes through here: it calls `.run_without_coordination`
+  #    and coordinates the step itself (research D2).
+  #    `.undo` and `.compensate` NEVER enforce the input contract: rollback
+  #    must not fail on the very inputs that may have caused the failure.
   # 3. Build a FRESH instance, never reused across actions. An ivar set in
   #    `run` is gone by the time `undo` runs on its own instance, so async
   #    execution running `run` and `undo` in different processes behaves
@@ -23,10 +29,13 @@ module RubyReactor
   # 4. Invoke the matching instance method, translating any `StepSignals`
   #    throw (`success!`/`skip!`/`fail!`/`halt!`) into its result wrapper.
   #
-  # No `prepend`/`extend`/`define_method`/`method_missing` — every step in the
-  # class reads top to bottom as ordinary method calls.
+  # No `prepend`/`define_method`/`method_missing` — every step in the class
+  # reads top to bottom as ordinary method calls. The one `extend` is
+  # `Dsl::Lockable::ClassMethods` (the five coordination macros), a
+  # self-contained module with no hooks of its own (Finding 9).
   class Step
     include RubyReactor::StepSignals
+    extend RubyReactor::Dsl::Lockable::ClassMethods
 
     attr_reader :inputs, :context, :result, :reason
 
@@ -68,12 +77,28 @@ module RubyReactor
     # rubocop:enable Naming/MethodName
 
     class << self
-      def run(arguments, context)
+      # A DIRECT invocation — application code, or another step's body. It is
+      # its own unit of work: it takes this class's coordination itself and,
+      # having no queue to park into, waits then fails (FR-016/FR-023).
+      # `context` is optional: a stand-alone `ChargeStep.run(args)` is its own
+      # execution, with no reactor context to inherit ownership from.
+      def run(arguments, context = nil)
         validated = enforce_contract!(arguments)
-        catch(StepSignals::TAG) { new(validated, context).run }
+        coordinate(validated, context) { run_without_coordination(validated, context) }
       end
       alias call run
 
+      # The reactor's entry (`StepConfig#call_body`): the executor or
+      # `StepWorker` has already taken this step's EFFECTIVE coordination —
+      # the class's declarations and any inline ones, in one fixed order — so
+      # taking the class's here again would split acquisition across two
+      # layers. Still enforces the contract (idempotent: it only applies
+      # defaults to already-valid arguments).
+      def run_without_coordination(arguments, context)
+        validated = enforce_contract!(arguments)
+        catch(StepSignals::TAG) { new(validated, context).run }
+      end
+
       # Same `inputs` as `.run` (defaults applied), but NEVER enforces the contract.
       def undo(result, arguments, context)
         catch(StepSignals::TAG) { new(with_defaults(arguments), context, result: result).undo }
@@ -117,6 +142,23 @@ module RubyReactor
 
       private
 
+      # Step-scoped coordination for a DIRECT call (`direct: true`): never
+      # parks, keeps no state on `context`, and so can never be mistaken for
+      # the coordination of the reactor step whose body made the call.
+      # After `enforce_contract!`, so a step that will fail validation never
+      # takes a hold (Finding 8). A bare step pays one check. `context` may be
+      # nil (a stand-alone `MyStep.run(args)`) — `StepCoordination#owner`
+      # handles that case.
+      def coordinate(validated, context, &block)
+        return block.call if Executor::StepCoordination.none?(self)
+
+        ctx = context if context.is_a?(RubyReactor::Context)
+        Executor::StepCoordination.new(
+          step_config: self, arguments: validated, context: ctx, reactor_class: ctx&.reactor_class,
+          middlewares: ctx&.middlewares || Executor.middlewares_for(ctx&.reactor_class), direct: true
+        ).around_run(&block)
+      end
+
       def own_input_contract
         @own_input_contract ||= Step::InputContract.new(owner: self)
       end

data/lib/ruby_reactor/step_sweeper.rb

@@ -42,9 +42,10 @@ module RubyReactor
 
         arguments = dispatch_arguments(record)
         next unless arguments
+        next if parked?(record)
         next if live?(arguments)
 
-        @async_router.perform_step_async(**arguments)
+        redispatch(record, arguments)
         redispatched += 1
       rescue StandardError => e
         # One bad record must not abort the whole sweep.
@@ -56,6 +57,17 @@ module RubyReactor
 
     private
 
+    # A swept park must not restart its contention counter: rebuilt from the
+    # record alone, a zeroed payload would let `lock_snooze_max_attempts`
+    # never bite. `perform_step_in(0, ...)` is the only re-dispatch that
+    # carries the count.
+    def redispatch(record, arguments)
+      attempts = record["contention_attempts"].to_i
+      return @async_router.perform_step_async(**arguments) unless attempts.positive?
+
+      @async_router.perform_step_in(0, **arguments, contention_attempts: attempts)
+    end
+
     def dispatch_arguments(record)
       values = record.values_at(*DISPATCH_KEYS)
       return nil if values.any?(&:nil?)
@@ -63,6 +75,22 @@ module RubyReactor
       DISPATCH_KEYS.map(&:to_sym).zip(values).to_h
     end
 
+    # A unit parked on step-level contention (StepWorker#handle_contention)
+    # released its liveness lock on purpose and has a redelivery already
+    # scheduled. It looks exactly like a lost unit, so without this the sweep
+    # would dispatch a duplicate that runs the body a second time once the
+    # redelivery fires. Past the stamped window it is fair game again — a
+    # redelivery that eventually lands on a finished unit is dropped by
+    # `StepWorker#already_completed?`.
+    def parked?(record)
+      parked_until = record["parked_until"]
+      return false unless parked_until
+
+      Time.iso8601(parked_until) > Time.now
+    rescue ArgumentError, TypeError
+      false
+    end
+
     def live?(arguments)
       @storage.lock_held?(
         RubyReactor.async_step_lock_key(arguments[:step_context_id], arguments[:step_name])