ruby_reactor 0.5.1 → 0.5.3

data/lib/ruby_reactor/executor.rb

@@ -7,10 +7,13 @@ require_relative "executor/retry_manager"
 require_relative "executor/compensation_manager"
 require_relative "executor/result_handler"
 require_relative "executor/step_executor"
+require_relative "executor/ordered_lock_support"
 
 module RubyReactor
   # rubocop:disable Metrics/ClassLength
   class Executor
+    include OrderedLockSupport
+
     attr_reader :reactor_class, :context, :dependency_graph, :compensation_manager, :retry_manager, :result_handler,
                 :step_executor, :result, :middlewares
 
@@ -35,12 +38,24 @@ module RubyReactor
           retry_manager: @retry_manager,
           result_handler: @result_handler,
           compensation_manager: @compensation_manager,
-          middlewares: @middlewares
+          middlewares: @middlewares,
+          # Save-per-step durable checkpoint. checkpoint! resolves the ROOT
+          # context, so this same callback — wired into every executor including
+          # the nested ones ComposeStep builds — always advances the root blob
+          # (F8): a mid-child crash re-runs one sub-step, not the whole child.
+          # `throttle: true` lets checkpoint_min_interval coalesce these mid-run
+          # writes (default 0 = write every step); the terminal save still runs.
+          on_step_complete: -> { checkpoint!(throttle: true) }
         }
       )
       @result = nil
       @acquired_lock = nil
       @acquired_semaphore = nil
+      @acquired_context_lock = nil
+      @context_lock_owner = nil
+      @contention_snooze = false
+      @skip_context_persist = false
+      @last_checkpoint_at = nil
     end
 
     def self.resolve_middlewares(reactor_class)
@@ -71,9 +86,13 @@ module RubyReactor
       middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
       completed = false
 
-      if (skipped = check_period_gate)
+      enter_ordered_lock_scope
+      # short_circuit_result covers both the strict ordered-lock chain skip
+      # and the already-marked period bucket.
+      short = short_circuit_result
+      if short
         completed = true
-        return finalize_skipped(skipped)
+        return short_circuit!(short)
       end
 
       # Validate inputs BEFORE consuming a rate-limit slot or grabbing a
@@ -109,7 +128,9 @@ module RubyReactor
     rescue RubyReactor::Lock::AcquisitionError,
            RubyReactor::Semaphore::AcquisitionError,
            RubyReactor::RateLimit::ExceededError,
-           RubyReactor::RateLimitRegistry::UnknownLimitError => e
+           RubyReactor::RateLimitRegistry::UnknownLimitError,
+           RubyReactor::OrderedLock::WaitError => e
+      @contention_snooze = true
       raise e
     rescue StandardError => e
       @result = @result_handler.handle_execution_error(e)
@@ -118,77 +139,107 @@ module RubyReactor
       @result
     ensure
       release_locks
-      save_context if persist_context?
+      leave_ordered_lock_scope
+      save_context if persist_context? && !skip_context_persist?
 
+      emit_lifecycle_completion(completed)
+    end
+
+    # Contention errors (lock/semaphore/rate-limit/ordered-lock wait) are
+    # expected "try again later" signals, not failures — the worker snoozes
+    # and re-runs. Emitting `failed_reactor` for them floods dashboards with
+    # phantom failures (one per snooze round), so route them to a distinct
+    # `snooze_reactor` event instead.
+    def emit_lifecycle_completion(completed)
       if completed
         middlewares.on(:complete_reactor, reactor_class.name, @result, @context)
+      elsif @contention_snooze
+        middlewares.on(:snooze_reactor, reactor_class.name, $ERROR_INFO, @context)
       else
         middlewares.on(:failed_reactor, reactor_class.name, $ERROR_INFO, @context)
       end
     end
 
-    def resume_execution # rubocop:disable Metrics/MethodLength
+    def resume_execution # rubocop:disable Metrics/MethodLength,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity
       middlewares.on(:start_reactor, reactor_class.name, context.inputs, @context)
       completed = false
+
       # A fresh async reactor run reaches the worker through resume_execution
       # (it never calls execute), so the period and rate-limit gates that live
       # in execute must be applied here too. Genuine resumes (a step already ran
       # or we paused mid-flight, so current_step is set) must NOT re-gate: a
       # paused reactor must not throttle or skip itself on the way back in.
       first_run = first_execution?
-      begin
-        @context.status = :running
 
-        if first_run && (skipped = check_period_gate)
-          completed = true
-          return finalize_skipped(skipped)
-        end
-        check_rate_limit if first_run
+      enter_ordered_lock_scope
+      # ordered-lock skip applies on any run; the period gate only on a fresh
+      # first run (a genuine resume must not skip itself when its own marker
+      # eventually lands).
+      short = ordered_lock_short_circuit
+      short ||= check_period_gate if first_run
+      if short
+        completed = true
+        return short_circuit!(short)
+      end
 
-        acquire_concurrency_primitives
+      @context.status = :running
+      check_rate_limit if first_run
+
+      # Per-context liveness lock: serializes duplicate deliveries of the same
+      # root context (e.g. a sweeper re-enqueue racing a still-live worker) and
+      # doubles as the sweeper's "worker alive" signal. Only the ROOT executor
+      # holds it — composed/nested children resume inline under the root worker
+      # and must not contend on the root's own key.
+      acquire_context_lock
+
+      # Resumes intentionally skip check_rate_limit (a paused run must not
+      # block itself on resume), so acquire lock/semaphore directly rather
+      # than via acquire_locks.
+      acquire_exclusive_lock if @reactor_class.respond_to?(:lock_config) && @reactor_class.lock_config
+      acquire_semaphore if @reactor_class.respond_to?(:semaphore_config) && @reactor_class.semaphore_config
 
-        # Post-lock re-check (see execute) — closes the period race for the
-        # first run of a locked async reactor.
-        if first_run && (skipped = check_period_gate)
-          completed = true
-          return finalize_skipped(skipped)
-        end
+      # Post-lock re-check (see execute) — closes the period race for the
+      # first run of a locked async reactor.
+      if first_run && (skipped = check_period_gate)
+        completed = true
+        return finalize_skipped(skipped)
+      end
 
-        prepare_for_resume
-        save_context
+      prepare_for_resume
+      save_context
 
-        @result = if @context.current_step
-                    execute_current_step_and_continue
-                  else
-                    execute_remaining_steps
-                  end
+      @result = if @context.current_step
+                  execute_current_step_and_continue
+                else
+                  execute_remaining_steps
+                end
 
-        update_context_status(@result)
-        mark_period_on_success(@result)
+      update_context_status(@result)
+      mark_period_on_success(@result)
 
-        handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
-        completed = true
-        @result
-      rescue RubyReactor::Lock::AcquisitionError,
-             RubyReactor::Semaphore::AcquisitionError,
-             RubyReactor::RateLimit::ExceededError,
-             RubyReactor::RateLimitRegistry::UnknownLimitError
-        raise
-      rescue StandardError => e
-        handle_resume_error(e)
-        update_context_status(@result)
-        completed = true
-        @result
-      ensure
-        release_locks
-        save_context
+      handle_interrupt(@result) if @result.is_a?(RubyReactor::InterruptResult)
+      completed = true
+      @result
+    rescue RubyReactor::Lock::AcquisitionError,
+           RubyReactor::Semaphore::AcquisitionError,
+           RubyReactor::RateLimit::ExceededError,
+           RubyReactor::RateLimitRegistry::UnknownLimitError,
+           RubyReactor::OrderedLock::WaitError => e
+      @contention_snooze = true
+      raise e
+    rescue StandardError => e
+      handle_resume_error(e)
+      update_context_status(@result)
+      completed = true
+      @result
+    ensure
+      release_locks
+      @acquired_context_lock&.release
+      @acquired_context_lock = nil
+      leave_ordered_lock_scope
+      save_context unless skip_context_persist?
 
-        if completed
-          middlewares.on(:complete_reactor, reactor_class.name, @result, @context)
-        else
-          middlewares.on(:failed_reactor, reactor_class.name, $ERROR_INFO, @context)
-        end
-      end
+      emit_lifecycle_completion(completed)
     end
 
     def undo_all
@@ -209,13 +260,40 @@ module RubyReactor
 
     def save_context
       storage = RubyReactor::Configuration.instance.storage_adapter
-      reactor_class_name = @reactor_class.name || "AnonymousReactor-#{@reactor_class.object_id}"
+      reactor_class_name = RubyReactor.reactor_storage_name(@reactor_class)
 
       # Serialize context
       serialized_context = ContextSerializer.serialize(@context)
       storage.store_context(@context.context_id, serialized_context, reactor_class_name)
     end
 
+    # Durable per-step checkpoint. Unlike save_context (which serializes THIS
+    # executor's @context — the observability path, F1), checkpoint! always
+    # serializes and stores the ROOT context under the root's key — the unit the
+    # async worker rehydrates by id. For a top-level reactor root == @context; for
+    # a composed/nested child it stores the root with the child's live state
+    # embedded via composed_contexts. TTL is re-stamped on every write (Phase 4).
+    def checkpoint!(throttle: false)
+      return if throttle && !checkpoint_due?
+
+      root = @context.root_context || @context
+      storage = RubyReactor::Configuration.instance.storage_adapter
+      reactor_class_name = RubyReactor.reactor_storage_name(root.reactor_class)
+      storage.store_context(root.context_id, ContextSerializer.serialize(root), reactor_class_name)
+      @last_checkpoint_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    # Whether a throttled (per-step) checkpoint is due. With checkpoint_min_interval
+    # <= 0 (default) every step checkpoints; otherwise mid-run checkpoints are
+    # coalesced to at most one per interval. The first step of a run always writes
+    # (@last_checkpoint_at is nil), and the run's terminal save is never throttled.
+    def checkpoint_due?
+      interval = RubyReactor.configuration.checkpoint_min_interval.to_f
+      return true if interval <= 0 || @last_checkpoint_at.nil?
+
+      (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @last_checkpoint_at) >= interval
+    end
+
     def persist_context?
       @context.status.to_s != "pending" ||
         @context.execution_trace.any? ||
@@ -308,6 +386,42 @@ module RubyReactor
       RubyReactor::Period.key(base, config[:every])
     end
 
+    # Per-execution liveness lock on the root context id. Owner is a fresh UUID
+    # per execution (NOT the context_id): a duplicate delivery of the *same*
+    # context from a different worker must be blocked, so reentrancy by id would
+    # defeat the guard. Only the root executor acquires — a composed/nested child
+    # resumes inline under the root worker and shares the root's lock, so it must
+    # not try to re-acquire the same key with a different owner (self-deadlock).
+    def acquire_context_lock
+      root = @context.root_context || @context
+      return unless root.equal?(@context) # only the root executor holds it
+      # In Sidekiq::Testing.inline! the retry/snooze `perform_in` re-enters the
+      # worker synchronously, nested inside this still-running frame that holds
+      # the lock — it would self-contend forever. The lock guards concurrent
+      # cross-process delivery, which cannot happen under inline testing, so skip.
+      return if inline_testing_mode?
+
+      lock = RubyReactor::Lock.new(
+        "async:#{root.context_id}",
+        owner: @context_lock_owner ||= SecureRandom.uuid,
+        ttl: RubyReactor.configuration.context_lock_ttl,
+        wait: 0,            # fail fast -> snooze; never block the worker thread
+        auto_extend: true   # keep the liveness signal fresh while we run
+      )
+      lock.acquire
+      @acquired_context_lock = lock
+    rescue RubyReactor::Lock::AcquisitionError => e
+      # We lost the race to a live original holding this context's lock. We did
+      # no work, so we must NOT persist on the way out — saving our (older)
+      # rehydrated snapshot would clobber the original's newer checkpoint.
+      @skip_context_persist = true
+      raise RubyReactor::Lock::ContextLockContention.new(e.message, context_lock_key: "async:#{root.context_id}")
+    end
+
+    def inline_testing_mode?
+      defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
+    end
+
     def acquire_exclusive_lock
       config = @reactor_class.lock_config
       key = config[:key_proc].call(@context.inputs)

data/lib/ruby_reactor/lock.rb

@@ -4,6 +4,19 @@ module RubyReactor
   class Lock
     class AcquisitionError < StandardError; end
 
+    # Raised specifically for the per-context liveness lock (`async:<id>`).
+    # Carries the bare key so the worker can exempt it from the snooze cap:
+    # a duplicate of the *same* execution may legitimately wait arbitrarily
+    # long for the live original to finish.
+    class ContextLockContention < AcquisitionError
+      attr_reader :context_lock_key
+
+      def initialize(message, context_lock_key:)
+        super(message)
+        @context_lock_key = context_lock_key
+      end
+    end
+
     # Minimum interval between auto-extend pings; protects very small TTLs.
     MIN_EXTEND_INTERVAL = 1.0
 

data/lib/ruby_reactor/map/collector.rb

@@ -8,6 +8,42 @@ module RubyReactor
       def self.perform(arguments)
         arguments = arguments.transform_keys(&:to_sym)
         map_id = arguments[:map_id]
+
+        # Serialize concurrent collector deliveries for the SAME map (eager queue +
+        # counter-zero trigger + sweeper re-trigger could otherwise all resume the
+        # parent at once and both write its context). A dedicated map_collect lock
+        # is used rather than the parent's own lock so it never conflicts with the
+        # context lock the parent's resume_execution acquires for itself.
+        lock = acquire_collect_lock(map_id)
+        return if lock == :contended
+
+        begin
+          perform_collection(arguments)
+        ensure
+          lock.release if lock.respond_to?(:release)
+        end
+      end
+
+      def self.acquire_collect_lock(map_id)
+        return :inline if inline_testing_mode?
+
+        lock = RubyReactor::Lock.new(
+          "map_collect:#{map_id}",
+          owner: SecureRandom.uuid, ttl: RubyReactor.configuration.context_lock_ttl,
+          wait: 0, auto_extend: true
+        )
+        lock.acquire
+        lock
+      rescue RubyReactor::Lock::AcquisitionError
+        :contended
+      end
+
+      def self.inline_testing_mode?
+        defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
+      end
+
+      def self.perform_collection(arguments)
+        map_id = arguments[:map_id]
         parent_context_id = arguments[:parent_context_id]
         parent_reactor_class_name = arguments[:parent_reactor_class_name]
         step_name = arguments[:step_name]
@@ -18,6 +54,11 @@ module RubyReactor
         parent_context_data = storage.retrieve_context(parent_context_id, parent_reactor_class_name)
         parent_context = RubyReactor::Context.deserialize_from_retry(parent_context_data)
 
+        # Idempotency: if the parent already recorded this map step's result, a
+        # prior collector already resumed it. Re-resuming would double-execute the
+        # steps after the map. Skip.
+        return if parent_context.intermediate_results.key?(step_name.to_sym)
+
         # Check if all tasks are completed
         metadata = storage.retrieve_map_metadata(map_id, parent_reactor_class_name)
         total_count = metadata ? metadata["count"].to_i : 0

data/lib/ruby_reactor/map/dispatcher.rb

@@ -104,6 +104,48 @@ module RubyReactor
         end
       end
 
+      # Re-dispatch a SPECIFIC index whose result slot is missing (Phase 5c, used
+      # by the map sweeper). Index-driven rather than offset-driven: resolve the
+      # source from the stored parent context and pick source[index]. Idempotent
+      # because store_map_result HSETs by index — a re-run overwrites slot `index`,
+      # never duplicates.
+      def self.requeue_index(map_meta, index)
+        storage = RubyReactor.configuration.storage_adapter
+        parent_class_name = map_meta["parent_reactor_class_name"]
+        parent_context = load_parent_context_from_storage(map_meta["parent_context_id"], parent_class_name, storage)
+
+        arguments = {
+          map_id: map_meta["map_id"],
+          step_name: map_meta["step_name"],
+          strict_ordering: map_meta["strict_ordering"],
+          parent_context_id: map_meta["parent_context_id"],
+          parent_reactor_class_name: parent_class_name,
+          fail_fast: map_meta["fail_fast"],
+          batch_size: map_meta["batch_size"]
+        }
+
+        source = resolve_source(arguments, parent_context)
+        element = element_at(source, index)
+
+        queue_element_job(element, index, {
+                            map_id: map_meta["map_id"],
+                            arguments: arguments,
+                            context: parent_context,
+                            reactor_class_info: map_meta["reactor_class_info"],
+                            step_name: map_meta["step_name"]
+                          })
+      end
+
+      def self.element_at(source, index)
+        if source.is_a?(Array)
+          source[index]
+        elsif source.respond_to?(:offset) && source.respond_to?(:limit)
+          source.offset(index).limit(1).to_a.first
+        else
+          source.drop(index).first
+        end
+      end
+
       def self.queue_element_job(element, index, options)
         arguments = options[:arguments]
         context = options[:context]

data/lib/ruby_reactor/map/element_executor.rb

@@ -8,6 +8,45 @@ module RubyReactor
       def self.perform(arguments)
         arguments = arguments.transform_keys(&:to_sym)
 
+        # Per-element liveness lock (Phase 5b): its presence is the map sweeper's
+        # "element alive" signal, and it serializes duplicate deliveries so a
+        # re-run can't double-decrement the counter (M3). A duplicate of a live
+        # element is dropped — the live original stores the result and finalizes.
+        lock = acquire_element_lock(arguments)
+        return if lock == :contended
+
+        begin
+          perform_element(arguments)
+        ensure
+          lock.release if lock.respond_to?(:release)
+        end
+      end
+
+      def self.acquire_element_lock(arguments)
+        # In Sidekiq::Testing.inline! an element's async-retry perform_map_element_in
+        # re-enters synchronously inside this frame; the lock would self-contend.
+        # It only guards concurrent cross-process delivery, impossible inline.
+        return :inline if inline_testing_mode?
+
+        lock = RubyReactor::Lock.new(
+          "map_element:#{arguments[:map_id]}:#{arguments[:index]}",
+          owner: SecureRandom.uuid, ttl: RubyReactor.configuration.context_lock_ttl,
+          wait: 0, auto_extend: true
+        )
+        lock.acquire
+        lock
+      rescue RubyReactor::Lock::AcquisitionError
+        RubyReactor.configuration.logger.info(
+          "RubyReactor map element #{arguments[:map_id]}:#{arguments[:index]} already in flight; dropping duplicate"
+        )
+        :contended
+      end
+
+      def self.inline_testing_mode?
+        defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
+      end
+
+      def self.perform_element(arguments)
         context = hydrate_or_create_context(arguments)
         # The element already runs inside its own background worker, so any async
         # steps (and async retries) must execute inline here rather than handing

data/lib/ruby_reactor/map/helpers.rb

@@ -108,10 +108,17 @@ module RubyReactor
           executor.resume_execution
         end
 
+        # Checkpoint the ROOT, not the sub (F9/C2). When the map is embedded in a
+        # composed sub-reactor, parent_context is the *sub*; storing only the sub
+        # would leave the root blob stale and a rehydrate-by-root-id resume would
+        # lose the map's completion. Resolve the root (which embeds the sub's
+        # post-map state via composed_contexts) and store that. For a top-level
+        # map parent_context IS the root, so this is unchanged.
+        root = parent_context.root_context || parent_context
         storage.store_context(
-          parent_context.context_id,
-          ContextSerializer.serialize(parent_context),
-          parent_context.reactor_class.name
+          root.context_id,
+          ContextSerializer.serialize(root),
+          RubyReactor.reactor_storage_name(root.reactor_class)
         )
       end
     end

data/lib/ruby_reactor/map/sweeper.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Map
+    # Recovers map fan-out from a hard kill (Phase 5d). Maps are the path most
+    # exposed to a lost job: one missing element result hangs the whole map and
+    # its parent forever. The unifying signal is the results hash — index-keyed
+    # and idempotent (HSET) — so completion is authoritative on `missing`, not on
+    # the fragile counter:
+    #
+    #   missing = (0...count) - HKEYS(results)
+    #
+    # For each active map:
+    #   * missing indices with NO live element lock are re-dispatched (M1/M4/M5).
+    #   * if nothing is missing but the parent never resumed, the collector is
+    #     re-triggered (M2) — gated so it never fires while a collector or the
+    #     parent is alive, or after the parent already collected.
+    #
+    # `run_once` is pure and idempotent; the host wires the cadence (same contract
+    # as RubyReactor::Sweeper).
+    class Sweeper
+      def self.run_once(limit: 1000)
+        new.run_once(limit: limit)
+      end
+
+      def initialize(storage: nil, async_router: nil, logger: nil)
+        @storage = storage || RubyReactor.configuration.storage_adapter
+        @async_router = async_router || RubyReactor.configuration.async_router
+        @logger = logger || RubyReactor.configuration.logger
+      end
+
+      # Returns { redispatched:, recollected: } counts.
+      def run_once(limit: 1000)
+        redispatched = 0
+        recollected = 0
+
+        @storage.scan_maps(count: limit).each do |meta|
+          missing = missing_indices(meta)
+          if missing.any?
+            redispatched += redispatch_missing(meta, missing)
+          elsif recollect?(meta)
+            retrigger_collector(meta)
+            recollected += 1
+          end
+        rescue StandardError => e
+          @logger.warn("RubyReactor::Map::Sweeper failed on map #{meta["map_id"]}: #{e.class}: #{e.message}")
+        end
+
+        { redispatched: redispatched, recollected: recollected }
+      end
+
+      private
+
+      def missing_indices(meta)
+        @storage.missing_map_indices(meta["map_id"], meta["count"].to_i, meta["parent_reactor_class_name"])
+      end
+
+      def redispatch_missing(meta, missing)
+        count = 0
+        missing.each do |index|
+          next if @storage.lock_held?("map_element:#{meta["map_id"]}:#{index}") # element alive
+
+          RubyReactor::Map::Dispatcher.requeue_index(meta, index)
+          count += 1
+        end
+        count
+      end
+
+      # All results are in. Re-trigger the collector only if no collector/parent is
+      # alive and the parent has not already collected this step.
+      def recollect?(meta)
+        return false if @storage.lock_held?("map_collect:#{meta["map_id"]}")  # a collector is running
+        return false if parent_live_lock?(meta)                               # parent execution alive
+        return false if parent_already_collected?(meta)
+
+        true
+      end
+
+      # N1: a nested map's parent is a map element running under a `map_element:`
+      # lock, not an `async:` lock. Derive the right key from metadata.
+      def parent_live_lock?(meta)
+        if meta["parent_is_map_element"]
+          @storage.lock_held?("map_element:#{meta["outer_map_id"]}:#{meta["outer_index"]}")
+        else
+          @storage.lock_held?("async:#{meta["parent_context_id"]}")
+        end
+      end
+
+      def parent_already_collected?(meta)
+        data = @storage.retrieve_context(meta["parent_context_id"], meta["parent_reactor_class_name"])
+        return false unless data
+
+        results = data["intermediate_results"] || {}
+        status = data["status"].to_s
+        results.key?(meta["step_name"].to_s) || %w[completed failed skipped].include?(status)
+      end
+
+      def retrigger_collector(meta)
+        @async_router.perform_map_collection_async(
+          parent_context_id: meta["parent_context_id"],
+          map_id: meta["map_id"],
+          parent_reactor_class_name: meta["parent_reactor_class_name"],
+          step_name: meta["step_name"],
+          strict_ordering: meta["strict_ordering"],
+          timeout: 3600
+        )
+      end
+    end
+  end
+end