ruby_reactor 0.5.0 → 0.5.2

data/lib/ruby_reactor/rspec/sidekiq_helpers.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module RSpec
+    # Async-job manipulation helpers. Names like `drain_async_jobs` are too
+    # generic to live in the global spec namespace, so this module is only
+    # auto-included into examples tagged `type: :reactor`. Specs that need it
+    # outside that tag should `include RubyReactor::RSpec::SidekiqHelpers`
+    # explicitly.
+    module SidekiqHelpers
+      # Drain every queued async job across all RubyReactor worker classes
+      # until the queues are empty. Recursive — handles jobs that re-enqueue
+      # themselves (e.g. ordered_lock snoozes) and worker chains that queue
+      # additional jobs.
+      def drain_async_jobs(max_iterations: 100)
+        SidekiqHelpers.drain_async_jobs(max_iterations: max_iterations)
+      end
+
+      # All currently-pending async jobs, wrapped in `PendingJob` so callers
+      # can perform individual jobs out-of-order (e.g. to assert
+      # ordered_lock's snooze behavior) without touching Sidekiq internals.
+      def pending_async_jobs
+        SidekiqHelpers.pending_async_jobs
+      end
+
+      PendingJob = Struct.new(:worker_class, :raw) do
+        def perform!
+          worker_class.jobs.delete(raw)
+          worker_class.new.perform(*raw["args"])
+        end
+
+        def args
+          raw["args"]
+        end
+      end
+
+      def self.worker_classes
+        @worker_classes ||= [
+          RubyReactor::SidekiqWorkers::Worker,
+          RubyReactor::SidekiqWorkers::MapElementWorker,
+          RubyReactor::SidekiqWorkers::MapCollectorWorker
+        ]
+      end
+
+      def self.drain_async_jobs(max_iterations: 100)
+        return unless defined?(Sidekiq::Testing)
+
+        max_iterations.times do
+          processed_any = false
+          worker_classes.each do |worker_class|
+            while (job = worker_class.jobs.shift)
+              worker_class.new.perform(*job["args"])
+              processed_any = true
+            end
+          end
+
+          break unless processed_any
+        end
+      end
+
+      def self.pending_async_jobs
+        return [] unless defined?(Sidekiq::Testing)
+
+        worker_classes.flat_map do |worker_class|
+          worker_class.jobs.map { |raw| PendingJob.new(worker_class, raw) }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_reactor/rspec/storage_reset.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module RSpec
+    # Test-only `reset!` impls layered onto storage adapters at framework
+    # load time. Kept out of `lib/ruby_reactor/storage/*` so production code
+    # never gains a "wipe everything" entry point.
+    module StorageReset
+      module RedisAdapterReset
+        def reset!
+          @redis.flushdb
+        end
+      end
+
+      def self.install!
+        return if @installed
+
+        ::RubyReactor::Storage::RedisAdapter.prepend(RedisAdapterReset)
+        @installed = true
+      end
+    end
+  end
+end

data/lib/ruby_reactor/rspec/test_subject.rb

@@ -246,6 +246,8 @@ module RubyReactor
                   end
                 end
           RubyReactor::Success.new(val)
+        when "skipped"
+          skipped_result(ctx)
         when "running"
           # Try to determine if it is truly running or if we just missed the completion
           if @process_jobs && defined?(Sidekiq::Testing)
@@ -269,6 +271,17 @@ module RubyReactor
         end
       end
 
+      # A clean halt: either a `with_period` gate or a step returning
+      # `RubyReactor.Skipped(...)`. The sync run already produced the exact
+      # Skipped (reason/step intact) — surface it. For async runs the worker
+      # swallows the return value, so rebuild from the trace.
+      def skipped_result(ctx)
+        return @run_result if @run_result.is_a?(RubyReactor::Skipped)
+
+        entry = ctx.execution_trace.reverse.find { |t| t[:type].to_s == "skipped" }
+        RubyReactor::Skipped.new(reason: entry&.dig(:reason), step_name: entry&.dig(:step))
+      end
+
       def success?
         ensure_executed!
         @reactor_instance.context.status.to_s == "completed"
@@ -419,34 +432,7 @@ module RubyReactor
       def process_pending_jobs
         return unless defined?(Sidekiq::Testing)
 
-        # Loop until no more jobs are being queued
-        # This handles batched map execution where jobs queue more jobs
-        max_iterations = 100
-        iterations = 0
-
-        while iterations < max_iterations
-          iterations += 1
-          jobs_processed = false
-
-          # Known worker classes to check
-          worker_classes = [
-            RubyReactor::SidekiqWorkers::Worker,
-            RubyReactor::SidekiqWorkers::MapElementWorker,
-            RubyReactor::SidekiqWorkers::MapCollectorWorker
-          ]
-
-          worker_classes.each do |worker_class|
-            while worker_class.jobs.any?
-              job = worker_class.jobs.shift
-              worker_class.new.perform(*job["args"])
-              jobs_processed = true
-            end
-          end
-
-          break unless jobs_processed
-        end
-
-        # Final reload
+        SidekiqHelpers.drain_async_jobs
         @reactor_instance = @reactor_class.find(@reactor_instance.context.context_id)
       end
 

data/lib/ruby_reactor/rspec.rb

@@ -2,17 +2,54 @@
 
 require_relative "rspec/helpers"
 require_relative "rspec/matchers"
+require_relative "rspec/sidekiq_helpers"
+require_relative "rspec/storage_reset"
 require_relative "rspec/test_subject"
 
 module RubyReactor
   module RSpec
+    # Examples opt into RubyReactor's RSpec setup (Sidekiq fake mode,
+    # storage wipe, snooze knob reset) by declaring `type: :reactor`.
+    REACTOR_METADATA = { type: :reactor }.freeze
+
+    DEFAULT_SNOOZE_BASE_DELAY = 5
+    DEFAULT_SNOOZE_JITTER = 5
+    DEFAULT_SNOOZE_MAX_ATTEMPTS = 20
+
     def self.configure(config)
       require_relative "rspec/step_executor_patch"
 
       config.include RubyReactor::RSpec::Helpers
       config.include RubyReactor::RSpec::Matchers
+      config.include RubyReactor::RSpec::SidekiqHelpers, REACTOR_METADATA
 
       ::RubyReactor::Executor::StepExecutor.prepend(RubyReactor::RSpec::StepExecutorPatch)
+      StorageReset.install!
+
+      config.before(:each, REACTOR_METADATA) do
+        RubyReactor::RSpec.prepare_example!
+      end
+    end
+
+    # Idempotent setup invoked before each `type: :reactor` example. Restores
+    # Sidekiq fake mode, clears the queues, wipes the storage adapter, and
+    # rolls back snooze knobs so cross-example bleed-through can't happen.
+    def self.prepare_example!
+      if defined?(::Sidekiq::Testing)
+        begin
+          ::Sidekiq::Testing.fake! unless ::Sidekiq::Testing.fake?
+        rescue ::Sidekiq::Testing::TestModeAlreadySetError
+          # Nested fake!/inline! block already active in this thread; leave it.
+        end
+        ::Sidekiq::Worker.clear_all
+      end
+
+      adapter = ::RubyReactor.configuration.storage_adapter
+      adapter.reset! if adapter.respond_to?(:reset!)
+
+      ::RubyReactor.configuration.lock_snooze_base_delay = DEFAULT_SNOOZE_BASE_DELAY
+      ::RubyReactor.configuration.lock_snooze_jitter = DEFAULT_SNOOZE_JITTER
+      ::RubyReactor.configuration.lock_snooze_max_attempts = DEFAULT_SNOOZE_MAX_ATTEMPTS
     end
   end
 end

data/lib/ruby_reactor/sidekiq_workers/worker.rb

@@ -48,18 +48,29 @@ module RubyReactor
           # Resume execution from the failed step
           executor = Executor.new(context.reactor_class, {}, context)
           executor.resume_execution
-          executor.save_context
+          # Skip the post-run save when the executor deliberately suppressed
+          # persistence (stale-batch redelivery of an already-terminal context)
+          # — re-saving here would clobber the stored terminal record with this
+          # run's stale in-memory status.
+          executor.save_context unless executor.skip_context_persist?
 
           # Return the executor (which now has the result stored in it)
           executor
         rescue RubyReactor::Lock::AcquisitionError,
                RubyReactor::Semaphore::AcquisitionError,
-               RubyReactor::RateLimit::ExceededError => e
-          # Snooze on expected concurrency or rate contention. We avoid
-          # Sidekiq's native retry path so this doesn't burn the job's retry
-          # budget or appear as an error in dashboards. After the configured
-          # cap is reached we escalate by marking the reactor as failed.
+               RubyReactor::RateLimit::ExceededError,
+               RubyReactor::OrderedLock::WaitError => e
+          # Snooze on expected concurrency, rate, or ordering contention.
+          # OrderedLock::WaitError carries a poison-pill-derived retry hint,
+          # consumed by compute_snooze_delay below. We avoid Sidekiq's native
+          # retry path so this doesn't burn the job's retry budget or appear
+          # as an error in dashboards. After the configured cap is reached we
+          # escalate by marking the reactor as failed.
           handle_snooze(serialized_context, reactor_class_name, context, snooze_count, e)
+        rescue RubyReactor::RateLimitRegistry::UnknownLimitError => e
+          # Permanent configuration error — snoozing or retrying the same job
+          # will keep failing. Mark the context failed immediately.
+          escalate_snooze(context, snooze_count, e)
         end
       end
 
@@ -69,7 +80,15 @@ module RubyReactor
         config = RubyReactor.configuration
         max = config.lock_snooze_max_attempts
 
-        if max != :infinity && snooze_count >= max
+        # OrderedLock::WaitError bypasses the snooze cap. The gate's
+        # poison_pill_timeout is the only meaningful upper bound on how long a
+        # nonce can legitimately wait; capping snoozes would either fail jobs
+        # prematurely or strand the nonce in `assigned_at` until poison_pill
+        # eventually advances past it. Snooze until the gate passes (or poison
+        # auto-advance moves the cursor past us).
+        capped = !error.is_a?(RubyReactor::OrderedLock::WaitError)
+
+        if capped && max != :infinity && snooze_count >= max
           escalate_snooze(context, snooze_count, error)
           return
         end
@@ -82,17 +101,32 @@ module RubyReactor
       # (RateLimit::ExceededError carries the time until the bucket rolls);
       # otherwise fall back to the configured base + jitter for lock/semaphore
       # contention which has no precise hint.
+      #
+      # OrderedLock::WaitError is deliberately excluded from the hint path: its
+      # `retry_after_seconds` is the poison-pill window (the upper bound before
+      # a *dead* blocker is force-advanced), NOT how long the *live* blocker
+      # will take — which is usually milliseconds. Snoozing for the full window
+      # would make every out-of-order nonce sleep up to poison_pill_timeout even
+      # though its blocker finishes immediately, collapsing throughput. Re-poll
+      # at the base delay instead; poison auto-advance still clears a genuinely
+      # dead blocker on a later gate.
       def compute_snooze_delay(config, error)
         jitter = config.lock_snooze_jitter.to_f
         jitter_amount = jitter.positive? ? rand(0.0..jitter) : 0.0
 
-        if error.respond_to?(:retry_after_seconds) && error.retry_after_seconds
+        if hinted_retry?(error)
           [error.retry_after_seconds.to_f, 0.1].max + jitter_amount
         else
           config.lock_snooze_base_delay.to_f + jitter_amount
         end
       end
 
+      def hinted_retry?(error)
+        return false if error.is_a?(RubyReactor::OrderedLock::WaitError)
+
+        error.respond_to?(:retry_after_seconds) && error.retry_after_seconds
+      end
+
       def escalate_snooze(context, snooze_count, error)
         RubyReactor.configuration.logger.warn(
           "RubyReactor snooze limit reached after #{snooze_count} attempts " \
@@ -113,6 +147,14 @@ module RubyReactor
           serialized,
           reactor_class_name
         )
+
+        # Escalation is a terminal Failure that never reaches the Executor's
+        # ensure path, so advance the ordered-lock cursor here. Without this
+        # the nonce stays stranded in assigned_at (successors stall for the
+        # full poison_pill_timeout) and, worse, the strict-mode chain marker
+        # is never recorded — successors would RUN instead of being skipped.
+        info = Executor::OrderedLockSupport.info_from(context)
+        Executor::OrderedLockSupport.advance_with_retry(info, failed: true) if info
       end
 
       def log_infrastructure_failure(msg, exception)

data/lib/ruby_reactor/storage/redis_adapter.rb

@@ -7,6 +7,7 @@ module RubyReactor
   module Storage
     class RedisAdapter < Adapter
       include RedisLocking
+      include RedisOrderedLocking
 
       def initialize(redis_config)
         super()