ruby_reactor 0.3.1 → 0.3.2

data/lib/ruby_reactor/lock.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  class Lock
+    class AcquisitionError < StandardError; end
+
+    # Minimum interval between auto-extend pings; protects very small TTLs.
+    MIN_EXTEND_INTERVAL = 1.0
+
+    attr_reader :key, :owner, :ttl, :wait, :auto_extend
+
+    def initialize(key, owner:, ttl: 60, wait: 0, auto_extend: true)
+      @key = "lock:#{key}"
+      @owner = owner
+      @ttl = ttl
+      @wait = wait
+      @auto_extend = auto_extend
+      @extender = nil
+      @extender_running = false
+    end
+
+    def acquire
+      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      loop do
+        if adapter.lock_acquire(@key, @owner, @ttl)
+          start_extender if @auto_extend
+          return true
+        end
+
+        if @wait.zero? || (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) >= @wait
+          raise AcquisitionError, "Could not acquire lock '#{@key}' for owner '#{@owner}'"
+        end
+
+        sleep 0.1
+      end
+    end
+
+    def release
+      stop_extender
+      adapter.lock_release(@key, @owner)
+    end
+
+    def synchronize
+      acquire
+      yield
+    ensure
+      release
+    end
+
+    private
+
+    def start_extender
+      return if @extender&.alive?
+
+      interval = [@ttl / 3.0, MIN_EXTEND_INTERVAL].max
+      @extender_running = true
+
+      @extender = Thread.new do
+        while @extender_running
+          sleep interval
+          break unless @extender_running
+
+          begin
+            adapter.lock_extend(@key, @owner, @ttl)
+          rescue StandardError => e
+            RubyReactor.configuration.logger.warn(
+              "Lock auto-extend failed for '#{@key}': #{e.message}"
+            )
+            break
+          end
+        end
+      end
+    end
+
+    def stop_extender
+      @extender_running = false
+      thread = @extender
+      @extender = nil
+      return unless thread
+
+      thread.wakeup if thread.alive?
+      thread.join(0.1)
+    rescue StandardError
+      # Best-effort shutdown; never let extender teardown break release.
+    end
+
+    def adapter
+      RubyReactor.configuration.storage_adapter
+    end
+  end
+end

data/lib/ruby_reactor/period.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  # Calendar-aligned bucket helpers for `with_period` dedup gating.
+  #
+  # A *bucket* is a deterministic string derived from the current UTC time and
+  # the configured period. Two calls in the same bucket dedup to the same
+  # Redis marker key; calls that cross a calendar boundary land in different
+  # buckets and run again.
+  module Period
+    SYMBOLIC_PERIODS = {
+      second: 1,
+      minute: 60,
+      hour: 60 * 60,
+      day: 60 * 60 * 24,
+      week: 60 * 60 * 24 * 7,
+      month: 60 * 60 * 24 * 31,
+      year: 60 * 60 * 24 * 366
+    }.freeze
+
+    # Build the bucket id for a given period at a given moment. UTC, calendar
+    # aligned for symbolic periods, index-based for integer seconds.
+    def self.bucket_id(every, now: Time.now.utc)
+      case every
+      when :second then now.strftime("%Y-%m-%dT%H-%M-%S")
+      when :minute then now.strftime("%Y-%m-%dT%H-%M")
+      when :hour   then now.strftime("%Y-%m-%dT%H")
+      when :day    then now.strftime("%Y-%m-%d")
+      when :week   then now.strftime("%G-W%V")
+      when :month  then now.strftime("%Y-%m")
+      when :year   then now.strftime("%Y")
+      when Integer
+        raise ArgumentError, "Period seconds must be positive" unless every.positive?
+
+        "i#{now.to_i / every}"
+      else
+        raise ArgumentError, "Unknown period: #{every.inspect}"
+      end
+    end
+
+    # TTL for the marker. Twice the period length so the marker survives clock
+    # skew across the boundary and reliably dedups the very next attempt.
+    def self.ttl_seconds(every)
+      base = period_seconds(every)
+      base * 2
+    end
+
+    def self.period_seconds(every)
+      case every
+      when Symbol
+        SYMBOLIC_PERIODS.fetch(every) do
+          raise ArgumentError, "Unknown period: #{every.inspect}"
+        end
+      when Integer
+        raise ArgumentError, "Period seconds must be positive" unless every.positive?
+
+        every
+      else
+        raise ArgumentError, "Unknown period: #{every.inspect}"
+      end
+    end
+
+    def self.key(base, every, now: Time.now.utc)
+      "period:#{base}:#{bucket_id(every, now: now)}"
+    end
+  end
+end

data/lib/ruby_reactor/rate_limit.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  # Distributed rate limiter (fixed-window counter, multi-window aware).
+  #
+  # A `RateLimit` is configured with one or more (period, limit) tuples.
+  # `check_and_increment!` atomically verifies every window has headroom and,
+  # if so, increments all of them. The check uses a single Lua script so
+  # nothing slips through between read and write.
+  #
+  # When any window is over-limit the call raises `ExceededError` carrying a
+  # `retry_after_seconds` hint (time until the tightest failing bucket rolls).
+  # The Sidekiq worker uses this hint to schedule a precise snooze.
+  class RateLimit
+    class ExceededError < StandardError
+      attr_reader :retry_after_seconds, :key_base, :limit, :period_seconds, :period_name
+
+      def initialize(message, retry_after_seconds:, key_base:, limit:, period_seconds:, period_name:)
+        super(message)
+        @retry_after_seconds = retry_after_seconds
+        @key_base = key_base
+        @limit = limit
+        @period_seconds = period_seconds
+        @period_name = period_name
+      end
+    end
+
+    attr_reader :key_base, :limits
+
+    # @param key_base [String] caller-provided key (e.g. "stripe:account_42")
+    # @param limits [Array<Hash>] each hash needs :period_seconds, :limit,
+    #   and :name (used in the Redis bucket key and the error message)
+    def initialize(key_base, limits:)
+      @key_base = key_base
+      @limits = limits
+    end
+
+    def check_and_increment!
+      now = Time.now.to_i
+      keys = @limits.map { |spec| bucket_key(spec, now) }
+      argv = [now]
+      @limits.each do |spec|
+        argv << spec[:period_seconds]
+        argv << spec[:limit]
+        argv << spec[:period_seconds] * 2 # TTL: generous, auto-cleans stale buckets
+      end
+
+      allowed, retry_after, failed_index = adapter.rate_limit_check_and_increment(keys, argv)
+      return true if allowed == 1
+
+      failed = @limits[failed_index - 1]
+      raise ExceededError.new(
+        "Rate limit '#{@key_base}' exceeded (#{failed[:limit]}/#{failed[:name]}); " \
+        "retry in #{retry_after}s",
+        retry_after_seconds: retry_after,
+        key_base: @key_base,
+        limit: failed[:limit],
+        period_seconds: failed[:period_seconds],
+        period_name: failed[:name]
+      )
+    end
+
+    private
+
+    def bucket_key(spec, now)
+      bucket_id = now / spec[:period_seconds]
+      "rate:#{@key_base}:#{spec[:name]}:#{bucket_id}"
+    end
+
+    def adapter
+      RubyReactor.configuration.storage_adapter
+    end
+  end
+end

data/lib/ruby_reactor/reactor.rb

@@ -4,6 +4,7 @@ module RubyReactor
   # rubocop:disable Metrics/ClassLength
   class Reactor
     include RubyReactor::Dsl::Reactor
+    include RubyReactor::Dsl::Lockable
 
     attr_reader :context, :result, :undo_trace, :execution_trace
 

data/lib/ruby_reactor/rspec/matchers.rb

@@ -2,20 +2,23 @@
 
 module RubyReactor
   module RSpec
+    # rubocop:disable Metrics/ModuleLength
     module Matchers
       # rubocop:disable Metrics/BlockLength
       ::RSpec::Matchers.define :be_success do
         match do |subject|
-          subject.ensure_executed!
+          subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
           subject.success?
         end
 
         failure_message do |subject|
-          result = subject.result
+          result = subject.respond_to?(:result) ? subject.result : subject
           if result&.failure?
             format_failure_message(result)
-          else
+          elsif subject.respond_to?(:reactor_instance)
             "expected reactor to be success, but failed (Status: #{subject.reactor_instance.context.status})"
+          else
+            "expected #{subject.inspect} to be success"
           end
         end
 
@@ -62,7 +65,7 @@ module RubyReactor
 
       ::RSpec::Matchers.define :be_failure do
         match do |subject|
-          subject.ensure_executed!
+          subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
           subject.failure?
         end
 
@@ -249,8 +252,172 @@ module RubyReactor
         end
       end
 
+      # ---------------------------------------------------------------------
+      # Lock / Semaphore / Rate-limit / Period state matchers
+      # ---------------------------------------------------------------------
+      #
+      # These assert against the live Redis state via the configured storage
+      # adapter, so they work for any test that has actually exercised the
+      # reactor (or interacted with the primitives directly).
+
+      def self.coordination_adapter
+        RubyReactor.configuration.storage_adapter
+      end
+
+      # Distinguishes `RubyReactor::Skipped` from a plain `Success`. Works on
+      # any object with a `skipped?` predicate.
+      #
+      # Examples:
+      #   expect(result).to be_skipped
+      #   expect(result).to be_skipped.because(:period)
+      #   expect(result).to be_skipped.at_step(:second)
+      ::RSpec::Matchers.define :be_skipped do
+        match do |subject|
+          subject.ensure_executed! if subject.respond_to?(:ensure_executed!)
+          actual = subject.respond_to?(:result) ? subject.result : subject
+          next false unless actual.respond_to?(:skipped?) && actual.skipped?
+          next false if @expected_reason && actual.reason != @expected_reason
+          next false if @expected_step && actual.step_name != @expected_step
+
+          true
+        end
+
+        chain :because do |reason|
+          @expected_reason = reason
+        end
+
+        chain :at_step do |step|
+          @expected_step = step
+        end
+
+        failure_message do |subject|
+          actual = subject.respond_to?(:result) ? subject.result : subject
+          if !actual.respond_to?(:skipped?) || !actual.skipped?
+            "expected result to be Skipped, got #{actual.class}"
+          elsif @expected_reason && actual.reason != @expected_reason
+            "expected Skipped reason #{@expected_reason.inspect}, got #{actual.reason.inspect}"
+          else
+            "expected Skipped at_step #{@expected_step.inspect}, got #{actual.step_name.inspect}"
+          end
+        end
+
+        failure_message_when_negated do
+          "expected result not to be Skipped"
+        end
+      end
+
+      # Asserts that an exclusive lock is currently held in Redis. Subject is
+      # the user-provided lock key (without the "lock:" prefix).
+      #
+      #   expect("order:42").to be_locked
+      #   expect("order:42").to be_locked.by("ctx-abc")
+      ::RSpec::Matchers.define :be_locked do
+        match do |key|
+          info = Matchers.coordination_adapter.lock_info("lock:#{key}")
+          next false unless info
+          next true unless @expected_owner
+
+          info[:owner] == @expected_owner
+        end
+
+        chain :by do |owner|
+          @expected_owner = owner
+        end
+
+        failure_message do |key|
+          info = Matchers.coordination_adapter.lock_info("lock:#{key}")
+          if info.nil?
+            "expected lock 'lock:#{key}' to be held, but it is free"
+          else
+            "expected lock 'lock:#{key}' to be held by #{@expected_owner.inspect}, " \
+              "but is held by #{info[:owner].inspect}"
+          end
+        end
+
+        failure_message_when_negated do |key|
+          info = Matchers.coordination_adapter.lock_info("lock:#{key}")
+          "expected lock 'lock:#{key}' not to be held, but is held by #{info[:owner].inspect}"
+        end
+      end
+
+      # Asserts the number of unallocated semaphore tokens. Subject is the
+      # user-provided semaphore name (without the "semaphore:" prefix).
+      #
+      #   expect("api_limit").to have_available_tokens(3)
+      ::RSpec::Matchers.define :have_available_tokens do |expected|
+        match do |name|
+          Matchers.coordination_adapter.semaphore_state(name)[:available] == expected
+        end
+
+        failure_message do |name|
+          state = Matchers.coordination_adapter.semaphore_state(name)
+          "expected semaphore '#{name}' to have #{expected} available tokens, " \
+            "got #{state[:available]} (held: #{state[:held]}, limit: #{state[:limit]})"
+        end
+      end
+
+      # Asserts the number of currently-checked-out semaphore tokens.
+      #
+      #   expect("api_limit").to have_held_tokens(2)
+      ::RSpec::Matchers.define :have_held_tokens do |expected|
+        match do |name|
+          Matchers.coordination_adapter.semaphore_state(name)[:held] == expected
+        end
+
+        failure_message do |name|
+          state = Matchers.coordination_adapter.semaphore_state(name)
+          "expected semaphore '#{name}' to have #{expected} held tokens, " \
+            "got #{state[:held]} (available: #{state[:available]}, limit: #{state[:limit]})"
+        end
+      end
+
+      # Asserts the current rate-limit counter for a (key_base, period) pair.
+      # Use `.for(period_unit)` to specify which window.
+      #
+      #   expect("stripe:42").to have_rate_limit_count(3).for(:second)
+      ::RSpec::Matchers.define :have_rate_limit_count do |expected|
+        match do |key_base|
+          raise ArgumentError, "have_rate_limit_count requires .for(period)" unless @period
+
+          Matchers.coordination_adapter.rate_limit_count(key_base, @period) == expected
+        end
+
+        chain :for do |period|
+          @period = period
+        end
+
+        failure_message do |key_base|
+          actual = Matchers.coordination_adapter.rate_limit_count(key_base, @period)
+          "expected rate-limit '#{key_base}' (#{@period}) count to be #{expected}, got #{actual}"
+        end
+      end
+
+      # Asserts that a `with_period` bucket has been marked. Use `.for(period)`.
+      #
+      #   expect("daily_report:7").to be_period_marked.for(:day)
+      ::RSpec::Matchers.define :be_period_marked do
+        match do |key_base|
+          raise ArgumentError, "be_period_marked requires .for(period)" unless @period
+
+          Matchers.coordination_adapter.period_marker?(key_base, @period)
+        end
+
+        chain :for do |period|
+          @period = period
+        end
+
+        failure_message do |key_base|
+          "expected period bucket #{RubyReactor::Period.key(key_base, @period).inspect} to be marked, but it is not"
+        end
+
+        failure_message_when_negated do |key_base|
+          "expected period bucket #{RubyReactor::Period.key(key_base, @period).inspect} not to be marked, but it is"
+        end
+      end
+
       # Add more matchers as per plan
       # rubocop:enable Metrics/BlockLength
     end
+    # rubocop:enable Metrics/ModuleLength
   end
 end

data/lib/ruby_reactor/semaphore.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  class Semaphore
+    class AcquisitionError < StandardError; end
+
+    attr_reader :key, :limit, :wait, :token
+
+    def initialize(key, limit: 1, wait: 0)
+      @key = "semaphore:#{key}"
+      @limit = limit
+      @wait = wait
+      @token = nil
+    end
+
+    def acquire # rubocop:disable Naming/PredicateMethod
+      ensure_initialized
+
+      token = adapter.semaphore_acquire(@key, timeout: @wait)
+      raise AcquisitionError, "Could not acquire semaphore '#{@key}' within #{@wait} seconds" unless token
+
+      @token = token
+      true
+    end
+
+    # Returns true if a held token was returned to the pool.
+    # Idempotent: a second release (or one without a prior acquire) is a no-op.
+    def release
+      return false unless @token
+
+      result = adapter.semaphore_release(@key, @token, @limit)
+      @token = nil
+      result
+    end
+
+    def synchronize
+      acquire
+      yield
+    ensure
+      release
+    end
+
+    private
+
+    def ensure_initialized
+      # Double-checked locking pattern optimized for Redis
+      # 1. Optimistic check (if key exists)
+      return if adapter.semaphore_exists?(@key)
+
+      # 2. Try to init
+      adapter.semaphore_init(@key, @limit)
+    end
+
+    def adapter
+      RubyReactor.configuration.storage_adapter
+    end
+  end
+end

data/lib/ruby_reactor/sidekiq_workers/worker.rb

@@ -17,7 +17,7 @@ module RubyReactor
         # Handle infrastructure failures (network, Redis, etc.)
       end
 
-      def perform(serialized_context, reactor_class_name = nil)
+      def perform(serialized_context, reactor_class_name = nil, snooze_count = 0)
         context = ContextSerializer.deserialize(serialized_context)
 
         # If reactor_class_name is provided, use it to get the reactor class
@@ -35,18 +35,80 @@ module RubyReactor
         # Mark that we're executing inline to prevent nested async calls
         context.inline_async_execution = true
 
-        # Resume execution from the failed step
-        executor = Executor.new(context.reactor_class, {}, context)
-        executor.resume_execution
-        executor.save_context
-        executor.result
+        begin
+          # Resume execution from the failed step
+          executor = Executor.new(context.reactor_class, {}, context)
+          executor.resume_execution
+          executor.save_context
+
+          # Return the executor (which now has the result stored in it)
+          executor
+        rescue RubyReactor::Lock::AcquisitionError,
+               RubyReactor::Semaphore::AcquisitionError,
+               RubyReactor::RateLimit::ExceededError => e
+          # Snooze on expected concurrency or rate contention. We avoid
+          # Sidekiq's native retry path so this doesn't burn the job's retry
+          # budget or appear as an error in dashboards. After the configured
+          # cap is reached we escalate by marking the reactor as failed.
+          handle_snooze(serialized_context, reactor_class_name, context, snooze_count, e)
+        end
       end
 
       private
 
+      def handle_snooze(serialized_context, reactor_class_name, context, snooze_count, error)
+        config = RubyReactor.configuration
+        max = config.lock_snooze_max_attempts
+
+        if max != :infinity && snooze_count >= max
+          escalate_snooze(context, snooze_count, error)
+          return
+        end
+
+        delay = compute_snooze_delay(config, error)
+        self.class.perform_in(delay, serialized_context, reactor_class_name, snooze_count + 1)
+      end
+
+      # Use the error's `retry_after_seconds` hint when available
+      # (RateLimit::ExceededError carries the time until the bucket rolls);
+      # otherwise fall back to the configured base + jitter for lock/semaphore
+      # contention which has no precise hint.
+      def compute_snooze_delay(config, error)
+        jitter = config.lock_snooze_jitter.to_f
+        jitter_amount = jitter.positive? ? rand(0.0..jitter) : 0.0
+
+        if error.respond_to?(:retry_after_seconds) && error.retry_after_seconds
+          [error.retry_after_seconds.to_f, 0.1].max + jitter_amount
+        else
+          config.lock_snooze_base_delay.to_f + jitter_amount
+        end
+      end
+
+      def escalate_snooze(context, snooze_count, error)
+        RubyReactor.configuration.logger.warn(
+          "RubyReactor snooze limit reached after #{snooze_count} attempts " \
+          "for context #{context.context_id}: #{error.message}"
+        )
+
+        context.status = :failed
+        context.failure_reason = {
+          message: error.message,
+          exception_class: error.class.name,
+          snooze_attempts: snooze_count
+        }
+
+        serialized = ContextSerializer.serialize(context)
+        reactor_class_name = context.reactor_class&.name || "AnonymousReactor"
+        RubyReactor.configuration.storage_adapter.store_context(
+          context.context_id,
+          serialized,
+          reactor_class_name
+        )
+      end
+
       def log_infrastructure_failure(msg, exception)
-        ::Sidekiq.logger.error("RubyReactor infrastructure failure: #{exception.message}")
-        ::Sidekiq.logger.error("Job details: #{msg.inspect}")
+        RubyReactor.configuration.logger.error("RubyReactor infrastructure failure: #{exception.message}")
+        RubyReactor.configuration.logger.error("Job details: #{msg.inspect}")
       end
     end
   end

data/lib/ruby_reactor/storage/redis_adapter.rb

@@ -6,6 +6,8 @@ require "json"
 module RubyReactor
   module Storage
     class RedisAdapter < Adapter
+      include RedisLocking
+
       def initialize(redis_config)
         super()
         @redis = Redis.new(redis_config)