ruby_reactor 0.5.4 → 0.7.0

data/lib/ruby_reactor/step_worker.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  # The body of one dispatched `async_step`, shared by every queueing backend
+  # (`Adapters::Sidekiq::StepWorker`, `Adapters::ActiveJob::StepWorker`) exactly
+  # as `Map::ElementExecutor` is shared by the map element workers.
+  #
+  # It is deliberately NOT a reactor run: it loads the parent context, resolves
+  # just this one step's arguments against it, runs the step body, writes the
+  # durable Step Result Record, and publishes the completion signal. Ordering is
+  # load-bearing — the record is written BEFORE the signal, so a reader that
+  # misses the (at-most-once) signal still finds the answer on its next
+  # fallback re-check.
+  class StepWorker
+    class << self
+      def perform(arguments)
+        arguments = arguments.transform_keys(&:to_sym)
+        new(**slice_arguments(arguments)).perform
+      end
+
+      private
+
+      def slice_arguments(arguments)
+        {
+          root_context_id: arguments[:root_context_id],
+          reactor_class_name: arguments[:reactor_class_name],
+          step_context_id: arguments[:step_context_id],
+          step_name: arguments[:step_name].to_sym
+        }
+      end
+    end
+
+    def initialize(root_context_id:, reactor_class_name:, step_context_id:, step_name:)
+      @root_context_id = root_context_id
+      @reactor_class_name = reactor_class_name
+      @step_context_id = step_context_id || root_context_id
+      @step_name = step_name
+    end
+
+    # The lock is what makes a lost unit recoverable: the record alone cannot say
+    # whether a `dispatched` unit is mid-flight or gone, so StepSweeper reads this
+    # lock as the liveness signal. It also drops a duplicate delivery rather than
+    # running the body a second time.
+    def perform
+      lock = acquire_liveness_lock
+      return if lock == :contended
+
+      perform_unit
+    ensure
+      lock.release if lock.respond_to?(:release)
+    end
+
+    private
+
+    def perform_unit
+      context = load_step_context
+      return record_missing_parent unless context
+
+      step_config = context.reactor_class&.steps&.[](@step_name)
+      return record_missing_step unless step_config
+
+      complete(run_step(context, step_config), context)
+    rescue StandardError => e
+      # The unit's failure belongs in its record, where a reader can see it.
+      # Raising instead would hand the job to the backend's retry machinery to
+      # fail identically N more times while every reader waits out its timeout.
+      log(:error, "failed", error: "#{e.class}: #{e.message}")
+      complete(RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name), nil)
+    end
+
+    def acquire_liveness_lock
+      # Inline testing re-enters this frame synchronously, so the lock would
+      # self-contend; it only guards cross-process delivery, impossible inline.
+      return :inline if inline_testing_mode?
+
+      lock = RubyReactor::Lock.new(
+        RubyReactor.async_step_lock_key(@step_context_id, @step_name),
+        owner: SecureRandom.uuid, ttl: RubyReactor.configuration.context_lock_ttl,
+        wait: 0, auto_extend: true
+      )
+      lock.acquire
+      lock
+    rescue RubyReactor::Lock::AcquisitionError
+      log(:info, "duplicate_dropped")
+      :contended
+    end
+
+    def inline_testing_mode?
+      defined?(Sidekiq::Testing) && Sidekiq::Testing.respond_to?(:inline?) && Sidekiq::Testing.inline?
+    end
+
+    def run_step(context, step_config)
+      arguments = resolve_arguments(step_config, context)
+      log(:info, "running")
+
+      attempt = 0
+      result = nil
+
+      loop do
+        attempt += 1
+        result = execute_step_body(step_config, arguments, context)
+        break unless retry?(step_config, result, attempt)
+
+        delay = backoff_delay(step_config, attempt)
+        log(:warn, "retrying", attempt: attempt, delay: delay)
+        sleep(delay)
+      end
+
+      result
+    end
+
+    def execute_step_body(step_config, arguments, context)
+      result =
+        if step_config.has_run_block?
+          args = arguments.empty? ? context.inputs : arguments
+          step_config.run_block.call(args, context)
+        elsif step_config.has_impl?
+          step_config.impl.run(arguments, context)
+        else
+          RubyReactor.Failure("Step '#{@step_name}' has no implementation")
+        end
+
+      normalize(result)
+    rescue StandardError => e
+      RubyReactor.Failure(e, step_name: @step_name, reactor_name: @reactor_class_name)
+    end
+
+    # Mirrors `Executor::RetryManager#can_retry_step?` for the one path that
+    # never reaches it: an `async_step`'s body runs entirely inside this
+    # worker, so retries here must be attempted synchronously in-process
+    # rather than requeued as a new job.
+    def retry?(step_config, result, attempt)
+      return false unless result.is_a?(RubyReactor::Failure) && result.retryable?
+
+      step_config.retryable? && attempt < step_config.retry_config[:max_attempts]
+    end
+
+    def backoff_delay(step_config, attempt)
+      RetryContext.calculate_backoff_delay(
+        attempt, step_config.retry_config[:backoff], step_config.retry_config[:base_delay]
+      )
+    end
+
+    def normalize(result)
+      return result if result.is_a?(RubyReactor::Success) || result.is_a?(RubyReactor::Failure)
+
+      RubyReactor.Success(result)
+    end
+
+    def resolve_arguments(step_config, context)
+      step_config.arguments.to_h do |arg_name, arg_config|
+        value = arg_config[:source].resolve(context)
+        value = arg_config[:transform].call(value) if arg_config[:transform]
+        [arg_name, value]
+      end
+    end
+
+    # Write first, publish second. The record is the answer; the signal only
+    # saves the reader a fallback interval.
+    def complete(result, context)
+      storage.store_step_result(
+        @step_context_id, @step_name,
+        {
+          "status" => "completed",
+          "success" => result.success?,
+          "result" => ContextSerializer.serialize_value(result.success? ? result.value : result.to_h),
+          "completed_at" => Time.now.iso8601
+        },
+        @reactor_class_name
+      )
+      log(result.success? ? :info : :warn, result.success? ? "completed" : "completed_with_failure")
+      storage.publish(RubyReactor.async_step_channel(@step_context_id, @step_name), "done")
+      result
+    ensure
+      # A step body may have mutated the sub-context; nothing else will persist
+      # it, and the dashboard reads the parent's blob.
+      save_root(context) if context
+    end
+
+    def record_missing_parent
+      # The parent was swept or outlived its retention window, so this
+      # unit's arguments can never be resolved. A record saying so beats a
+      # reader waiting out the full timeout for an answer that will never come.
+      log(:error, "parent_context_missing")
+      complete(
+        RubyReactor.Failure(
+          "Parent context #{@step_context_id} for async_step :#{@step_name} is no longer in storage " \
+          "(swept, or dispatched longer ago than `context_ttl`). The step's arguments cannot be resolved."
+        ),
+        nil
+      )
+    end
+
+    def record_missing_step
+      log(:error, "step_not_found")
+      complete(
+        RubyReactor.Failure("async_step :#{@step_name} is not defined on #{@reactor_class_name}"),
+        nil
+      )
+    end
+
+    def load_step_context
+      data = storage.retrieve_context(@root_context_id, @reactor_class_name)
+      return nil unless data
+
+      root = ContextSerializer.deserialize_hash(data)
+      @root_context = root
+      found = find_context(root, @step_context_id)
+      # The step runs in its own job; nothing it reaches should hand off again.
+      found&.inline_async_execution = true
+      found
+    rescue RubyReactor::Error::DeserializationError, RubyReactor::Error::SchemaVersionError => e
+      log(:error, "parent_context_unreadable", error: "#{e.class}: #{e.message}")
+      nil
+    end
+
+    def find_context(context, target_id)
+      return context if context.context_id == target_id
+
+      context.composed_contexts.each_value do |entry|
+        next unless entry.is_a?(Hash) && entry[:context].is_a?(RubyReactor::Context)
+
+        found = find_context(entry[:context], target_id)
+        return found if found
+      end
+      nil
+    end
+
+    def save_root(_context)
+      return unless @root_context
+
+      storage.store_context(@root_context.context_id, ContextSerializer.serialize(@root_context),
+                            @reactor_class_name)
+    rescue StandardError => e
+      RubyReactor.configuration.logger.warn(
+        "RubyReactor: async_step :#{@step_name} could not persist its parent context: #{e.message}"
+      )
+    end
+
+    def storage
+      RubyReactor.configuration.storage_adapter
+    end
+
+    # Machine-parseable, and carrying enough identity to correlate a
+    # worker-side outcome with the parent execution — which matters more here
+    # than elsewhere, because a fire-and-forget failure may have no other surface.
+    def log(level, event, **extra)
+      fields = {
+        event: "ruby_reactor.async_step.#{event}",
+        reactor: @reactor_class_name,
+        step: @step_name,
+        execution_id: @step_context_id
+      }.merge(extra)
+
+      RubyReactor.configuration.logger.public_send(
+        level, fields.map { |k, v| "#{k}=#{v.inspect}" }.join(" ")
+      )
+    end
+  end
+end

data/lib/ruby_reactor/storage/adapter.rb

@@ -15,6 +15,27 @@ module RubyReactor
         raise NotImplementedError
       end
 
+      # The durable outcome of one `async_step`, keyed by (parent context, step
+      # name). A separate worker writes it concurrently with the still-running
+      # parent, so it deliberately lives OUTSIDE the parent's context blob —
+      # writing into that blob from two processes would race.
+      #
+      # `record` is a plain hash: at minimum `status` ("dispatched" or
+      # "completed"); a completed record also carries the serialized outcome.
+      # The `dispatched` record is written before the job is enqueued, so it
+      # doubles as the re-attach marker on recovery.
+      def store_step_result(context_id, step_name, record, reactor_class_name)
+        raise NotImplementedError
+      end
+
+      def retrieve_step_result(context_id, step_name, reactor_class_name)
+        raise NotImplementedError
+      end
+
+      def scan_step_results(count: 1000)
+        raise NotImplementedError
+      end
+
       def retrieve_map_results(map_id, reactor_class_name, strict_ordering: true)
         raise NotImplementedError
       end
@@ -63,7 +84,7 @@ module RubyReactor
         raise NotImplementedError
       end
 
-      def scan_reactors(pattern: "*", count: 50)
+      def scan_reactors(pattern: "*", count: 50, include_dispatched_children: false)
         raise NotImplementedError
       end
 

data/lib/ruby_reactor/storage/redis_adapter.rb

@@ -8,9 +8,12 @@ module RubyReactor
     class RedisAdapter < Adapter
       include RedisLocking
       include RedisOrderedLocking
+      include RedisStepResults
+      include RedisPubSub
 
       def initialize(redis_config)
         super()
+        @redis_config = redis_config
         @redis = Redis.new(redis_config)
       end
 
@@ -174,20 +177,12 @@ module RubyReactor
         @redis.del(key)
       end
 
-      def subscribe(channel, &block)
-        @redis.subscribe(channel, &block)
-      end
-
-      def publish(channel, message)
-        @redis.publish(channel, message)
-      end
-
       def expire(key, seconds)
         @redis.expire(key, seconds)
       end
 
       # New methods for API
-      def scan_reactors(pattern: "reactor:*:context:*", count: 50)
+      def scan_reactors(pattern: "reactor:*:context:*", count: 50, include_dispatched_children: false)
         # Use SCAN to find keys matching the pattern
         results = []
         batch_keys = []
@@ -199,7 +194,7 @@ module RubyReactor
 
           # specific batch size for MGET processing
           if batch_keys.size >= 50
-            results.concat(fetch_and_filter_reactors(batch_keys))
+            results.concat(fetch_and_filter_reactors(batch_keys, include_dispatched_children))
             batch_keys = []
 
             # Stop if we have enough results
@@ -208,7 +203,7 @@ module RubyReactor
         end
 
         # Process remaining keys
-        results.concat(fetch_and_filter_reactors(batch_keys)) if batch_keys.any?
+        results.concat(fetch_and_filter_reactors(batch_keys, include_dispatched_children)) if batch_keys.any?
 
         results.take(count)
       end
@@ -320,14 +315,18 @@ module RubyReactor
         RubyReactor.configuration.context_ttl
       end
 
-      def fetch_and_filter_reactors(keys)
+      def fetch_and_filter_reactors(keys, include_dispatched_children = false)
         return [] if keys.empty?
 
         json_results = @redis.mget(*keys)
 
         json_results.compact.map do |json|
           data = JSON.parse(json)
-          next if data["parent_context_id"] # Skip nested reactors
+          next if data["parent_context_id"] && !(include_dispatched_children && dispatched_child?(data))
+          # Skip non-context records (e.g. async_step Step Result Records) whose
+          # keys are a "reactor:*:context:*" substring match on the SCAN glob
+          # (context:#{id}:step_result:#{name}) but aren't a reactor context.
+          next unless data["reactor_class"]
 
           {
             id: data["context_id"],
@@ -339,6 +338,11 @@ module RubyReactor
         end.compact
       end
 
+      # An `async_reactor` child owns its own job, so a lost job strands it like
+      # a top-level reactor. Compose children (inline) and map elements
+      # (Map::Sweeper's) carry no marker, so neither is swept.
+      def dispatched_child?(data) = data.dig("private_data", "async_dispatched")
+
       def context_key(context_id, reactor_class_name)
         "reactor:#{reactor_class_name}:context:#{context_id}"
       end

data/lib/ruby_reactor/storage/redis_locking.rb

@@ -123,6 +123,13 @@ module RubyReactor
         @redis.del("#{key}:init")
       end
 
+      # Whether a token is still checked out — used to re-adopt a slot held
+      # across a parked async wait.
+      def semaphore_held(key, token)
+        @redis.sismember("#{key}:held", token)
+      end
+      alias semaphore_held? semaphore_held
+
       def semaphore_acquire(key, timeout: 0)
         held_key = "#{key}:held"
 

data/lib/ruby_reactor/storage/redis_pub_sub.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Storage
+    # The completion-signal channel behind the notified wait. Pure latency
+    # optimisation: at-most-once, unpersisted, never load-bearing — every waiting
+    # path ends at a durable record, so a lost signal costs a fallback interval
+    # and never correctness.
+    module RedisPubSub
+      # SUBSCRIBE puts a connection into subscriber mode — every other command on
+      # it then fails — so this MUST NOT use the shared client, or one waiter
+      # would poison storage for the whole process. A dedicated connection is
+      # opened per subscription and closed on the way out.
+      #
+      # Blocks the calling thread until the block returns truthy for a message
+      # (completion signals are one-shot) or the thread is killed.
+      def subscribe(channel, &block)
+        connection = Redis.new(@redis_config)
+        connection.subscribe(channel) do |on|
+          on.message { |_channel, message| connection.unsubscribe if block.call(message) }
+        end
+      ensure
+        connection&.close
+      end
+
+      def publish(channel, message)
+        @redis.publish(channel, message)
+      end
+    end
+  end
+end

data/lib/ruby_reactor/storage/redis_step_results.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Storage
+    # The durable outcome of one `async_step`, keyed by (parent context, step
+    # name). It lives OUTSIDE the parent's serialized context blob on purpose: a
+    # worker writes it concurrently with the still-running parent, and two
+    # writers on one blob race.
+    #
+    # The `dispatched` record is written before the job is enqueued, so it also
+    # serves as the re-attach marker on recovery — "a record exists" is
+    # exactly the question "was this already dispatched?".
+    module RedisStepResults
+      def store_step_result(context_id, step_name, record, reactor_class_name)
+        key = step_result_key(context_id, step_name, reactor_class_name)
+        # Shares context_ttl with the parent: the record must not outlive what it
+        # belongs to, and must not expire before it either.
+        @redis.set(key, JSON.generate(record), ex: durability_ttl)
+      end
+
+      def retrieve_step_result(context_id, step_name, reactor_class_name)
+        json = @redis.get(step_result_key(context_id, step_name, reactor_class_name))
+        return nil unless json
+
+        JSON.parse(json)
+      end
+
+      # Every record, for StepSweeper — a unit whose job was lost leaves nothing
+      # else behind to find it by, since the parent only parks on the read side.
+      def scan_step_results(count: 1000)
+        records = []
+
+        @redis.scan_each(match: "reactor:*:context:*:step_result:*", count: 100) do |key|
+          json = @redis.get(key)
+          records << JSON.parse(json) if json
+          return records if records.size >= count
+        end
+
+        records
+      end
+
+      private
+
+      def step_result_key(context_id, step_name, reactor_class_name)
+        "reactor:#{reactor_class_name}:context:#{context_id}:step_result:#{step_name}"
+      end
+    end
+  end
+end

data/lib/ruby_reactor/sweeper.rb

@@ -17,6 +17,12 @@ module RubyReactor
   # race) and re-enqueued while its worker is actually alive, the duplicate hits
   # the live lock -> ContextLockContention -> uncapped snooze -> no double run.
   #
+  # `async_reactor` children ARE covered: each runs as its own job, so a lost
+  # job strands one exactly like a top-level reactor, and a fire-and-forget
+  # child has no parent left waiting to notice. They are identified by the
+  # `async_dispatched` marker rather than by absence of a parent, so compose
+  # children — same parent link, but run inline — are never re-enqueued.
+  #
   # Map fan-out (element/collector jobs) is NOT covered here — those contexts
   # carry parent_context_id and scan_reactors filters them out (F6). The map
   # sweeper (Phase 5) owns them.
@@ -41,7 +47,7 @@ module RubyReactor
     def run_once(limit: DEFAULT_LIMIT)
       reenqueued = 0
 
-      @storage.scan_reactors(count: limit).each do |reactor|
+      @storage.scan_reactors(count: limit, include_dispatched_children: true).each do |reactor|
         next unless reactor[:status] == "running" # non-terminal only
         next if @storage.lock_held?("async:#{reactor[:id]}") # worker alive -> leave alone
 

data/lib/ruby_reactor/sweeper_job.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module RubyReactor
+  # Self-rescheduling recovery tick, shared by every queueing backend's
+  # sweeper job class. Each run sweeps both the top-level reactor sweeper and
+  # the map sweeper, then schedules the next tick — a perpetual chain the
+  # host kicks once via `RubyReactor.start_sweeper!`.
+  #
+  # super_fetch safety. Sidekiq Enterprise `super_fetch` reliably re-runs a job
+  # whose worker died mid-execution. For a self-rescheduling chain that is a
+  # hazard: a tick can crash AFTER enqueuing its successor but BEFORE acking, so
+  # super_fetch recovers the crashed tick *alongside* the successor it already
+  # scheduled — the chain forks and then doubles every interval. We therefore do
+  # NOT rely on "exactly one job exists". The next tick is claimed by a
+  # per-time-window lock: every duplicate computes the SAME target window and
+  # only one wins the claim, so recovered/duplicated ticks collapse back to a
+  # single chain. The claim lock is never released — it simply expires — so no
+  # delete can race two duplicates into both winning.
+  module SweeperJob
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def perform
+      config = RubyReactor.configuration
+      return unless config.sweeper_enabled
+
+      run_sweeps(config)
+    ensure
+      # Always chain forward (unless disabled), even after an error above, so a
+      # single bad sweep can't kill recovery. The window lock keeps this from
+      # forking under super_fetch.
+      self.class.schedule_next if RubyReactor.configuration.sweeper_enabled
+    end
+
+    def run_sweeps(config)
+      RubyReactor::Sweeper.run_once(limit: config.sweeper_limit)
+      RubyReactor::Map::Sweeper.run_once(limit: config.sweeper_limit)
+      RubyReactor::StepSweeper.run_once(limit: config.sweeper_limit)
+    rescue StandardError => e
+      config.logger.error("RubyReactor sweeper sweep failed: #{e.class}: #{e.message}")
+    end
+
+    module ClassMethods
+      # Enqueue the next tick for the upcoming time window, claiming that window
+      # so concurrent/duplicate/recovered ticks produce exactly one successor.
+      # Idempotent: also safe to call from `start_sweeper!` on every process boot.
+      def schedule_next
+        interval = RubyReactor.configuration.sweeper_interval
+        window = (Time.now.to_i / interval) + 1
+
+        lock = RubyReactor::Lock.new(
+          "sweeper:window:#{window}",
+          owner: SecureRandom.uuid,
+          ttl: interval * 2, # outlive the window; expires on its own (never released)
+          wait: 0,
+          auto_extend: false
+        )
+        lock.acquire # raises AcquisitionError if this window is already claimed
+
+        delay = (window * interval) - Time.now.to_i
+        perform_in([delay, 1].max)
+      rescue RubyReactor::Lock::AcquisitionError
+        # Another tick already scheduled this window — collapse the duplicate.
+        nil
+      end
+    end
+  end
+end