ruby_reactor 0.6.0 → 0.7.0

data/lib/ruby_reactor/dsl/step_builder.rb

@@ -24,7 +24,6 @@ module RubyReactor
         @validate_args_input = nil
         @args_validator = nil
         @output_validator = nil
-        @async = false
         @retry_config = {}
       end
 
@@ -85,8 +84,22 @@ module RubyReactor
           end
       end
 
-      def async(async = true)
-        @async = async
+      # The per-step hand-off flag is gone. Only the FIRST flagged step
+      # in a reactor ever took effect — every later one was silently ignored —
+      # so this must fail at class-definition time rather than surprise someone
+      # at run time. Kept as a stub purely to say what to use instead.
+      def async(*)
+        raise RubyReactor::Error::DeprecatedDslError.new(
+          "`async` inside a `step` block has been removed: it was ambiguous (only the first " \
+          "flagged step in a reactor ever took effect). Replacements:\n  " \
+          "* `background after: :#{@name}` — hand every REMAINING step to a worker once " \
+          ":#{@name} finishes in the calling process (declared on the reactor, not the step);\n  " \
+          "* `background before: :#{@name}` — hand off starting WITH :#{@name};\n  " \
+          "* `async_step :#{@name}` — dispatch just this step's work to its own job while the " \
+          "reactor keeps running;\n  " \
+          "* `async_reactor :name, ChildReactor` — dispatch a whole nested reactor independently.",
+          step: @name
+        )
       end
 
       def retries(max_attempts: 3, backoff: :exponential, base_delay: 1)
@@ -97,8 +110,12 @@ module RubyReactor
         }
       end
 
-      def build
+      # `async_dispatch` marks a step whose work is dispatched as an independent
+      # unit rather than run inline — `:step` for `async_step`, `:reactor` for
+      # `async_reactor`. Nil for an ordinary step.
+      def build(async_dispatch: nil)
         step_config = {
+          async_dispatch: async_dispatch,
           name: @name,
           impl: @impl,
           arguments: @arguments,
@@ -110,7 +127,6 @@ module RubyReactor
           dependencies: @dependencies,
           args_validator: @args_validator || build_args_validator(@arg_validations, @validate_args_input),
           output_validator: @output_validator,
-          async: @async,
           retry_config: @retry_config.empty? ? (@reactor&.retry_defaults || {}) : @retry_config
         }
 
@@ -120,9 +136,10 @@ module RubyReactor
 
     class StepConfig
       attr_reader :name, :impl, :arguments, :run_block, :compensate_block, :undo_block, :conditions, :guards,
-                  :dependencies, :args_validator, :output_validator, :async, :retry_config
+                  :dependencies, :args_validator, :output_validator, :retry_config, :async_dispatch
 
       def initialize(config)
+        @async_dispatch = config[:async_dispatch]
         @name = config[:name]
         @impl = config[:impl]
         @arguments = config[:arguments] || {}
@@ -134,10 +151,16 @@ module RubyReactor
         @dependencies = config[:dependencies] || []
         @args_validator = config[:args_validator]
         @output_validator = config[:output_validator]
-        @async = config[:async] || false
         @retry_config = { max_attempts: 1 }.merge(config[:retry_config] || {})
       end
 
+      # True for `async_step` / `async_reactor` — the step's work leaves this
+      # process instead of running inline. `RSpec::TestSubject`'s `async: false`
+      # clears the marker to run the whole reactor in one process.
+      def async_dispatch?
+        !@async_dispatch.nil?
+      end
+
       def has_impl?
         !@impl.nil?
       end
@@ -146,10 +169,6 @@ module RubyReactor
         !@run_block.nil?
       end
 
-      def async?
-        @async
-      end
-
       def retryable?
         (retry_config[:max_attempts] || 0) > 1
       end

data/lib/ruby_reactor/error/async_result_pending.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Error
+    # Raised inside a WORKER when `result(:name)` references an `async_step` /
+    # `async_reactor` that is not yet terminal. Instead of blocking the worker
+    # thread for the whole wait (the sync-caller behavior), the executor parks:
+    # exclusive lock / semaphore stay HELD (recorded on the context), the job
+    # re-enqueues itself via the snooze path, and the wait resumes on
+    # redelivery. Bounded by `Configuration#async_park_timeout`, enforced at
+    # the wait site before this is raised.
+    class AsyncResultPending < Base
+      attr_reader :channel
+
+      def initialize(message, channel: nil)
+        super(message)
+        @channel = channel
+      end
+    end
+  end
+end

data/lib/ruby_reactor/error/async_wait_timeout_error.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Error
+    # Raised when a notified wait on an `async_step` / `async_reactor`
+    # result exceeds `Configuration#async_wait_timeout`. Never an unbounded wait.
+    class AsyncWaitTimeoutError < Base
+    end
+  end
+end

data/lib/ruby_reactor/error/deprecated_dsl_error.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  module Error
+    # Definition-time error for DSL that has been removed. Subclasses
+    # ValidationError so existing `rescue Error::ValidationError` sites keep
+    # catching it.
+    class DeprecatedDslError < ValidationError
+    end
+  end
+end

data/lib/ruby_reactor/executor/async_step_dispatch.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module RubyReactor
+  class Executor
+    # The dispatching half of `async_step`, mixed into StepExecutor. Split out
+    # because it is a self-contained concern — write the durable record and the
+    # context reference, enqueue, then mark the node graph-complete — and because
+    # it is where the structured logging for every hand-off lives.
+    module AsyncStepDispatch
+      private
+
+      # Send one step's work off as its own job and KEEP GOING.
+      #
+      # The ordering below is load-bearing (F2): the durable record and the
+      # context reference are written BEFORE the enqueue, so a crash in between
+      # can never leave a job with no record — and that same record is what
+      # tells a recovery pass the work is already out there, so it
+      # re-attaches instead of dispatching a duplicate side effect.
+      #
+      # Deliberately NOT gated on `inline_async_execution`: that flag stops the
+      # `background` hand-off from re-triggering inside a worker, but an
+      # `async_step` reached during a worker resume must still get its own job,
+      # or the feature silently degrades to inline execution exactly where the
+      # spec says it must not.
+      def dispatch_async_step(step_config)
+        if already_dispatched?(step_config)
+          @dependency_graph.complete_step(step_config.name)
+          return RubyReactor.Success(nil)
+        end
+
+        record_async_step_dispatch(step_config)
+        enqueue_async_step(step_config)
+
+        # Mark complete for SCHEDULING only — no result is recorded, so
+        # `result(:name)` still routes through the notified wait. This is what
+        # lets unrelated siblings become ready and run while the unit is in
+        # flight, instead of the loop returning early on an DispatchResult.
+        @dependency_graph.complete_step(step_config.name)
+        RubyReactor.Success(nil)
+      end
+
+      def already_dispatched?(step_config)
+        !storage.retrieve_step_result(@context.context_id, step_config.name, async_step_class_name).nil?
+      end
+
+      def record_async_step_dispatch(step_config)
+        root = @context.root_context || @context
+        @context.composed_contexts[step_config.name] = {
+          name: step_config.name,
+          type: :async_step_ref,
+          # Carried on the ref so the dashboard can find the Step Result Record
+          # from the reference alone, without re-deriving which context owns it.
+          context_id: @context.context_id,
+          dispatched_at: Time.now
+        }
+        storage.store_step_result(
+          @context.context_id, step_config.name,
+          {
+            "status" => "dispatched", "dispatched_at" => Time.now.iso8601,
+            # The re-dispatch arguments, verbatim. The record's own key names the
+            # reactor that OWNS the step, which for a composed child is not the
+            # root the worker must load, so recovery cannot re-derive them.
+            "root_context_id" => root.context_id,
+            "reactor_class_name" => RubyReactor.reactor_storage_name(root.reactor_class),
+            "step_context_id" => @context.context_id,
+            "step_name" => step_config.name.to_s
+          },
+          async_step_class_name
+        )
+
+        # The worker loads the parent by id, so the parent must be durable
+        # before the job exists AND must outlive the dispatched unit — including
+        # the fire-and-forget case where this reactor finishes immediately and
+        # nothing ever waits.
+        checkpoint_root!(root, RubyReactor.reactor_storage_name(root.reactor_class))
+      end
+
+      def enqueue_async_step(step_config)
+        root = @context.root_context || @context
+        log_async_event("async_step.dispatched", step_config.name)
+        configuration.async_router.perform_step_async(
+          root_context_id: root.context_id,
+          reactor_class_name: RubyReactor.reactor_storage_name(root.reactor_class),
+          step_context_id: @context.context_id,
+          step_name: step_config.name
+        )
+      end
+
+      # Step Result Records are namespaced by the reactor that OWNS the step,
+      # which for a composed child is the child — the same name the reader's
+      # `Template::Result` will look under.
+      def async_step_class_name
+        RubyReactor.reactor_storage_name(@context.reactor_class || @reactor_class)
+      end
+
+      def storage
+        RubyReactor::Configuration.instance.storage_adapter
+      end
+
+      # One machine-parseable line per hand-off / dispatch, carrying the
+      # three identifiers needed to correlate it with everything else.
+      def log_async_event(event, step_name)
+        configuration.logger.info(
+          "event=\"ruby_reactor.#{event}\" reactor=#{@reactor_class&.name.inspect} " \
+          "step=#{step_name.inspect} execution_id=#{@context.context_id.inspect}"
+        )
+      end
+    end
+  end
+end

data/lib/ruby_reactor/executor/ordered_lock_support.rb

@@ -69,7 +69,7 @@ module RubyReactor
       #
       # The strict-mode chain-skip only fires on a *fresh* start (no step
       # has run yet on this context). This lets an in-flight run that paused
-      # (Interrupt / AsyncResult) complete on resume regardless of chain
+      # (Interrupt / DispatchResult) complete on resume regardless of chain
       # failures that landed while it was parked, while still applying
       # strict to a fresh Sidekiq job (which enters via `resume_execution`
       # but has no prior step state).
@@ -296,7 +296,7 @@ module RubyReactor
 
       def terminal_for_ordered_lock?(result)
         case result
-        when RubyReactor::AsyncResult, RubyReactor::InterruptResult, RetryQueuedResult
+        when RubyReactor::DispatchResult, RubyReactor::InterruptResult, RetryQueuedResult
           false
         when RubyReactor::Success, RubyReactor::Failure
           true

data/lib/ruby_reactor/executor/result_handler.rb

@@ -97,11 +97,22 @@ module RubyReactor
       def handle_success(step_config, result, resolved_arguments)
         validate_step_output(step_config, result.value, resolved_arguments)
         @step_results[step_config.name] = result
-        @compensation_manager.add_to_undo_stack({ step: step_config, arguments: resolved_arguments, result: result })
+        # `async_step` / `async_reactor` dispatches are independent units of
+        # work with their own compensation flows — the parent rolling back must
+        # not "undo" a dispatch whose unit runs (and may still succeed)
+        # elsewhere. They never enter the parent's undo stack.
+        unless async_unit?(step_config)
+          @compensation_manager.add_to_undo_stack({ step: step_config, arguments: resolved_arguments,
+                                                    result: result })
+        end
         @context.set_result(step_config.name, result.value)
         @dependency_graph.complete_step(step_config.name)
       end
 
+      def async_unit?(step_config)
+        step_config.respond_to?(:async_dispatch?) && step_config.async_dispatch?
+      end
+
       def handle_retries_exhausted(step_config, result, resolved_arguments)
         @compensation_manager.handle_step_failure(step_config, result.original_error, resolved_arguments)
         orig_err = result.original_error.is_a?(Exception) ? result.original_error : nil

data/lib/ruby_reactor/executor/retry_manager.rb

@@ -99,7 +99,7 @@ module RubyReactor
           result
         when RubyReactor::Failure
           handle_failure_result(step_config, reactor_class, result)
-        when RetryQueuedResult, RubyReactor::AsyncResult
+        when RetryQueuedResult, RubyReactor::DispatchResult
           # Pass through async results
           result
         else
@@ -126,8 +126,12 @@ module RubyReactor
           @context
         )
 
-        # Check if we should requeue (async retry)
-        is_async = reactor_class.async? || step_config.async? ||
+        # Check if we should requeue (async retry). The per-step `async` flag is
+        # gone: a step relocated by `background` fails inside the worker, where
+        # `inline_async_execution` already answers this — and a step failing
+        # BEFORE the hand-off point genuinely has no worker to requeue into, so
+        # it must retry synchronously.
+        is_async = reactor_class.async? ||
                    @context.root_context&.reactor_class&.async? ||
                    @context.inline_async_execution
 
@@ -142,9 +146,9 @@ module RubyReactor
       def handle_async_retry(step_config, reactor_class, result)
         requeue_result = requeue_job_for_step_retry(step_config, result.error, reactor_class)
 
-        # If it returned an AsyncResult, we are truly async.
+        # If it returned an DispatchResult, we are truly async.
         # Otherwise, it ran inline and we should return the result of that execution.
-        if requeue_result.is_a?(RubyReactor::AsyncResult)
+        if requeue_result.is_a?(RubyReactor::DispatchResult)
           RetryQueuedResult.new(
             step_config.name,
             @context.retry_context.attempts_for_step(step_config.name),

data/lib/ruby_reactor/executor/step_executor.rb

@@ -3,6 +3,8 @@
 module RubyReactor
   class Executor
     class StepExecutor
+      include AsyncStepDispatch
+
       def initialize(context:, dependency_graph:, reactor_class:, managers:)
         @context = context
         @dependency_graph = dependency_graph
@@ -30,7 +32,7 @@ module RubyReactor
             result = execute_step(step_config)
 
             # If step execution was handed off to async, return the async result
-            return result if result.is_a?(RubyReactor::AsyncResult)
+            return result if result.is_a?(RubyReactor::DispatchResult)
 
             # If a step returns RetryQueuedResult, we need to stop and return it
             return result if result.is_a?(RetryQueuedResult)
@@ -62,25 +64,36 @@ module RubyReactor
       end
 
       def execute_step(step_config)
-        # If we're already in inline async execution mode (inside Worker),
-        # treat async steps as sync to avoid infinite recursion
-
         if @dependency_graph.completed.include?(step_config.name)
           return RubyReactor.Success(@context.get_result(step_config.name))
         end
 
-        resolved_arguments = resolve_arguments(step_config)
+        # Decided BEFORE argument resolution: resolving can block (or park) on
+        # an async `result(:name)`, and when the step body is about to be
+        # dispatched elsewhere — a `before:` hand-off, or an `async_step`'s
+        # own worker — that wait belongs to the process that will actually run
+        # it, not this one. (`async_reactor` still resolves here: its resolved
+        # values are the child's INPUTS, needed at dispatch.)
+        deferred_body = step_config.async_dispatch == :step || handoff_at?(step_config, :before)
+        resolved_arguments = deferred_body ? {} : resolve_arguments(step_config)
 
         @middlewares.on(:start_step, step_config.name, resolved_arguments, @context)
         completed = false
         begin
           result = if step_config.interrupt?
                      handle_interrupt_step(step_config)
-                   elsif step_config.async? && !@context.inline_async_execution
-                     handle_async_step(step_config)
+                   elsif step_config.async_dispatch == :step
+                     dispatch_async_step(step_config)
+                   elsif handoff_at?(step_config, :before)
+                     # `before: :x` hands off INSTEAD of running :x, leaving its
+                     # graph node incomplete for the worker to pick up.
+                     handle_background_handoff(step_config)
                    else
                      execute_step_with_retry(step_config, resolved_arguments)
                    end
+          # `after: :x` hands off once :x's result is recorded — the step really
+          # did run here, and only what remains moves to the worker.
+          result = handle_background_handoff(step_config) if handoff_after?(step_config, result)
           completed = true
           if result.is_a?(RubyReactor::Failure)
             @middlewares.on(:failed_step, step_config.name, result, @context)
@@ -96,6 +109,37 @@ module RubyReactor
 
       private
 
+      # The reactor's single hand-off point, `{ mode: :after|:before, step: }`.
+      # Nil for a reactor that never declares `background`.
+      def background_handoff
+        return @background_handoff if defined?(@background_handoff)
+
+        @background_handoff =
+          (@reactor_class.background_handoff if @reactor_class.respond_to?(:background_handoff))
+      end
+
+      # Hand-off is keyed to REACHING the named step, not to where the
+      # declaration sits in the class body. A step whose `where`/guard says it
+      # must not run never triggers it — the hand-off only ever relocates work
+      # that is actually going to happen. Inside the worker the whole thing is
+      # suppressed (`inline_async_execution`) so it cannot re-trigger.
+      def handoff_at?(step_config, mode)
+        point = background_handoff
+        return false unless point && point[:mode] == mode && point[:step] == step_config.name
+        return false if @context.inline_async_execution
+
+        step_config.should_run?(@context)
+      end
+
+      # Post-execution trigger for `after:`. Only a plain continue-Success means
+      # the named step actually completed here — a Failure, Skipped, interrupt,
+      # queued retry or an already-async result each own the flow instead.
+      def handoff_after?(step_config, result)
+        return false unless result.is_a?(RubyReactor::Success) && !result.is_a?(RubyReactor::Skipped)
+
+        handoff_at?(step_config, :after)
+      end
+
       def reconstruct_failure(data)
         return data if data.is_a?(RubyReactor::Failure)
         return nil unless data.is_a?(Hash)
@@ -125,7 +169,7 @@ module RubyReactor
           safe_execute_step_sync(step_config, resolved_arguments)
         end
 
-        unless result.is_a?(RetryQueuedResult) || result.is_a?(RubyReactor::AsyncResult)
+        unless result.is_a?(RetryQueuedResult) || result.is_a?(RubyReactor::DispatchResult)
           @result_handler.handle_step_result(step_config, result, resolved_arguments)
         end
 
@@ -197,9 +241,12 @@ module RubyReactor
         end
       end
 
-      def handle_async_step(step_config)
-        # Step-level async: hand off execution to worker
-
+      # Hand every step not yet executed to a worker job, and return the
+      # `DispatchResult` that halts `execute_all_steps` in the calling process.
+      # Shared verbatim by both `background` forms — they differ only in WHERE
+      # the trigger sits, never in what the hand-off does.
+      def handle_background_handoff(step_config)
+        log_async_event("background.handoff", step_config.name)
         @context.current_step = step_config.name
         @context.undo_stack = @compensation_manager.undo_stack