geneva_drive 0.5.0 → 0.7.0

data/lib/geneva_drive/executor.rb

@@ -15,7 +15,8 @@
 class GenevaDrive::Executor
   # Valid state transitions for step executions
   STEP_TRANSITIONS = {
-    "scheduled" => %w[scheduled in_progress canceled skipped failed completed],
+    "scheduled" => %w[scheduled waiting in_progress canceled skipped failed completed],
+    "waiting" => %w[scheduled canceled skipped],
     "in_progress" => %w[in_progress completed failed canceled skipped]
   }.freeze
 
@@ -25,7 +26,14 @@ class GenevaDrive::Executor
     "performing" => %w[ready performing canceled paused finished]
   }.freeze
 
+  # Result of a resumable step interrupting itself mid-iteration: the current
+  # execution completes with outcome "continued" and a successor execution is
+  # created to carry on from the persisted cursor. `wait` optionally delays
+  # the successor.
+  Interruption = Struct.new(:wait)
+
   # Executes a step execution with full flow control and exception handling.
+  # Handles both regular steps and resumable (cursor-iterating) steps.
   #
   # @param step_execution [GenevaDrive::StepExecution] the step to execute
   # @param logger [Logger, nil] optional base logger to inject into the workflow.
@@ -33,28 +41,39 @@ class GenevaDrive::Executor
   #   with workflow and step-specific tags added on top. This allows callers
   #   (background jobs, controllers, etc.) to pass in a logger that already
   #   has appropriate context tags (e.g., job_id, request_id).
+  # @param interruptible [Boolean] whether resumable steps respect interruption
+  #   conditions (max_iterations, max_runtime, shutdown, external pause/cancel).
+  #   Test helpers pass false to run a resumable step to completion in one call.
+  # @param max_iterations [Integer, nil] overrides the step definition's
+  #   max_iterations for this execution (used by the run_iterations test helper)
   # @return [void]
   #
   # @example Execute with a pre-tagged logger from a background job
   #   logger = Rails.logger.tagged("job_id=#{job_id}")
   #   GenevaDrive::Executor.execute!(step_execution, logger: logger)
-  def self.execute!(step_execution, logger: nil)
-    new.call(step_execution, logger: logger)
+  def self.execute!(step_execution, logger: nil, interruptible: true, max_iterations: nil)
+    new.call(step_execution, logger: logger, interruptible: interruptible, max_iterations: max_iterations)
   end
 
   # Performs the step execution.
   #
   # @param step_execution [GenevaDrive::StepExecution] the step to execute
   # @param logger [Logger, nil] optional base logger to inject into the workflow
+  # @param interruptible [Boolean] whether resumable steps respect interruption conditions
+  # @param max_iterations [Integer, nil] per-execution override of the step's max_iterations
   # @return [void]
-  def call(step_execution, logger: nil)
+  def call(step_execution, logger: nil, interruptible: true, max_iterations: nil)
     @step_execution = step_execution
     @workflow = step_execution.workflow
+    @interruptible = interruptible
+    @max_iterations_override = max_iterations
 
     # Build the full logger chain (base -> workflow -> step tags) and inject
     # it so step code calling `logger` gets the fully-tagged step execution
     # logger. Falls back to Rails.logger if no logger is provided.
     step_logger = build_step_logger(logger || Rails.logger, step_execution)
+    Rails.error.set_context(geneva_drive: error_context)
+
     @workflow.with_logger(step_logger) do
       step_execution.with_logger(step_logger) do
         execute_with_logger(step_execution)
@@ -62,6 +81,23 @@ class GenevaDrive::Executor
     end
   end
 
+  # Checks if a resumable step should be interrupted at a checkpoint.
+  # Called by IterableStep during checkpoint!.
+  #
+  # @param iterations [Integer, nil] iterations completed so far in this execution
+  # @return [Boolean]
+  def should_interrupt?(iterations: nil)
+    # If interruption is disabled (e.g., in tests), never interrupt
+    return false unless @interruptible
+
+    max_iterations = @max_iterations_override || (@step_definition.respond_to?(:max_iterations) ? @step_definition.max_iterations : nil)
+    return true if max_iterations && iterations && iterations >= max_iterations
+    return true if max_runtime_exceeded?
+    return true if job_should_exit?
+    return true if workflow_interrupted?
+    false
+  end
+
   private
 
   # Executes the step with the current logger configuration.
@@ -75,17 +111,26 @@ class GenevaDrive::Executor
     step_def = prepare_execution
     return unless step_def
 
-    # Phase 2: Execute step block (locks released)
-    @logger.debug("Running before_step_execution hook")
-    @workflow.before_step_execution(@step_execution)
+    @step_definition = step_def
+    @start_time = Time.current
 
-    @logger.debug("Running the actual step code")
-    flow_result = @workflow.around_step_execution(@step_execution) do
-      execute_step(step_def)
-    end
+    # Phase 2: Execute step block (locks released). The signal this execution
+    # is attached to (if any) is injected for the duration, so step code can
+    # read `received_signal`.
+    flow_result = @workflow.with_received_signal(@received_signal) do
+      @logger.debug("Running before_step_execution hook")
+      @workflow.before_step_execution(@step_execution)
 
-    @logger.debug("Running after_step_execution hook")
-    @workflow.after_step_execution(@step_execution)
+      @logger.debug("Running the actual step code")
+      result = @workflow.around_step_execution(@step_execution) do
+        execute_step(step_def)
+      end
+
+      @logger.debug("Running after_step_execution hook")
+      @workflow.after_step_execution(@step_execution)
+
+      result
+    end
 
     # Phase 3: Acquire locks and handle result
     @logger.info("Finished step with outcome #{flow_result.inspect}")
@@ -94,6 +139,20 @@ class GenevaDrive::Executor
 
   attr_reader :step_execution, :workflow, :logger
 
+  # Builds execution context to attach to Error Monitoring reports.
+  #
+  # @return [Hash]
+  def error_context
+    {
+      workflow_id: workflow.id,
+      workflow_class: workflow.class.name,
+      step_execution_id: step_execution.id,
+      step_name: step_execution.step_name,
+      hero_type: workflow.hero_type,
+      hero_id: workflow.hero_id
+    }.compact
+  end
+
   # Builds a fully-tagged logger for the step execution.
   # Adds workflow tags and step execution tags to the base logger.
   #
@@ -118,16 +177,19 @@ class GenevaDrive::Executor
 
   # Executes the step block with instrumentation.
   #
+  # For resumable steps, the block receives an IterableStep and runs inside a
+  # catch(:interrupt) so that checkpoints (and skip_to!) can interrupt the
+  # iteration. An interrupt is normalized into an {Interruption} result.
+  #
   # @param step_def [StepDefinition]
-  # @return [Symbol, FlowControlSignal, Hash] the execution result
+  # @return [Symbol, FlowControlSignal, Hash, Interruption] the execution result
   def execute_step(step_def)
     payload = instrumentation_payload
     payload[:step_name] = step_def.name
 
     ActiveSupport::Notifications.instrument("step.geneva_drive", payload) do |p|
       result = catch(:flow_control) do
-        step_def.execute_in_context(workflow)
-        :completed
+        run_step_code(step_def)
       rescue => e
         logger.error("Encountered #{e.class}, cleaning up and re-raising")
         # Don't transition here - just capture the error info
@@ -136,15 +198,47 @@ class GenevaDrive::Executor
 
       p[:outcome] = case result
       when :completed then :completed
+      when GenevaDrive::Executor::Interruption then :interrupted
       when GenevaDrive::FlowControlSignal then result.action
       when Hash then :exception
       end
       p[:exception] = result[:error] if result.is_a?(Hash)
+      p[:wait] = result.wait if result.is_a?(GenevaDrive::Executor::Interruption)
 
       result
     end
   end
 
+  # Runs the user's step code, handling the resumable interrupt channel.
+  #
+  # @param step_def [StepDefinition]
+  # @return [Symbol, Interruption] :completed, or an Interruption for resumable steps
+  def run_step_code(step_def)
+    unless step_def.resumable?
+      step_def.execute_in_context(workflow)
+      return :completed
+    end
+
+    iter = GenevaDrive::IterableStep.new(
+      step_def.name,
+      step_execution.cursor_value,
+      execution: step_execution,
+      resumed: step_execution.resuming?,
+      interrupter: self
+    )
+
+    # catch(:interrupt) returns:
+    # - :completed if the block finishes normally
+    # - nil if throw :interrupt (immediate successor)
+    # - Duration/Numeric if throw :interrupt, wait (delayed successor)
+    result = catch(:interrupt) do
+      step_def.execute_in_context(workflow, iter)
+      :completed
+    end
+
+    (result == :completed) ? :completed : Interruption.new(result)
+  end
+
   # Finalizes execution with instrumentation.
   #
   # @param flow_result [Symbol, FlowControlSignal, Hash]
@@ -166,11 +260,13 @@ class GenevaDrive::Executor
   #
   # @return [Hash]
   def instrumentation_payload
-    {
+    payload = {
       execution_id: step_execution.id,
       workflow_id: workflow.id,
       workflow_class: workflow.class.name
     }
+    payload[:resumable] = true if resumable_step?
+    payload
   end
 
   # Phase 1: Validates states and transitions to executing.
@@ -231,6 +327,38 @@ class GenevaDrive::Executor
         next nil
       end
 
+      # Resumable steps need the cursor and continues_from_id columns. Fail
+      # loudly (instead of degrading) - without cursor persistence an
+      # interrupted iteration would silently restart from the beginning.
+      if step_def.resumable? && !GenevaDrive::StepExecution.resumable_columns?
+        error_message = "Step '#{step_execution.step_name}' is a resumable_step, but the cursor and " \
+          "continues_from_id columns are missing from geneva_drive_step_executions. " \
+          "Run `bin/rails generate geneva_drive:install` and migrate."
+        logger.error(error_message)
+
+        step_execution.update!(error_message: error_message)
+        transition_step!("failed", outcome: "failed")
+        transition_workflow!("paused")
+        exception_to_raise = GenevaDrive::StepConfigurationError.new(error_message)
+        next nil
+      end
+
+      # Steps that wait for a signal need the signal_id and waiting_since
+      # columns. Fail loudly (instead of degrading) - without them a step
+      # would run without the event it was written to react to.
+      if step_def.waits_for_signal? && !GenevaDrive::StepExecution.signal_columns?
+        error_message = "Step '#{step_execution.step_name}' declares wait_for:, but the signal_id and " \
+          "waiting_since columns are missing from geneva_drive_step_executions. " \
+          "Run `bin/rails generate geneva_drive:install` and migrate."
+        logger.error(error_message)
+
+        step_execution.update!(error_message: error_message)
+        transition_step!("failed", outcome: "failed")
+        transition_workflow!("paused")
+        exception_to_raise = GenevaDrive::StepConfigurationError.new(error_message)
+        next nil
+      end
+
       # Evaluate preconditions with instrumentation
       precondition_result = evaluate_preconditions(step_def)
       if precondition_result[:abort]
@@ -238,6 +366,13 @@ class GenevaDrive::Executor
         next nil
       end
 
+      # The rendezvous, receiver side: skip_if has had its say (skip beats
+      # wait), so now find the signal or park. Parking ends the job without
+      # enqueueing anything - dispatch will wake us.
+      if step_def.waits_for_signal? && run_signal_gate(step_def) == :parked
+        next nil
+      end
+
       # All checks passed - transition to in_progress
       logger.debug("Step may be performed - changing state flags and proceeding to perform")
 
@@ -250,6 +385,8 @@ class GenevaDrive::Executor
       # completion, so that resume! on a failed step will retry it.
       workflow.update!(current_step_name: step_def.name)
 
+      logger.info("Resuming from cursor: #{step_execution.cursor_value.inspect}") if step_execution.resuming?
+
       step_def
     rescue => e
       exception_to_raise = handle_prepare_exception(e)
@@ -260,6 +397,69 @@ class GenevaDrive::Executor
     result
   end
 
+  # The receiver side of the rendezvous. Runs under the workflow and step
+  # execution locks, inside prepare_execution.
+  #
+  # An execution that already carries an attachment pin (dispatch woke it, or
+  # it inherited the pin from its predecessor) runs straight away. Otherwise
+  # the oldest matching non-consumed signal is attached, and failing that the
+  # execution parks.
+  #
+  # @param step_def [StepDefinition] the waiting step's definition
+  # @return [Symbol] :attached or :parked
+  def run_signal_gate(step_def)
+    if step_execution.signal_id.present?
+      @received_signal = step_execution.signal
+      logger.info("Step is attached to signal #{step_execution.signal_id}, proceeding")
+      return :attached
+    end
+
+    matcher = step_def.signal_matcher
+    candidate = workflow.attachable_signals.detect { |signal| matcher.matches?(signal) }
+
+    unless candidate
+      logger.info("No signal matching #{matcher} has arrived, parking step execution")
+      transition_step!("waiting")
+      step_execution.update!(waiting_since: Time.current)
+      return :parked
+    end
+
+    logger.info("Attaching signal #{candidate.id} (#{candidate.name}) to step execution")
+    candidate.claim!
+    step_execution.update!(signal_id: candidate.id)
+    @received_signal = candidate
+    :attached
+  end
+
+  # Books one clean resolution against the attached signal. Called from
+  # finalization on clean outcomes only (completion, skip, finished!), right
+  # after the step's own terminal transition and in the same transaction: a
+  # crash before commit resolves nothing and completes nothing, so replay
+  # stays coherent.
+  #
+  # The signal only flips to consumed once every chain attached to it has
+  # resolved - for a single claimant that is this very moment, and when one
+  # signal has been dispatched onto several executions it is whichever of
+  # them finishes last. Reattempts, failures and cancellations deliberately
+  # leave the signal claimed, so the retry (or resume) re-attaches to it and
+  # sees the same payload while the audit trail reads truthfully.
+  #
+  # @return [void]
+  def resolve_attached_signal!
+    return unless GenevaDrive::StepExecution.signal_columns?
+    return if step_execution.signal_id.blank?
+
+    signal = step_execution.signal
+    return unless signal
+
+    signal.record_consumption!
+    if signal.state == "consumed"
+      logger.info("Consumed signal #{signal.id} (#{signal.name}) after #{signal.consumed} clean resolution(s)")
+    else
+      logger.info("Signal #{signal.id} (#{signal.name}) stays claimed - other attached executions are unresolved")
+    end
+  end
+
   # Evaluates preconditions (cancel_if and skip_if) with instrumentation.
   #
   # @param step_def [StepDefinition]
@@ -292,6 +492,11 @@ class GenevaDrive::Executor
         if step_def.should_skip?(workflow)
           logger.info("skip_if condition matched, skipping step")
           transition_step!("skipped", outcome: "skipped")
+          # The step may have been woken by a signal and only then decided it
+          # had nothing to do (or a retry re-attached before skip_if flipped).
+          # Skipping settles that event instead of leaving it for the next
+          # waiter on the same name.
+          workflow.settle_signal_for_skipped!(step_execution)
           transition_workflow!("ready")
           workflow.schedule_next_step!
           p[:outcome] = :skipped
@@ -331,13 +536,23 @@ class GenevaDrive::Executor
         logger.warn(
           "Workflow #{workflow.id} state unexpectedly changed during execution: #{workflow.state}"
         )
-        transition_step!("canceled", outcome: "canceled")
+        if flow_result.is_a?(Interruption) && workflow.paused?
+          # A resumable step was interrupted because the workflow got paused
+          # externally mid-iteration. Preserve the cursor handoff: complete
+          # this execution with the "workflow_paused" marker so resume! can
+          # create a successor that continues from the cursor.
+          transition_step!("completed", outcome: "workflow_paused")
+        else
+          transition_step!("canceled", outcome: "canceled")
+        end
         next
       end
 
       case flow_result
       when :completed
         handle_completion
+      when Interruption
+        handle_continuation(wait: flow_result.wait)
       when GenevaDrive::FlowControlSignal
         handle_flow_control_signal(flow_result)
       when Hash
@@ -467,10 +682,26 @@ class GenevaDrive::Executor
   # @param step_def [StepDefinition]
   # @return [Hash] captured exception context
   def capture_exception(error, step_def)
-    Rails.error.report(error)
     {type: :exception, error: error, step_def: step_def}
   end
 
+  # Reports an exception via Rails.error.report based on the resolved policy's
+  # report setting. Called after policy resolution so that expected exceptions
+  # (e.g. rate limiting) can suppress error reporting.
+  #
+  # @param error [Exception]
+  # @param result [Hash] the policy result with :report and optional :terminal keys
+  # @return [void]
+  def report_exception(error, result)
+    case result[:report]
+    when :never
+      return
+    when :terminal_only
+      return unless result[:terminal]
+    end
+    Rails.error.report(error)
+  end
+
   # Handles exceptions that occur during pre-condition evaluation (cancel_if, skip_if).
   # Builds a resolution policy, applies it, and returns the original exception
   # to be re-raised after the transaction commits.
@@ -480,9 +711,9 @@ class GenevaDrive::Executor
   # @return [Exception] the original exception to be re-raised
   def handle_precondition_exception(error, step_def)
     logger.error("Pre-condition evaluation failed: #{error.class} - #{error.message}")
-    Rails.error.report(error)
 
     result = apply_resolution_policy(error, step_def)
+    report_exception(error, result)
     apply_policy_result(result, reattempt_reason: "precondition")
 
     error
@@ -497,7 +728,6 @@ class GenevaDrive::Executor
   # @return [Exception] the original exception to be re-raised
   def handle_prepare_exception(error)
     logger.error("Unexpected exception during prepare_execution: #{error.class} - #{error.message}")
-    Rails.error.report(error)
 
     # Try to get step_def for policy resolution, but it may not be available yet
     step_def = begin
@@ -512,7 +742,9 @@ class GenevaDrive::Executor
     combined = build_resolution_policy(step_def)
     count = consecutive_reattempt_count(step_def&.name || step_execution.step_name)
     result = combined.apply(error, reattempt_count: count, workflow: workflow)
-    apply_prepare_policy_result(result || {action: :pause, error: error})
+    result ||= {action: :pause, error: error}
+    report_exception(error, result)
+    apply_prepare_policy_result(result)
 
     error
   end
@@ -523,10 +755,25 @@ class GenevaDrive::Executor
   def handle_completion
     logger.info("Step completed successfully, scheduling next step")
     transition_step!("completed", outcome: "success")
+    resolve_attached_signal!
     transition_workflow!("ready")
     workflow.schedule_next_step!
   end
 
+  # Handles a resumable step interrupting itself mid-iteration: completes the
+  # current execution with outcome "continued" and creates a successor
+  # execution that carries on from the persisted cursor.
+  #
+  # @param wait [ActiveSupport::Duration, Numeric, nil] optional delay before the successor runs
+  # @return [void]
+  def handle_continuation(wait: nil)
+    wait_msg = wait ? " (continuing after #{wait.inspect})" : ""
+    logger.info("Resumable step interrupted at cursor #{step_execution.cursor_value.inspect}#{wait_msg}")
+    transition_step!("completed", outcome: "continued")
+    transition_workflow!("ready")
+    workflow.create_successor_execution!(step_execution, wait: wait)
+  end
+
   # Handles a captured exception based on the resolved exception policy.
   # Returns the original exception to be re-raised after the transaction commits.
   #
@@ -537,6 +784,7 @@ class GenevaDrive::Executor
     step_def = context[:step_def]
 
     result = apply_resolution_policy(error, step_def)
+    report_exception(error, result)
     apply_policy_result(result, reattempt_reason: "exception_policy")
 
     error
@@ -594,7 +842,13 @@ class GenevaDrive::Executor
       transition_step!("completed", outcome: "reattempted")
       write_reattempt_metadata(reattempt_reason, error: error)
       transition_workflow!("ready")
-      workflow.reschedule_current_step!(wait: result[:wait])
+      if resumable_step?
+        # Resumable steps continue from the persisted cursor via a successor
+        # execution instead of a fresh (cursor-less) execution.
+        workflow.create_successor_execution!(step_execution, wait: result[:wait])
+      else
+        workflow.reschedule_current_step!(wait: result[:wait])
+      end
     when :cancel
       logger.info("Exception policy: cancel!")
       step_execution.update!(error_attributes_for(error))
@@ -603,6 +857,7 @@ class GenevaDrive::Executor
     when :skip
       logger.info("Exception policy: skip!")
       transition_step!("skipped", outcome: "skipped")
+      resolve_attached_signal!
       transition_workflow!("ready")
       workflow.schedule_next_step!
     when :pause
@@ -613,6 +868,7 @@ class GenevaDrive::Executor
     when :finished
       logger.info("Exception policy: finished!")
       transition_step!("completed", outcome: "success")
+      resolve_attached_signal!
       transition_workflow!("finished")
     else
       logger.info("Exception policy: default (pause!)")
@@ -633,7 +889,11 @@ class GenevaDrive::Executor
     when :reattempt
       logger.info("Prepare exception policy: reattempt!")
       transition_workflow!("ready")
-      workflow.reschedule_current_step!(wait: result[:wait])
+      if resumable_step?
+        workflow.create_successor_execution!(step_execution, wait: result[:wait])
+      else
+        workflow.reschedule_current_step!(wait: result[:wait])
+      end
     when :cancel
       logger.info("Prepare exception policy: cancel!")
       transition_workflow!("canceled")
@@ -726,28 +986,94 @@ class GenevaDrive::Executor
 
     when :pause
       logger.info("Processing pause signal: pausing workflow")
-      transition_step!("canceled", outcome: "canceled")
+      if resumable_step?
+        # Complete with the "workflow_paused" marker so the cursor is
+        # preserved and resume! can continue from it via a successor.
+        transition_step!("completed", outcome: "workflow_paused")
+      else
+        transition_step!("canceled", outcome: "canceled")
+      end
       transition_workflow!("paused")
 
     when :reattempt
       wait_msg = signal.options[:wait] ? " after #{signal.options[:wait].inspect}" : ""
       logger.info("Processing reattempt signal: rescheduling step#{wait_msg}")
+      if resumable_step? && signal.options[:rewind]
+        logger.info("Rewinding cursor for reattempt")
+        step_execution.update!(cursor: nil)
+      end
       transition_step!("completed", outcome: "reattempted")
       write_reattempt_metadata("flow_control")
       transition_workflow!("ready")
-      workflow.reschedule_current_step!(wait: signal.options[:wait])
+      if resumable_step?
+        workflow.create_successor_execution!(step_execution, wait: signal.options[:wait])
+      else
+        workflow.reschedule_current_step!(wait: signal.options[:wait])
+      end
 
     when :skip
       logger.info("Processing skip signal: scheduling next step")
       transition_step!("skipped", outcome: "skipped")
+      resolve_attached_signal!
       transition_workflow!("ready")
       workflow.schedule_next_step!
 
     when :finished
       logger.info("Processing finished signal: finishing workflow")
       transition_step!("completed", outcome: "success")
+      resolve_attached_signal!
       transition_workflow!("finished")
+
+    when :suspend
+      # Only reachable from resumable steps: FlowControl#suspend! raises
+      # when called from a non-resumable step.
+      wait = signal.options[:wait]
+      logger.info("Processing suspend signal: completing and creating successor")
+      transition_step!("completed", outcome: "continued")
+      transition_workflow!("ready")
+      workflow.create_successor_execution!(step_execution, wait: wait)
+    end
+  end
+
+  # Whether the step being executed is a resumable (cursor-iterating) step.
+  # Falls back to looking up the step definition when called before
+  # prepare_execution has completed (e.g. from precondition handling).
+  #
+  # @return [Boolean]
+  def resumable_step?
+    step_def = @step_definition || begin
+      step_execution.step_definition
+    rescue
+      nil
     end
+    !!step_def&.resumable?
+  end
+
+  # Checks if max runtime has been exceeded for the current execution.
+  #
+  # @return [Boolean]
+  def max_runtime_exceeded?
+    max_runtime = @step_definition.respond_to?(:max_runtime) ? @step_definition.max_runtime : nil
+    return false unless max_runtime
+    return false unless @start_time
+    Time.current - @start_time > max_runtime
+  end
+
+  # Checks if the job should exit (e.g., Sidekiq shutdown).
+  #
+  # @return [Boolean]
+  def job_should_exit?
+    Thread.current[:geneva_drive_should_exit] ||
+      (defined?(Sidekiq) && Sidekiq.const_defined?(:CLI) &&
+       Sidekiq::CLI.instance&.stopping?)
+  end
+
+  # Checks if the workflow has been externally paused or canceled.
+  #
+  # @return [Boolean]
+  def workflow_interrupted?
+    workflow.reload
+    workflow.paused? || workflow.canceled?
   end
 end