geneva_drive 0.4.0 → 0.6.0

data/lib/geneva_drive/executor.rb

@@ -25,27 +25,94 @@ class GenevaDrive::Executor
     "performing" => %w[ready performing canceled paused finished]
   }.freeze
 
+  # Result of a resumable step interrupting itself mid-iteration: the current
+  # execution completes with outcome "continued" and a successor execution is
+  # created to carry on from the persisted cursor. `wait` optionally delays
+  # the successor.
+  Interruption = Struct.new(:wait)
+
   # Executes a step execution with full flow control and exception handling.
+  # Handles both regular steps and resumable (cursor-iterating) steps.
   #
   # @param step_execution [GenevaDrive::StepExecution] the step to execute
+  # @param logger [Logger, nil] optional base logger to inject into the workflow.
+  #   If provided, this logger will be used as the base for all workflow logging,
+  #   with workflow and step-specific tags added on top. This allows callers
+  #   (background jobs, controllers, etc.) to pass in a logger that already
+  #   has appropriate context tags (e.g., job_id, request_id).
+  # @param interruptible [Boolean] whether resumable steps respect interruption
+  #   conditions (max_iterations, max_runtime, shutdown, external pause/cancel).
+  #   Test helpers pass false to run a resumable step to completion in one call.
+  # @param max_iterations [Integer, nil] overrides the step definition's
+  #   max_iterations for this execution (used by the run_iterations test helper)
   # @return [void]
-  def self.execute!(step_execution)
-    new.call(step_execution)
+  #
+  # @example Execute with a pre-tagged logger from a background job
+  #   logger = Rails.logger.tagged("job_id=#{job_id}")
+  #   GenevaDrive::Executor.execute!(step_execution, logger: logger)
+  def self.execute!(step_execution, logger: nil, interruptible: true, max_iterations: nil)
+    new.call(step_execution, logger: logger, interruptible: interruptible, max_iterations: max_iterations)
   end
 
   # Performs the step execution.
   #
   # @param step_execution [GenevaDrive::StepExecution] the step to execute
+  # @param logger [Logger, nil] optional base logger to inject into the workflow
+  # @param interruptible [Boolean] whether resumable steps respect interruption conditions
+  # @param max_iterations [Integer, nil] per-execution override of the step's max_iterations
   # @return [void]
-  def call(step_execution)
+  def call(step_execution, logger: nil, interruptible: true, max_iterations: nil)
     @step_execution = step_execution
     @workflow = step_execution.workflow
+    @interruptible = interruptible
+    @max_iterations_override = max_iterations
+
+    # Build the full logger chain (base -> workflow -> step tags) and inject
+    # it so step code calling `logger` gets the fully-tagged step execution
+    # logger. Falls back to Rails.logger if no logger is provided.
+    step_logger = build_step_logger(logger || Rails.logger, step_execution)
+    Rails.error.set_context(geneva_drive: error_context)
+
+    @workflow.with_logger(step_logger) do
+      step_execution.with_logger(step_logger) do
+        execute_with_logger(step_execution)
+      end
+    end
+  end
+
+  # Checks if a resumable step should be interrupted at a checkpoint.
+  # Called by IterableStep during checkpoint!.
+  #
+  # @param iterations [Integer, nil] iterations completed so far in this execution
+  # @return [Boolean]
+  def should_interrupt?(iterations: nil)
+    # If interruption is disabled (e.g., in tests), never interrupt
+    return false unless @interruptible
+
+    max_iterations = @max_iterations_override || (@step_definition.respond_to?(:max_iterations) ? @step_definition.max_iterations : nil)
+    return true if max_iterations && iterations && iterations >= max_iterations
+    return true if max_runtime_exceeded?
+    return true if job_should_exit?
+    return true if workflow_interrupted?
+    false
+  end
+
+  private
+
+  # Executes the step with the current logger configuration.
+  #
+  # @param step_execution [GenevaDrive::StepExecution] the step to execute
+  # @return [void]
+  def execute_with_logger(step_execution)
     @logger = step_execution.logger
     # Phase 1: Acquire locks, validate, and prepare for execution
     @logger.debug("Preparing execution context")
     step_def = prepare_execution
     return unless step_def
 
+    @step_definition = step_def
+    @start_time = Time.current
+
     # Phase 2: Execute step block (locks released)
     @logger.debug("Running before_step_execution hook")
     @workflow.before_step_execution(@step_execution)
@@ -63,22 +130,59 @@ class GenevaDrive::Executor
     finalize_execution(flow_result)
   end
 
-  private
-
   attr_reader :step_execution, :workflow, :logger
 
+  # Builds execution context to attach to Error Monitoring reports.
+  #
+  # @return [Hash]
+  def error_context
+    {
+      workflow_id: workflow.id,
+      workflow_class: workflow.class.name,
+      step_execution_id: step_execution.id,
+      step_name: step_execution.step_name,
+      hero_type: workflow.hero_type,
+      hero_id: workflow.hero_id
+    }.compact
+  end
+
+  # Builds a fully-tagged logger for the step execution.
+  # Adds workflow tags and step execution tags to the base logger.
+  #
+  # @param base_logger [Logger] the base logger (typically from the job)
+  # @param step_execution [GenevaDrive::StepExecution] the step execution
+  # @return [Logger] the fully-tagged logger
+  def build_step_logger(base_logger, step_execution)
+    workflow = step_execution.workflow
+
+    # Wrap with TaggedLogging and add workflow tags
+    tagged_logger = ActiveSupport::TaggedLogging.new(base_logger)
+    tag_parts = [workflow.class.name, " id=", workflow.to_param]
+    if workflow.hero_id.present?
+      tag_parts.concat([" hero_type=", workflow.hero_type, " hero_id=", workflow.hero_id])
+    end
+    workflow_tag = tag_parts.join
+    workflow_logger = tagged_logger.tagged(workflow_tag)
+
+    # Add step execution tags
+    workflow_logger.tagged("execution_id=#{step_execution.id} step_name=#{step_execution.step_name}")
+  end
+
   # Executes the step block with instrumentation.
   #
+  # For resumable steps, the block receives an IterableStep and runs inside a
+  # catch(:interrupt) so that checkpoints (and skip_to!) can interrupt the
+  # iteration. An interrupt is normalized into an {Interruption} result.
+  #
   # @param step_def [StepDefinition]
-  # @return [Symbol, FlowControlSignal, Hash] the execution result
+  # @return [Symbol, FlowControlSignal, Hash, Interruption] the execution result
   def execute_step(step_def)
     payload = instrumentation_payload
     payload[:step_name] = step_def.name
 
     ActiveSupport::Notifications.instrument("step.geneva_drive", payload) do |p|
       result = catch(:flow_control) do
-        step_def.execute_in_context(workflow)
-        :completed
+        run_step_code(step_def)
       rescue => e
         logger.error("Encountered #{e.class}, cleaning up and re-raising")
         # Don't transition here - just capture the error info
@@ -87,15 +191,47 @@ class GenevaDrive::Executor
 
       p[:outcome] = case result
       when :completed then :completed
+      when GenevaDrive::Executor::Interruption then :interrupted
       when GenevaDrive::FlowControlSignal then result.action
       when Hash then :exception
       end
       p[:exception] = result[:error] if result.is_a?(Hash)
+      p[:wait] = result.wait if result.is_a?(GenevaDrive::Executor::Interruption)
 
       result
     end
   end
 
+  # Runs the user's step code, handling the resumable interrupt channel.
+  #
+  # @param step_def [StepDefinition]
+  # @return [Symbol, Interruption] :completed, or an Interruption for resumable steps
+  def run_step_code(step_def)
+    unless step_def.resumable?
+      step_def.execute_in_context(workflow)
+      return :completed
+    end
+
+    iter = GenevaDrive::IterableStep.new(
+      step_def.name,
+      step_execution.cursor_value,
+      execution: step_execution,
+      resumed: step_execution.resuming?,
+      interrupter: self
+    )
+
+    # catch(:interrupt) returns:
+    # - :completed if the block finishes normally
+    # - nil if throw :interrupt (immediate successor)
+    # - Duration/Numeric if throw :interrupt, wait (delayed successor)
+    result = catch(:interrupt) do
+      step_def.execute_in_context(workflow, iter)
+      :completed
+    end
+
+    (result == :completed) ? :completed : Interruption.new(result)
+  end
+
   # Finalizes execution with instrumentation.
   #
   # @param flow_result [Symbol, FlowControlSignal, Hash]
@@ -117,11 +253,13 @@ class GenevaDrive::Executor
   #
   # @return [Hash]
   def instrumentation_payload
-    {
+    payload = {
       execution_id: step_execution.id,
       workflow_id: workflow.id,
       workflow_class: workflow.class.name
     }
+    payload[:resumable] = true if resumable_step?
+    payload
   end
 
   # Phase 1: Validates states and transitions to executing.
@@ -137,14 +275,14 @@ class GenevaDrive::Executor
     result = with_execution_lock do
       # Validate step execution can start
       unless step_execution.state == "scheduled"
-        logger.info("Step execution is in #{step_execution.state.inspect} and can't be stepped, dropping through")
+        logger.info("Step execution #{step_execution.id} is already #{step_execution.state.inspect}, skipping duplicate job")
 
         next nil
       end
 
       # Validate workflow is in a state that allows execution
       unless %w[ready performing].include?(workflow.state)
-        logger.info("Workflow is in #{workflow.state.inspect} state and can't be stepped, canceling and dropping through")
+        logger.info("Workflow #{workflow.id} is #{workflow.state.inspect}, cannot execute step — canceling step execution")
 
         step_execution.update!(
           state: "canceled",
@@ -157,7 +295,7 @@ class GenevaDrive::Executor
 
       # Check hero exists (unless workflow opts out)
       if workflow.hero.blank? && !workflow.class._may_proceed_without_hero
-        logger.info("No hero present and this workflow is not set to run without one. Canceling and dropping through")
+        logger.info("Workflow #{workflow.id} has no hero and requires one — canceling workflow")
         transition_step!("canceled", outcome: "canceled")
         transition_workflow!("canceled")
         next nil
@@ -182,6 +320,22 @@ class GenevaDrive::Executor
         next nil
       end
 
+      # Resumable steps need the cursor and continues_from_id columns. Fail
+      # loudly (instead of degrading) - without cursor persistence an
+      # interrupted iteration would silently restart from the beginning.
+      if step_def.resumable? && !GenevaDrive::StepExecution.resumable_columns?
+        error_message = "Step '#{step_execution.step_name}' is a resumable_step, but the cursor and " \
+          "continues_from_id columns are missing from geneva_drive_step_executions. " \
+          "Run `bin/rails generate geneva_drive:install` and migrate."
+        logger.error(error_message)
+
+        step_execution.update!(error_message: error_message)
+        transition_step!("failed", outcome: "failed")
+        transition_workflow!("paused")
+        exception_to_raise = GenevaDrive::StepConfigurationError.new(error_message)
+        next nil
+      end
+
       # Evaluate preconditions with instrumentation
       precondition_result = evaluate_preconditions(step_def)
       if precondition_result[:abort]
@@ -201,6 +355,8 @@ class GenevaDrive::Executor
       # completion, so that resume! on a failed step will retry it.
       workflow.update!(current_step_name: step_def.name)
 
+      logger.info("Resuming from cursor: #{step_execution.cursor_value.inspect}") if step_execution.resuming?
+
       step_def
     rescue => e
       exception_to_raise = handle_prepare_exception(e)
@@ -282,13 +438,23 @@ class GenevaDrive::Executor
         logger.warn(
           "Workflow #{workflow.id} state unexpectedly changed during execution: #{workflow.state}"
         )
-        transition_step!("canceled", outcome: "canceled")
+        if flow_result.is_a?(Interruption) && workflow.paused?
+          # A resumable step was interrupted because the workflow got paused
+          # externally mid-iteration. Preserve the cursor handoff: complete
+          # this execution with the "workflow_paused" marker so resume! can
+          # create a successor that continues from the cursor.
+          transition_step!("completed", outcome: "workflow_paused")
+        else
+          transition_step!("canceled", outcome: "canceled")
+        end
         next
       end
 
       case flow_result
       when :completed
         handle_completion
+      when Interruption
+        handle_continuation(wait: flow_result.wait)
       when GenevaDrive::FlowControlSignal
         handle_flow_control_signal(flow_result)
       when Hash
@@ -408,135 +574,80 @@ class GenevaDrive::Executor
     end
   end
 
+  # Default exception policy: pause the workflow. Used as the ultimate
+  # fallback at the bottom of every resolution stack.
+  DEFAULT_EXCEPTION_POLICY = GenevaDrive::ExceptionPolicy.new(:pause!).freeze
+
   # Captures exception info without performing transitions.
   #
   # @param error [Exception]
   # @param step_def [StepDefinition]
   # @return [Hash] captured exception context
   def capture_exception(error, step_def)
+    {type: :exception, error: error, step_def: step_def}
+  end
+
+  # Reports an exception via Rails.error.report based on the resolved policy's
+  # report setting. Called after policy resolution so that expected exceptions
+  # (e.g. rate limiting) can suppress error reporting.
+  #
+  # @param error [Exception]
+  # @param result [Hash] the policy result with :report and optional :terminal keys
+  # @return [void]
+  def report_exception(error, result)
+    case result[:report]
+    when :never
+      return
+    when :terminal_only
+      return unless result[:terminal]
+    end
     Rails.error.report(error)
-    {
-      type: :exception,
-      error: error,
-      step_def: step_def,
-      on_exception: step_def.on_exception
-    }
   end
 
   # Handles exceptions that occur during pre-condition evaluation (cancel_if, skip_if).
-  # Uses the step's on_exception policy to determine how to handle the exception.
-  # Returns the original exception to be re-raised after the transaction commits.
+  # Builds a resolution policy, applies it, and returns the original exception
+  # to be re-raised after the transaction commits.
   #
   # @param error [Exception] the exception that occurred
   # @param step_def [StepDefinition] the step definition
   # @return [Exception] the original exception to be re-raised
   def handle_precondition_exception(error, step_def)
     logger.error("Pre-condition evaluation failed: #{error.class} - #{error.message}")
-    Rails.error.report(error)
-
-    on_exception = step_def.on_exception
-    logger.info("Precondition exception handling with on_exception: #{on_exception.inspect}")
 
-    case on_exception
-    when :reattempt!
-      if reattempt_limit_exceeded?(step_def)
-        logger.warn("Max reattempts (#{step_def.max_reattempts}) exceeded - pausing workflow instead")
-        step_execution.update!(error_attributes_for(error))
-        transition_step!("failed", outcome: "failed")
-        transition_workflow!("paused")
-      else
-        logger.info("Precondition exception policy: reattempt! - rescheduling step")
-        transition_step!("completed", outcome: "reattempted")
-        transition_workflow!("ready")
-        workflow.reschedule_current_step!
-      end
-
-    when :cancel!
-      logger.info("Precondition exception policy: cancel! - canceling workflow")
-      step_execution.update!(error_attributes_for(error))
-      transition_step!("failed", outcome: "canceled")
-      transition_workflow!("canceled")
-
-    when :skip!
-      logger.info("Precondition exception policy: skip! - skipping to next step")
-      transition_step!("skipped", outcome: "skipped")
-      transition_workflow!("ready")
-      workflow.schedule_next_step!
+    result = apply_resolution_policy(error, step_def)
+    report_exception(error, result)
+    apply_policy_result(result, reattempt_reason: "precondition")
 
-    when :pause!
-      logger.info("Precondition exception policy: pause! - pausing workflow")
-      step_execution.update!(error_attributes_for(error))
-      transition_step!("failed", outcome: "failed")
-      transition_workflow!("paused")
-
-    else
-      # Default: pause
-      logger.info("Precondition exception policy: default (pause!) - pausing workflow")
-      step_execution.update!(error_attributes_for(error))
-      transition_step!("failed", outcome: "failed")
-      transition_workflow!("paused")
-    end
-
-    # Return original exception to be re-raised after transaction commits
     error
   end
 
   # Handles unexpected exceptions that occur during prepare_execution before
   # the step actually runs (e.g., NameError when hero_type references a
-  # non-existent class). Uses the step's on_exception policy if available,
-  # otherwise defaults to :pause!.
+  # non-existent class).
   # Returns the original exception to be re-raised after the transaction commits.
   #
   # @param error [Exception] the exception that occurred
   # @return [Exception] the original exception to be re-raised
   def handle_prepare_exception(error)
     logger.error("Unexpected exception during prepare_execution: #{error.class} - #{error.message}")
-    Rails.error.report(error)
 
-    # Try to get step_def for on_exception policy, but it may not be available yet
+    # Try to get step_def for policy resolution, but it may not be available yet
     step_def = begin
       step_execution.step_definition
     rescue
       nil
     end
-    on_exception = step_def&.on_exception || :pause!
-
-    logger.info("Prepare exception handling with on_exception: #{on_exception.inspect}")
 
     step_execution.update!(error_attributes_for(error))
     transition_step!("failed", outcome: "failed")
 
-    case on_exception
-    when :reattempt!
-      if step_def && reattempt_limit_exceeded?(step_def)
-        logger.warn("Max reattempts (#{step_def.max_reattempts}) exceeded - pausing workflow instead")
-        transition_workflow!("paused")
-      else
-        logger.info("Prepare exception policy: reattempt! - rescheduling step")
-        transition_workflow!("ready")
-        workflow.reschedule_current_step!
-      end
-
-    when :cancel!
-      logger.info("Prepare exception policy: cancel! - canceling workflow")
-      transition_workflow!("canceled")
-
-    when :skip!
-      logger.info("Prepare exception policy: skip! - skipping to next step")
-      transition_workflow!("ready")
-      workflow.schedule_next_step!
-
-    when :pause!
-      logger.info("Prepare exception policy: pause! - pausing workflow")
-      transition_workflow!("paused")
+    combined = build_resolution_policy(step_def)
+    count = consecutive_reattempt_count(step_def&.name || step_execution.step_name)
+    result = combined.apply(error, reattempt_count: count, workflow: workflow)
+    result ||= {action: :pause, error: error}
+    report_exception(error, result)
+    apply_prepare_policy_result(result)
 
-    else
-      # Default: pause
-      logger.info("Prepare exception policy: default (pause!) - pausing workflow")
-      transition_workflow!("paused")
-    end
-
-    # Return original exception to be re-raised after transaction commits
     error
   end
 
@@ -550,7 +661,21 @@ class GenevaDrive::Executor
     workflow.schedule_next_step!
   end
 
-  # Handles a captured exception based on the step's on_exception configuration.
+  # Handles a resumable step interrupting itself mid-iteration: completes the
+  # current execution with outcome "continued" and creates a successor
+  # execution that carries on from the persisted cursor.
+  #
+  # @param wait [ActiveSupport::Duration, Numeric, nil] optional delay before the successor runs
+  # @return [void]
+  def handle_continuation(wait: nil)
+    wait_msg = wait ? " (continuing after #{wait.inspect})" : ""
+    logger.info("Resumable step interrupted at cursor #{step_execution.cursor_value.inspect}#{wait_msg}")
+    transition_step!("completed", outcome: "continued")
+    transition_workflow!("ready")
+    workflow.create_successor_execution!(step_execution, wait: wait)
+  end
+
+  # Handles a captured exception based on the resolved exception policy.
   # Returns the original exception to be re-raised after the transaction commits.
   #
   # @param context [Hash] captured exception context
@@ -558,56 +683,157 @@ class GenevaDrive::Executor
   def handle_captured_exception(context)
     error = context[:error]
     step_def = context[:step_def]
-    on_exception = context[:on_exception]
 
-    logger.info("Handling exception with on_exception: #{on_exception.inspect}")
+    result = apply_resolution_policy(error, step_def)
+    report_exception(error, result)
+    apply_policy_result(result, reattempt_reason: "exception_policy")
 
-    case on_exception
-    when :reattempt!
-      if reattempt_limit_exceeded?(step_def)
-        logger.warn("Max reattempts (#{step_def.max_reattempts}) exceeded - pausing workflow instead")
-        step_execution.update!(error_attributes_for(error))
-        transition_step!("failed", outcome: "failed")
-        transition_workflow!("paused")
+    error
+  end
+
+  # Builds a combined exception policy from step-level, class-level, and
+  # default policies, then applies it to the error.
+  #
+  # @param error [Exception] the exception that was raised
+  # @param step_def [StepDefinition, nil] the step definition
+  # @return [Hash] policy result (see {ExceptionPolicy#apply})
+  def apply_resolution_policy(error, step_def)
+    combined = build_resolution_policy(step_def)
+    count = consecutive_reattempt_count(step_def&.name || step_execution.step_name)
+    combined.apply(error, reattempt_count: count, workflow: workflow)
+  end
+
+  # Builds a combined exception policy that includes step-level, class-level,
+  # and default policies in precedence order.
+  #
+  # The resulting {CombinedExceptionPolicy} walks policies in order:
+  # 1. Step-level policies (if explicitly set, in definition order)
+  # 2. Class-level specific policies (most recently defined first)
+  # 3. Class-level blanket policies (most recently defined first)
+  # 4. Default pause policy (always matches — guarantees a result)
+  #
+  # @param step_def [StepDefinition, nil] the step definition
+  # @return [GenevaDrive::CombinedExceptionPolicy]
+  def build_resolution_policy(step_def)
+    policies = []
+
+    policies << step_def.exception_policy if step_def&.exception_policy
+
+    class_reversed = workflow.class._exception_policies.reverse
+    policies.concat(class_reversed.select(&:specific?))
+    policies.concat(class_reversed.select(&:blanket?))
+
+    policies << DEFAULT_EXCEPTION_POLICY
+
+    GenevaDrive::CombinedExceptionPolicy.new(policies)
+  end
+
+  # Applies a policy result to the step execution and workflow.
+  # This is the single place where policy decisions become state transitions.
+  #
+  # @param result [Hash] policy result with :action, :wait, and :error keys
+  # @param reattempt_reason [String] reason for reattempt metadata
+  # @return [void]
+  def apply_policy_result(result, reattempt_reason:)
+    error = result[:error]
+
+    case result[:action]
+    when :reattempt
+      logger.info("Exception policy: reattempt!")
+      transition_step!("completed", outcome: "reattempted")
+      write_reattempt_metadata(reattempt_reason, error: error)
+      transition_workflow!("ready")
+      if resumable_step?
+        # Resumable steps continue from the persisted cursor via a successor
+        # execution instead of a fresh (cursor-less) execution.
+        workflow.create_successor_execution!(step_execution, wait: result[:wait])
       else
-        logger.info("Exception policy: reattempt! - rescheduling step")
-        transition_step!("completed", outcome: "reattempted")
-        transition_workflow!("ready")
-        workflow.reschedule_current_step!
+        workflow.reschedule_current_step!(wait: result[:wait])
       end
-
-    when :cancel!
-      logger.info("Exception policy: cancel! - canceling workflow")
+    when :cancel
+      logger.info("Exception policy: cancel!")
       step_execution.update!(error_attributes_for(error))
       transition_step!("failed", outcome: "canceled")
       transition_workflow!("canceled")
-
-    when :skip!
-      logger.info("Exception policy: skip! - skipping to next step")
+    when :skip
+      logger.info("Exception policy: skip!")
       transition_step!("skipped", outcome: "skipped")
       transition_workflow!("ready")
       workflow.schedule_next_step!
-
-    when :pause!
-      logger.info("Exception policy: pause! - pausing workflow")
+    when :pause
+      logger.info("Exception policy: pause!")
       step_execution.update!(error_attributes_for(error))
       transition_step!("failed", outcome: "failed")
       transition_workflow!("paused")
-
+    when :finished
+      logger.info("Exception policy: finished!")
+      transition_step!("completed", outcome: "success")
+      transition_workflow!("finished")
     else
-      # Default: pause
-      logger.info("Exception policy: default (pause!) - pausing workflow")
+      logger.info("Exception policy: default (pause!)")
       step_execution.update!(error_attributes_for(error))
       transition_step!("failed", outcome: "failed")
       transition_workflow!("paused")
     end
+  end
 
-    # Return original exception to be re-raised after transaction commits
-    error
+  # Applies a policy result during the prepare phase.
+  # In prepare phase, the step has already been transitioned to "failed",
+  # so only the workflow state transition is needed.
+  #
+  # @param result [Hash] policy result with :action and :wait keys
+  # @return [void]
+  def apply_prepare_policy_result(result)
+    case result[:action]
+    when :reattempt
+      logger.info("Prepare exception policy: reattempt!")
+      transition_workflow!("ready")
+      if resumable_step?
+        workflow.create_successor_execution!(step_execution, wait: result[:wait])
+      else
+        workflow.reschedule_current_step!(wait: result[:wait])
+      end
+    when :cancel
+      logger.info("Prepare exception policy: cancel!")
+      transition_workflow!("canceled")
+    when :skip
+      logger.info("Prepare exception policy: skip!")
+      transition_workflow!("ready")
+      workflow.schedule_next_step!
+    else
+      logger.info("Prepare exception policy: pause!")
+      transition_workflow!("paused")
+    end
+  end
+
+  # Writes reattempt reason (and exception info when present) to step
+  # execution metadata and persists.
+  #
+  # @param reason [String] the reattempt reason
+  # @param error [Exception, nil] the exception that triggered the reattempt
+  # @return [void]
+  def write_reattempt_metadata(reason, error: nil)
+    step_execution.write_metadata("reattempt_reason", reason)
+    write_exception_metadata(error) if error
+    step_execution.save! if step_execution.changed?
+  end
+
+  # Writes exception details into metadata for future migration away
+  # from dedicated error columns. Silent no-op without the column.
+  #
+  # @param error [Exception]
+  # @return [void]
+  def write_exception_metadata(error)
+    step_execution.write_metadata("exception", {
+      "class" => error.class.name,
+      "message" => error.message,
+      "backtrace" => error.backtrace
+    })
   end
 
   # Builds the attributes hash for storing error information on a step execution.
   # Conditionally includes error_class_name if the column exists (migration may not have run yet).
+  # Also double-writes the error into metadata for future migration away from dedicated columns.
   #
   # @param error [Exception]
   # @return [Hash]
@@ -617,14 +843,15 @@ class GenevaDrive::Executor
       error_backtrace: error.backtrace&.join("\n")
     }
     attrs[:error_class_name] = error.class.name if step_execution.has_attribute?(:error_class_name)
+    write_exception_metadata(error)
     attrs
   end
 
-  # Counts consecutive reattempts for the current step since the last successful execution.
-  # This is used to enforce the max_reattempts limit.
+  # Counts consecutive exception-policy reattempts for the current step.
+  # Only counts reattempts caused by exception policies (not flow-control reattempts).
   #
   # @param step_name [String] the step name to count reattempts for
-  # @return [Integer] the number of consecutive reattempts
+  # @return [Integer] the number of consecutive exception-policy reattempts
   def consecutive_reattempt_count(step_name)
     # Find the most recent non-reattempt completion for this step
     # (success, skipped, canceled, failed - anything that isn't a reattempt)
@@ -637,22 +864,12 @@ class GenevaDrive::Executor
     # Count reattempts after that point (or all reattempts if no prior success)
     scope = workflow.step_executions.where(step_name: step_name, outcome: "reattempted")
     scope = scope.where("id > ?", last_non_reattempt_id) if last_non_reattempt_id
-    scope.count
-  end
-
-  # Checks if the max reattempts limit has been exceeded for the current step.
-  # Returns true if we should fall back to :pause! instead of reattempting.
-  #
-  # @param step_def [StepDefinition] the step definition
-  # @return [Boolean] true if limit exceeded, false otherwise
-  def reattempt_limit_exceeded?(step_def)
-    max_reattempts = step_def.max_reattempts
-    return false if max_reattempts.nil? # Limit disabled
 
-    count = consecutive_reattempt_count(step_def.name)
-    count >= max_reattempts
+    GenevaDrive::StepExecution.count_error_reattempts(scope)
   end
 
+  # Checks if the max reattempts limit has been exceeded using policy-level max_reattempts.
+  #
   # Handles a flow control signal.
   #
   # @param signal [FlowControlSignal]
@@ -668,15 +885,30 @@ class GenevaDrive::Executor
 
     when :pause
       logger.info("Processing pause signal: pausing workflow")
-      transition_step!("canceled", outcome: "canceled")
+      if resumable_step?
+        # Complete with the "workflow_paused" marker so the cursor is
+        # preserved and resume! can continue from it via a successor.
+        transition_step!("completed", outcome: "workflow_paused")
+      else
+        transition_step!("canceled", outcome: "canceled")
+      end
       transition_workflow!("paused")
 
     when :reattempt
       wait_msg = signal.options[:wait] ? " after #{signal.options[:wait].inspect}" : ""
       logger.info("Processing reattempt signal: rescheduling step#{wait_msg}")
+      if resumable_step? && signal.options[:rewind]
+        logger.info("Rewinding cursor for reattempt")
+        step_execution.update!(cursor: nil)
+      end
       transition_step!("completed", outcome: "reattempted")
+      write_reattempt_metadata("flow_control")
       transition_workflow!("ready")
-      workflow.reschedule_current_step!(wait: signal.options[:wait])
+      if resumable_step?
+        workflow.create_successor_execution!(step_execution, wait: signal.options[:wait])
+      else
+        workflow.reschedule_current_step!(wait: signal.options[:wait])
+      end
 
     when :skip
       logger.info("Processing skip signal: scheduling next step")
@@ -688,7 +920,57 @@ class GenevaDrive::Executor
       logger.info("Processing finished signal: finishing workflow")
       transition_step!("completed", outcome: "success")
       transition_workflow!("finished")
+
+    when :suspend
+      # Only reachable from resumable steps: FlowControl#suspend! raises
+      # when called from a non-resumable step.
+      wait = signal.options[:wait]
+      logger.info("Processing suspend signal: completing and creating successor")
+      transition_step!("completed", outcome: "continued")
+      transition_workflow!("ready")
+      workflow.create_successor_execution!(step_execution, wait: wait)
+    end
+  end
+
+  # Whether the step being executed is a resumable (cursor-iterating) step.
+  # Falls back to looking up the step definition when called before
+  # prepare_execution has completed (e.g. from precondition handling).
+  #
+  # @return [Boolean]
+  def resumable_step?
+    step_def = @step_definition || begin
+      step_execution.step_definition
+    rescue
+      nil
     end
+    !!step_def&.resumable?
+  end
+
+  # Checks if max runtime has been exceeded for the current execution.
+  #
+  # @return [Boolean]
+  def max_runtime_exceeded?
+    max_runtime = @step_definition.respond_to?(:max_runtime) ? @step_definition.max_runtime : nil
+    return false unless max_runtime
+    return false unless @start_time
+    Time.current - @start_time > max_runtime
+  end
+
+  # Checks if the job should exit (e.g., Sidekiq shutdown).
+  #
+  # @return [Boolean]
+  def job_should_exit?
+    Thread.current[:geneva_drive_should_exit] ||
+      (defined?(Sidekiq) && Sidekiq.const_defined?(:CLI) &&
+       Sidekiq::CLI.instance&.stopping?)
+  end
+
+  # Checks if the workflow has been externally paused or canceled.
+  #
+  # @return [Boolean]
+  def workflow_interrupted?
+    workflow.reload
+    workflow.paused? || workflow.canceled?
   end
 end