geneva_drive 0.4.0 → 0.6.0

data/lib/geneva_drive/flow_control.rb

@@ -5,7 +5,7 @@
 #
 # @api private
 class GenevaDrive::FlowControlSignal
-  # @return [Symbol] the action to take (:cancel, :pause, :reattempt, :skip, :finished)
+  # @return [Symbol] the action to take (:cancel, :pause, :reattempt, :skip, :finished, :suspend)
   attr_reader :action
 
   # @return [Hash] additional options for the flow control action
@@ -33,6 +33,11 @@ class GenevaDrive::InvalidStateError < StandardError; end
 #   raise StepConfigurationError, "Step requires either a block or method name"
 class GenevaDrive::StepConfigurationError < StandardError; end
 
+# Raised when a resumable step cursor exceeds GenevaDrive.max_cursor_size
+# once serialized to JSON. The cursor is a position marker, not a payload -
+# it is rewritten on every checkpoint and copied to every successor execution.
+class GenevaDrive::CursorTooLargeError < StandardError; end
+
 # Base class for errors that occur during step execution.
 # These errors are raised after recovery actions have been performed,
 # so the workflow/step states are already updated when the exception propagates.
@@ -94,13 +99,19 @@ class GenevaDrive::PreconditionError < GenevaDrive::StepExecutionError; end
 #   workflow.pause!  # Pauses workflow from outside a step
 module GenevaDrive::FlowControl
   # Cancels the workflow immediately.
-  # The current step is marked as canceled and the workflow transitions to 'canceled' state.
+  #
+  # When called inside a step (workflow state is 'performing'): interrupts execution via throw/catch.
+  # When called outside a step (workflow state is 'ready' or 'paused'): directly cancels the workflow.
   #
   # @return [void]
-  # @raise [UncaughtThrowError] if called outside of step execution context
+  # @raise [InvalidStateError] if called on a non-ready/non-paused/non-performing workflow
   def cancel!
-    logger.info("Flow control: cancel! called from step")
-    throw :flow_control, GenevaDrive::FlowControlSignal.new(:cancel)
+    if state == "performing"
+      logger.info("Flow control: cancel! called from step")
+      throw :flow_control, GenevaDrive::FlowControlSignal.new(:cancel)
+    else
+      external_cancel!
+    end
   end
 
   # Pauses the workflow for manual intervention.
@@ -124,16 +135,24 @@ module GenevaDrive::FlowControl
   # Reschedules the current step for another attempt.
   # Useful for handling temporary failures or rate limiting.
   #
+  # For resumable steps, by default continues from the current cursor position.
+  # Use `rewind: true` to restart the step from the beginning.
+  #
   # @param wait [ActiveSupport::Duration, nil] optional delay before retry
+  # @param rewind [Boolean] if true, resets cursor to beginning (resumable steps only)
   # @return [void]
   # @raise [UncaughtThrowError] if called outside of step execution context
   #
   # @example Retry after rate limit
   #   reattempt!(wait: 5.minutes)
-  def reattempt!(wait: nil)
+  #
+  # @example Retry resumable step from the beginning
+  #   reattempt!(rewind: true)
+  def reattempt!(wait: nil, rewind: false)
     wait_msg = wait ? " with wait #{wait.inspect}" : ""
-    logger.info("Flow control: reattempt! called from step#{wait_msg}")
-    throw :flow_control, GenevaDrive::FlowControlSignal.new(:reattempt, wait: wait)
+    rewind_msg = rewind ? " (rewinding cursor)" : ""
+    logger.info("Flow control: reattempt! called from step#{wait_msg}#{rewind_msg}")
+    throw :flow_control, GenevaDrive::FlowControlSignal.new(:reattempt, wait: wait, rewind: rewind)
   end
 
   # Skips the current step and proceeds to the next one.
@@ -162,27 +181,79 @@ module GenevaDrive::FlowControl
     throw :flow_control, GenevaDrive::FlowControlSignal.new(:finished)
   end
 
+  # Suspends the current resumable step and re-enqueues for later continuation.
+  # The step will resume from its current cursor position.
+  #
+  # This is primarily for use within resumable steps when you need to explicitly
+  # yield control (e.g., to respect rate limits or system load).
+  #
+  # @param wait [ActiveSupport::Duration, nil] optional delay before re-enqueue
+  # @return [void]
+  # @raise [UncaughtThrowError] if called outside of step execution context
+  #
+  # @example Suspend for rate limiting
+  #   suspend!(wait: 5.minutes)
+  #
+  # @example Suspend to yield control
+  #   suspend!
+  def suspend!(wait: nil)
+    executing_step = current_step_name && steps.named(current_step_name)
+    unless executing_step&.resumable?
+      raise GenevaDrive::InvalidStateError,
+        "suspend! can only be called from inside a resumable_step (it continues from the persisted cursor)"
+    end
+
+    wait_msg = wait ? " with wait #{wait.inspect}" : ""
+    logger.info("Flow control: suspend! called from step#{wait_msg}")
+    throw :flow_control, GenevaDrive::FlowControlSignal.new(:suspend, wait: wait)
+  end
+
   private
 
   # Pauses the workflow from outside a step execution.
-  # Cancels any pending step execution and transitions workflow to paused.
+  # Leaves any scheduled step execution intact (does NOT cancel it).
+  #
+  # This allows seeing that a step was scheduled, became overdue during pause,
+  # and when it actually ran - providing better timeline visibility.
   #
   # @raise [InvalidStateError] if workflow is not in 'ready' state
   # @return [void]
   def external_pause!
     raise GenevaDrive::InvalidStateError, "Cannot pause a #{state} workflow" unless state == "ready"
 
-    logger.info("Flow control: pause! called externally on step #{current_step_name.inspect}")
+    logger.info("Pausing workflow (preserving scheduled execution for step #{next_step_name.inspect})")
     with_lock do
       # with_lock reloads automatically; re-check state in case it changed
       raise GenevaDrive::InvalidStateError, "Cannot pause a #{state} workflow" unless state == "ready"
 
-      current_execution&.mark_canceled!(outcome: "workflow_paused")
       update!(state: "paused", transitioned_at: Time.current)
     end
     logger.info("Workflow paused")
   end
 
+  # Cancels the workflow from outside a step execution.
+  # Cancels any scheduled step execution and transitions the workflow to 'canceled'.
+  #
+  # @raise [InvalidStateError] if workflow is not in 'ready' or 'paused' state
+  # @return [void]
+  def external_cancel!
+    unless %w[ready paused].include?(state)
+      raise GenevaDrive::InvalidStateError, "Cannot cancel a #{state} workflow"
+    end
+
+    logger.info("Canceling workflow externally")
+    with_lock do
+      # with_lock reloads automatically; re-check state in case it changed
+      unless %w[ready paused].include?(state)
+        raise GenevaDrive::InvalidStateError, "Cannot cancel a #{state} workflow"
+      end
+
+      current_execution&.mark_canceled!(outcome: "canceled")
+      update!(state: "canceled", transitioned_at: Time.current)
+    end
+    logger.info("Workflow canceled")
+  end
+
   # Skips the current step from outside a step execution.
   # Marks step as skipped and schedules next step (or finishes if last).
   #

data/lib/geneva_drive/iterable_step.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+# Object passed to resumable step blocks for cursor-based iteration.
+#
+# Provides an API compatible with Rails 8.1's ActiveJob::Continuation::Step,
+# plus GenevaDrive-specific extensions for iterating over records and batches.
+#
+# @example Manual cursor control
+#   resumable_step :process_records do |iter|
+#     hero.records.where("id > ?", iter.cursor || 0).find_each do |record|
+#       process(record)
+#       iter.set!(record.id)
+#     end
+#   end
+#
+# @example Using iterate_over_records helper
+#   resumable_step :process_records do |iter|
+#     iter.iterate_over_records(hero.records) do |record|
+#       process(record)
+#     end
+#   end
+#
+class GenevaDrive::IterableStep
+  attr_reader :name, :cursor
+
+  # Creates a new IterableStep for a resumable step execution.
+  #
+  # @param name [String, Symbol] the step name
+  # @param cursor [Object, nil] the current cursor value
+  # @param execution [GenevaDrive::StepExecution] the step execution record
+  # @param resumed [Boolean] whether this is a resumed execution
+  # @param interrupter [#should_interrupt?] object to check for interruption
+  def initialize(name, cursor, execution:, resumed:, interrupter:)
+    @name = name.to_sym
+    @cursor = cursor
+    @execution = execution
+    @resumed = resumed
+    @interrupter = interrupter
+    @advanced = false
+    @iterations = 0
+  end
+
+  # Returns the number of iterations completed during this execution
+  # (checkpoints via set!, advance! or checkpoint!).
+  #
+  # @return [Integer]
+  attr_reader :iterations
+
+  # === Rails 8.1 ActiveJob::Continuation::Step compatible API ===
+
+  # Returns true if this execution is resuming from a prior suspension.
+  #
+  # @return [Boolean]
+  def resumed?
+    @resumed
+  end
+
+  # Returns true if the cursor was changed during this execution.
+  #
+  # @return [Boolean]
+  def advanced?
+    @advanced
+  end
+
+  # Sets the cursor to a new value and checkpoints.
+  # The cursor can be any value that ActiveJob can serialize.
+  #
+  # @param value [Object] the new cursor value
+  # @return [void]
+  def set!(value)
+    @cursor = value
+    @advanced = true
+    checkpoint!
+  end
+
+  # Increments an integer cursor by 1 and checkpoints.
+  # Only works with integer cursors.
+  #
+  # @raise [ArgumentError] if cursor is not an Integer
+  # @return [void]
+  def advance!
+    raise ArgumentError, "advance! requires integer cursor, got #{@cursor.inspect}" unless @cursor.is_a?(Integer)
+    set!(@cursor + 1)
+  end
+
+  # Persists the current cursor, counts the iteration, and checks for
+  # interruption (max_iterations, max_runtime, shutdown, external pause/cancel).
+  # Called automatically by set! and advance!.
+  #
+  # @return [void]
+  def checkpoint!
+    @iterations += 1
+    persist_cursor!
+    check_interruption!
+  end
+
+  # Returns array representation [name, cursor] for compatibility.
+  #
+  # @return [Array]
+  def to_a
+    [name.to_s, cursor]
+  end
+
+  # Returns a human-readable description of current position.
+  #
+  # @return [String]
+  def description
+    "at '#{name}', cursor '#{cursor.inspect}'"
+  end
+
+  # === GenevaDrive Extensions ===
+
+  # Skips to a new cursor position and suspends execution.
+  # The job will be re-enqueued to continue from the new cursor.
+  #
+  # @param value [Object] the new cursor value
+  # @param wait [ActiveSupport::Duration, Numeric, nil] optional delay before re-enqueue
+  # @return [void] (never returns - throws :interrupt)
+  def skip_to!(value, wait: nil)
+    @cursor = value
+    @advanced = true
+    persist_cursor!
+    throw :interrupt, wait
+  end
+
+  # Iterates over an enumerable using index as cursor.
+  # Resumes from the stored cursor position on subsequent runs.
+  #
+  # @param enumerable [Enumerable] the collection to iterate over
+  # @yield [Object] each element in the collection
+  # @return [void]
+  def iterate_over(enumerable)
+    raise ArgumentError, "Must respond to #each" unless enumerable.respond_to?(:each)
+
+    start_index = @cursor || 0
+    enumerable.each_with_index do |element, index|
+      next if index < start_index
+      yield element
+      set!(index + 1)
+    end
+  end
+
+  # Iterates over an ActiveRecord relation using a column as cursor.
+  # Uses find_each for memory-efficient batching.
+  #
+  # @param relation [ActiveRecord::Relation] the relation to iterate over
+  # @param cursor [Symbol] the column to use as cursor (default: :id)
+  # @yield [ActiveRecord::Base] each record in the relation
+  # @return [void]
+  def iterate_over_records(relation, cursor: :id)
+    filtered = if @cursor
+      relation.where(relation.arel_table[cursor].gt(@cursor)).order(cursor => :asc)
+    else
+      relation.order(cursor => :asc)
+    end
+
+    filtered.find_each do |record|
+      yield record
+      set!(record.public_send(cursor))
+    end
+  end
+
+  # Iterates over an ActiveRecord relation in subrelations (batches).
+  # Each subrelation is yielded as an ActiveRecord relation with offset applied,
+  # useful for bulk operations like updates or exports.
+  #
+  # @param relation [ActiveRecord::Relation] the relation to iterate over
+  # @param batch_size [Integer] the number of records per subrelation
+  # @param cursor [Symbol] the column to use as cursor (default: :id)
+  # @yield [ActiveRecord::Relation] each subrelation with offset applied
+  # @return [void]
+  def iterate_over_subrelations(relation, batch_size:, cursor: :id)
+    filtered = if @cursor
+      relation.where(relation.arel_table[cursor].gt(@cursor)).order(cursor => :asc)
+    else
+      relation.order(cursor => :asc)
+    end
+
+    filtered.in_batches(of: batch_size) do |relation_with_offset|
+      yield relation_with_offset
+      last_cursor = relation_with_offset.maximum(cursor)
+      set!(last_cursor) if last_cursor
+    end
+  end
+
+  private
+
+  def persist_cursor!
+    serialized = GenevaDrive::StepExecution.serialize_cursor(@cursor)
+
+    GenevaDrive::StepExecution
+      .where(id: @execution.id)
+      .update_all(cursor: serialized)
+  end
+
+  def check_interruption!
+    throw :interrupt if @interrupter&.should_interrupt?(iterations: @iterations)
+  end
+end

data/lib/geneva_drive/job_options.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# Validation helper for Active Job option hashes passed through the workflow DSL
+# (`set_step_job_options`, `step(job_options: ...)`, and `workflow.step_job_options=`).
+#
+# Active Job itself silently ignores unknown keys in `set` — a typo like
+# `piority:` would never raise, and the workflow would just run at the default
+# priority. This helper enforces the known key set so misconfigurations surface
+# at definition time, or at assignment time for per-instance overrides.
+#
+# The allowed keys mirror what `ActiveJob::Core#set` actually consumes.
+#
+# @api private
+module GenevaDrive::JobOptions
+  # Keys that Active Job's `set` recognizes. See `ActiveJob::Core#set` in
+  # activejob/lib/active_job/core.rb.
+  ALLOWED_KEYS = %i[wait wait_until queue priority].freeze
+
+  # Validates a job options hash and returns a symbolized copy.
+  #
+  # @param options [#to_h, Hash, nil] the value to validate — anything that
+  #   coerces to a Hash via `#to_h` is accepted (Hash, keyword-args splat,
+  #   ActiveSupport::HashWithIndifferentAccess, structs, etc.)
+  # @param context [String] human-readable identifier used in error messages
+  #   (e.g. "Step 'foo'" or "MyWorkflow.step_job_options")
+  # @return [Hash{Symbol=>Object}] symbolized copy safe to pass to `.set`
+  # @raise [GenevaDrive::StepConfigurationError] if options can't be coerced to
+  #   a Hash, contains unknown keys, or has values of the wrong type
+  def self.validate!(options, context:)
+    hash = begin
+      options.to_h
+    rescue NoMethodError, TypeError, ArgumentError
+      raise GenevaDrive::StepConfigurationError,
+        "#{context} has invalid job_options: cannot coerce #{options.class} to a Hash via #to_h"
+    end
+
+    symbolized = hash.symbolize_keys
+
+    unknown = symbolized.keys - ALLOWED_KEYS
+    unless unknown.empty?
+      raise GenevaDrive::StepConfigurationError,
+        "#{context} has unknown job_options key(s): #{unknown.map(&:inspect).join(", ")}. " \
+        "Active Job only recognizes: #{ALLOWED_KEYS.map(&:inspect).join(", ")}"
+    end
+
+    symbolized.each { |k, v| validate_value!(k, v, context) }
+    symbolized
+  end
+
+  # @api private
+  def self.validate_value!(key, value, context)
+    return if value.nil?
+
+    valid = case key
+    when :wait
+      value.respond_to?(:to_i) && value.to_i >= 0
+    when :wait_until
+      value.respond_to?(:to_time) || value.respond_to?(:to_f)
+    when :queue
+      value.is_a?(String) || value.is_a?(Symbol)
+    when :priority
+      value.is_a?(Integer)
+    end
+
+    return if valid
+
+    raise GenevaDrive::StepConfigurationError,
+      "#{context} has invalid job_options[#{key.inspect}]: #{value.inspect} (#{value.class})"
+  end
+end