geneva_drive 0.5.0 → 0.7.0

data/lib/geneva_drive/flow_control.rb

@@ -5,7 +5,7 @@
 #
 # @api private
 class GenevaDrive::FlowControlSignal
-  # @return [Symbol] the action to take (:cancel, :pause, :reattempt, :skip, :finished)
+  # @return [Symbol] the action to take (:cancel, :pause, :reattempt, :skip, :finished, :suspend)
   attr_reader :action
 
   # @return [Hash] additional options for the flow control action
@@ -33,6 +33,22 @@ class GenevaDrive::InvalidStateError < StandardError; end
 #   raise StepConfigurationError, "Step requires either a block or method name"
 class GenevaDrive::StepConfigurationError < StandardError; end
 
+# Raised when a resumable step cursor exceeds GenevaDrive.max_cursor_size
+# once serialized to JSON. The cursor is a position marker, not a payload -
+# it is rewritten on every checkpoint and copied to every successor execution.
+class GenevaDrive::CursorTooLargeError < StandardError; end
+
+# Raised when a signal payload exceeds GenevaDrive.max_signal_payload_size
+# once serialized to JSON. A payload describes the event; the data the step
+# works on belongs on the hero.
+class GenevaDrive::SignalPayloadTooLargeError < StandardError; end
+
+# Raised when a signal is delivered to a workflow that has already finished
+# or been canceled. Delivering the very same event twice (same idempotency
+# key) is a no-op rather than an error, so callers who want lenience can
+# rescue just this one class.
+class GenevaDrive::WorkflowNotOngoing < StandardError; end
+
 # Base class for errors that occur during step execution.
 # These errors are raised after recovery actions have been performed,
 # so the workflow/step states are already updated when the exception propagates.
@@ -130,16 +146,24 @@ module GenevaDrive::FlowControl
   # Reschedules the current step for another attempt.
   # Useful for handling temporary failures or rate limiting.
   #
+  # For resumable steps, by default continues from the current cursor position.
+  # Use `rewind: true` to restart the step from the beginning.
+  #
   # @param wait [ActiveSupport::Duration, nil] optional delay before retry
+  # @param rewind [Boolean] if true, resets cursor to beginning (resumable steps only)
   # @return [void]
   # @raise [UncaughtThrowError] if called outside of step execution context
   #
   # @example Retry after rate limit
   #   reattempt!(wait: 5.minutes)
-  def reattempt!(wait: nil)
+  #
+  # @example Retry resumable step from the beginning
+  #   reattempt!(rewind: true)
+  def reattempt!(wait: nil, rewind: false)
     wait_msg = wait ? " with wait #{wait.inspect}" : ""
-    logger.info("Flow control: reattempt! called from step#{wait_msg}")
-    throw :flow_control, GenevaDrive::FlowControlSignal.new(:reattempt, wait: wait)
+    rewind_msg = rewind ? " (rewinding cursor)" : ""
+    logger.info("Flow control: reattempt! called from step#{wait_msg}#{rewind_msg}")
+    throw :flow_control, GenevaDrive::FlowControlSignal.new(:reattempt, wait: wait, rewind: rewind)
   end
 
   # Skips the current step and proceeds to the next one.
@@ -168,6 +192,33 @@ module GenevaDrive::FlowControl
     throw :flow_control, GenevaDrive::FlowControlSignal.new(:finished)
   end
 
+  # Suspends the current resumable step and re-enqueues for later continuation.
+  # The step will resume from its current cursor position.
+  #
+  # This is primarily for use within resumable steps when you need to explicitly
+  # yield control (e.g., to respect rate limits or system load).
+  #
+  # @param wait [ActiveSupport::Duration, nil] optional delay before re-enqueue
+  # @return [void]
+  # @raise [UncaughtThrowError] if called outside of step execution context
+  #
+  # @example Suspend for rate limiting
+  #   suspend!(wait: 5.minutes)
+  #
+  # @example Suspend to yield control
+  #   suspend!
+  def suspend!(wait: nil)
+    executing_step = current_step_name && steps.named(current_step_name)
+    unless executing_step&.resumable?
+      raise GenevaDrive::InvalidStateError,
+        "suspend! can only be called from inside a resumable_step (it continues from the persisted cursor)"
+    end
+
+    wait_msg = wait ? " with wait #{wait.inspect}" : ""
+    logger.info("Flow control: suspend! called from step#{wait_msg}")
+    throw :flow_control, GenevaDrive::FlowControlSignal.new(:suspend, wait: wait)
+  end
+
   private
 
   # Pauses the workflow from outside a step execution.
@@ -236,14 +287,43 @@ module GenevaDrive::FlowControl
       end
 
       if state == "paused"
-        # Workflow was paused (e.g., due to failed step). Resume and skip to next step.
+        # Workflow was paused (e.g., due to failed step). Resume and skip to
+        # next step. The step being skipped past may hold a signal claim on a
+        # failed execution - no further execution of it is coming, so the
+        # operator's skip is what settles that event.
+        holder = execution_holding_signal_claim(next_step_name)
         update!(state: "ready", transitioned_at: nil)
+        settle_signal_for_skipped!(holder)
       else
-        # Workflow is ready with a scheduled step - mark it as skipped
-        current_execution&.mark_skipped!(outcome: "skipped")
+        # Workflow is ready with a scheduled step - mark it as skipped. A
+        # dispatched-but-not-yet-run execution carries an attached signal;
+        # skipping it settles that signal too.
+        skipped_execution = current_execution
+        skipped_execution&.mark_skipped!(outcome: "skipped")
+        settle_signal_for_skipped!(skipped_execution)
       end
 
       schedule_next_step!
     end
   end
+
+  # Finds the execution that still holds a signal claim for a step we are
+  # about to skip past on a paused workflow: the current one if it was
+  # dispatched before the pause, otherwise the failed attempt that paused the
+  # workflow in the first place (same lookup shape resume! uses).
+  #
+  # @param step_name [String, nil] the step being skipped past
+  # @return [GenevaDrive::StepExecution, nil]
+  def execution_holding_signal_claim(step_name)
+    return nil unless GenevaDrive::StepExecution.signal_columns?
+
+    candidate = current_execution
+    return candidate if candidate&.signal_id.present?
+    return nil if step_name.blank?
+
+    step_executions
+      .where(step_name: step_name, state: "failed")
+      .order(created_at: :desc, id: :desc)
+      .first
+  end
 end

data/lib/geneva_drive/iterable_step.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+# Object passed to resumable step blocks for cursor-based iteration.
+#
+# Provides an API compatible with Rails 8.1's ActiveJob::Continuation::Step,
+# plus GenevaDrive-specific extensions for iterating over records and batches.
+#
+# @example Manual cursor control
+#   resumable_step :process_records do |iter|
+#     hero.records.where("id > ?", iter.cursor || 0).find_each do |record|
+#       process(record)
+#       iter.set!(record.id)
+#     end
+#   end
+#
+# @example Using iterate_over_records helper
+#   resumable_step :process_records do |iter|
+#     iter.iterate_over_records(hero.records) do |record|
+#       process(record)
+#     end
+#   end
+#
+class GenevaDrive::IterableStep
+  attr_reader :name, :cursor
+
+  # Creates a new IterableStep for a resumable step execution.
+  #
+  # @param name [String, Symbol] the step name
+  # @param cursor [Object, nil] the current cursor value
+  # @param execution [GenevaDrive::StepExecution] the step execution record
+  # @param resumed [Boolean] whether this is a resumed execution
+  # @param interrupter [#should_interrupt?] object to check for interruption
+  def initialize(name, cursor, execution:, resumed:, interrupter:)
+    @name = name.to_sym
+    @cursor = cursor
+    @execution = execution
+    @resumed = resumed
+    @interrupter = interrupter
+    @advanced = false
+    @iterations = 0
+  end
+
+  # Returns the number of iterations completed during this execution
+  # (checkpoints via set!, advance! or checkpoint!).
+  #
+  # @return [Integer]
+  attr_reader :iterations
+
+  # === Rails 8.1 ActiveJob::Continuation::Step compatible API ===
+
+  # Returns true if this execution is resuming from a prior suspension.
+  #
+  # @return [Boolean]
+  def resumed?
+    @resumed
+  end
+
+  # Returns true if the cursor was changed during this execution.
+  #
+  # @return [Boolean]
+  def advanced?
+    @advanced
+  end
+
+  # Sets the cursor to a new value and checkpoints.
+  # The cursor can be any value that ActiveJob can serialize.
+  #
+  # @param value [Object] the new cursor value
+  # @return [void]
+  def set!(value)
+    @cursor = value
+    @advanced = true
+    checkpoint!
+  end
+
+  # Increments an integer cursor by 1 and checkpoints.
+  # Only works with integer cursors.
+  #
+  # @raise [ArgumentError] if cursor is not an Integer
+  # @return [void]
+  def advance!
+    raise ArgumentError, "advance! requires integer cursor, got #{@cursor.inspect}" unless @cursor.is_a?(Integer)
+    set!(@cursor + 1)
+  end
+
+  # Persists the current cursor, counts the iteration, and checks for
+  # interruption (max_iterations, max_runtime, shutdown, external pause/cancel).
+  # Called automatically by set! and advance!.
+  #
+  # @return [void]
+  def checkpoint!
+    @iterations += 1
+    persist_cursor!
+    check_interruption!
+  end
+
+  # Returns array representation [name, cursor] for compatibility.
+  #
+  # @return [Array]
+  def to_a
+    [name.to_s, cursor]
+  end
+
+  # Returns a human-readable description of current position.
+  #
+  # @return [String]
+  def description
+    "at '#{name}', cursor '#{cursor.inspect}'"
+  end
+
+  # === GenevaDrive Extensions ===
+
+  # Skips to a new cursor position and suspends execution.
+  # The job will be re-enqueued to continue from the new cursor.
+  #
+  # @param value [Object] the new cursor value
+  # @param wait [ActiveSupport::Duration, Numeric, nil] optional delay before re-enqueue
+  # @return [void] (never returns - throws :interrupt)
+  def skip_to!(value, wait: nil)
+    @cursor = value
+    @advanced = true
+    persist_cursor!
+    throw :interrupt, wait
+  end
+
+  # Iterates over an enumerable using index as cursor.
+  # Resumes from the stored cursor position on subsequent runs.
+  #
+  # @param enumerable [Enumerable] the collection to iterate over
+  # @yield [Object] each element in the collection
+  # @return [void]
+  def iterate_over(enumerable)
+    raise ArgumentError, "Must respond to #each" unless enumerable.respond_to?(:each)
+
+    start_index = @cursor || 0
+    enumerable.each_with_index do |element, index|
+      next if index < start_index
+      yield element
+      set!(index + 1)
+    end
+  end
+
+  # Iterates over an ActiveRecord relation using a column as cursor.
+  # Uses find_each for memory-efficient batching.
+  #
+  # @param relation [ActiveRecord::Relation] the relation to iterate over
+  # @param cursor [Symbol] the column to use as cursor (default: :id)
+  # @yield [ActiveRecord::Base] each record in the relation
+  # @return [void]
+  def iterate_over_records(relation, cursor: :id)
+    filtered = if @cursor
+      relation.where(relation.arel_table[cursor].gt(@cursor)).order(cursor => :asc)
+    else
+      relation.order(cursor => :asc)
+    end
+
+    filtered.find_each do |record|
+      yield record
+      set!(record.public_send(cursor))
+    end
+  end
+
+  # Iterates over an ActiveRecord relation in subrelations (batches).
+  # Each subrelation is yielded as an ActiveRecord relation with offset applied,
+  # useful for bulk operations like updates or exports.
+  #
+  # @param relation [ActiveRecord::Relation] the relation to iterate over
+  # @param batch_size [Integer] the number of records per subrelation
+  # @param cursor [Symbol] the column to use as cursor (default: :id)
+  # @yield [ActiveRecord::Relation] each subrelation with offset applied
+  # @return [void]
+  def iterate_over_subrelations(relation, batch_size:, cursor: :id)
+    filtered = if @cursor
+      relation.where(relation.arel_table[cursor].gt(@cursor)).order(cursor => :asc)
+    else
+      relation.order(cursor => :asc)
+    end
+
+    filtered.in_batches(of: batch_size) do |relation_with_offset|
+      yield relation_with_offset
+      last_cursor = relation_with_offset.maximum(cursor)
+      set!(last_cursor) if last_cursor
+    end
+  end
+
+  private
+
+  def persist_cursor!
+    serialized = GenevaDrive::StepExecution.serialize_cursor(@cursor)
+
+    GenevaDrive::StepExecution
+      .where(id: @execution.id)
+      .update_all(cursor: serialized)
+  end
+
+  def check_interruption!
+    throw :interrupt if @interrupter&.should_interrupt?(iterations: @iterations)
+  end
+end

data/lib/geneva_drive/job_options.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# Validation helper for Active Job option hashes passed through the workflow DSL
+# (`set_step_job_options`, `step(job_options: ...)`, and `workflow.step_job_options=`).
+#
+# Active Job itself silently ignores unknown keys in `set` — a typo like
+# `piority:` would never raise, and the workflow would just run at the default
+# priority. This helper enforces the known key set so misconfigurations surface
+# at definition time, or at assignment time for per-instance overrides.
+#
+# The allowed keys mirror what `ActiveJob::Core#set` actually consumes.
+#
+# @api private
+module GenevaDrive::JobOptions
+  # Keys that Active Job's `set` recognizes. See `ActiveJob::Core#set` in
+  # activejob/lib/active_job/core.rb.
+  ALLOWED_KEYS = %i[wait wait_until queue priority].freeze
+
+  # Validates a job options hash and returns a symbolized copy.
+  #
+  # @param options [#to_h, Hash, nil] the value to validate — anything that
+  #   coerces to a Hash via `#to_h` is accepted (Hash, keyword-args splat,
+  #   ActiveSupport::HashWithIndifferentAccess, structs, etc.)
+  # @param context [String] human-readable identifier used in error messages
+  #   (e.g. "Step 'foo'" or "MyWorkflow.step_job_options")
+  # @return [Hash{Symbol=>Object}] symbolized copy safe to pass to `.set`
+  # @raise [GenevaDrive::StepConfigurationError] if options can't be coerced to
+  #   a Hash, contains unknown keys, or has values of the wrong type
+  def self.validate!(options, context:)
+    hash = begin
+      options.to_h
+    rescue NoMethodError, TypeError, ArgumentError
+      raise GenevaDrive::StepConfigurationError,
+        "#{context} has invalid job_options: cannot coerce #{options.class} to a Hash via #to_h"
+    end
+
+    symbolized = hash.symbolize_keys
+
+    unknown = symbolized.keys - ALLOWED_KEYS
+    unless unknown.empty?
+      raise GenevaDrive::StepConfigurationError,
+        "#{context} has unknown job_options key(s): #{unknown.map(&:inspect).join(", ")}. " \
+        "Active Job only recognizes: #{ALLOWED_KEYS.map(&:inspect).join(", ")}"
+    end
+
+    symbolized.each { |k, v| validate_value!(k, v, context) }
+    symbolized
+  end
+
+  # @api private
+  def self.validate_value!(key, value, context)
+    return if value.nil?
+
+    valid = case key
+    when :wait
+      value.respond_to?(:to_i) && value.to_i >= 0
+    when :wait_until
+      value.respond_to?(:to_time) || value.respond_to?(:to_f)
+    when :queue
+      value.is_a?(String) || value.is_a?(Symbol)
+    when :priority
+      value.is_a?(Integer)
+    end
+
+    return if valid
+
+    raise GenevaDrive::StepConfigurationError,
+      "#{context} has invalid job_options[#{key.inspect}]: #{value.inspect} (#{value.class})"
+  end
+end

data/lib/geneva_drive/jobs/housekeeping_job.rb

@@ -33,6 +33,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
     results = {
       workflows_cleaned_up: 0,
       step_executions_cleaned_up: 0,
+      signals_cleaned_up: 0,
       stuck_in_progress_recovered: 0,
       stuck_scheduled_recovered: 0
     }
@@ -40,6 +41,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
     cleanup_completed_workflows!(results)
     recover_stuck_step_executions!(results)
     report_workflow_gauges!
+    report_waiting_gauges!
 
     logger.info("Completed: #{results}")
     results
@@ -48,9 +50,16 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
   private
 
   # Reports workflow count gauges to Measurometer.
-  # For each state (ready, paused, finished), sets:
-  # - A total gauge without tags
-  # - Per-workflow-class gauges tagged with `workflow: ClassName`
+  # For each state present in the workflows table (e.g. ready, paused,
+  # performing, finished, canceled), sets:
+  # - A total gauge `geneva_drive.<state>` without tags (summed across all classes)
+  # - Per-workflow-class gauges `geneva_drive.<state>` tagged with `workflow: ClassName`
+  #
+  # Additionally emits a normalized `geneva_drive.paused_ratio` gauge per
+  # workflow class — a float in `0.0..1.0` equal to that class's `paused` count
+  # divided by that class's total population (all states). This is easier to
+  # alert on than absolute paused counts (50 paused of 50 is a fire; 50 of
+  # 500,000 is noise). All gauges are derived from a single grouped query.
   #
   # @return [void]
   def report_workflow_gauges!
@@ -64,6 +73,58 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
     counts.each do |(state, type), count|
       Measurometer.set_gauge("geneva_drive.#{state}", count, workflow: type)
     end
+
+    # Per-class paused ratio (paused ÷ total population), deliberately bounded 0..1.
+    # Both accumulators default to 0 so a class with no paused workflows yields a clean 0.0.
+    totals_by_type = Hash.new(0)
+    paused_by_type = Hash.new(0)
+    counts.each do |(state, type), count|
+      totals_by_type[type] += count
+      paused_by_type[type] += count if state == "paused"
+    end
+
+    totals_by_type.each do |type, total|
+      # A type only appears here when it has >= 1 row, so total is always >= 1.
+      # Guard against division by zero anyway as belt-and-suspenders.
+      next if total.zero?
+      Measurometer.set_gauge("geneva_drive.paused_ratio", paused_by_type[type].to_f / total, workflow: type)
+    end
+  end
+
+  # Reports gauges for step executions parked waiting for a signal.
+  #
+  # Waiting indefinitely is a legitimate state, not a stuck one, so the stuck
+  # sweeps ignore parked rows. These gauges are what keeps a stalled
+  # rendezvous from being silent:
+  # - `geneva_drive.waiting_step_executions` - parked rows, total and per class
+  # - `geneva_drive.waiting_overdue` - parked longer than
+  #   GenevaDrive.waiting_visibility_threshold, total and per class
+  #
+  # @return [void]
+  def report_waiting_gauges!
+    return unless GenevaDrive::StepExecution.signal_columns?
+
+    report_waiting_gauge!("geneva_drive.waiting_step_executions", GenevaDrive::StepExecution.waiting)
+
+    threshold = GenevaDrive.waiting_visibility_threshold
+    return if threshold.blank?
+
+    overdue = GenevaDrive::StepExecution.waiting.where(waiting_since: ..threshold.ago)
+    report_waiting_gauge!("geneva_drive.waiting_overdue", overdue)
+  end
+
+  # Sets one gauge per workflow class plus an untagged total for the scope.
+  #
+  # @param gauge_name [String] the Measurometer gauge name
+  # @param scope [ActiveRecord::Relation] a step execution scope
+  # @return [void]
+  def report_waiting_gauge!(gauge_name, scope)
+    counts = scope.joins(:workflow).group("#{GenevaDrive::Workflow.table_name}.type").count
+
+    Measurometer.set_gauge(gauge_name, counts.values.sum)
+    counts.each do |type, count|
+      Measurometer.set_gauge(gauge_name, count, workflow: type)
+    end
   end
 
   # Cleans up completed/canceled workflows older than the configured threshold.
@@ -91,7 +152,17 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
       break if deleted_count < batch_size
     end
 
-    # Second pass: delete the workflows themselves
+    # Second pass: delete the signals delivered to old workflows
+    if GenevaDrive::Signal.table_available?
+      loop do
+        deleted_count = delete_signals_batch(cutoff_time, batch_size)
+        logger.info("Deleted #{deleted_count} signals")
+        results[:signals_cleaned_up] += deleted_count
+        break if deleted_count < batch_size
+      end
+    end
+
+    # Third pass: delete the workflows themselves
     loop do
       deleted_count = delete_workflows_batch(cutoff_time, batch_size)
       logger.info("Deleted #{deleted_count} workflows")
@@ -100,11 +171,43 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
     end
 
     logger.info(
-      "Cleaned up #{results[:workflows_cleaned_up]} workflows " \
-      "and #{results[:step_executions_cleaned_up]} step executions older than #{cutoff_time}"
+      "Cleaned up #{results[:workflows_cleaned_up]} workflows, " \
+      "#{results[:step_executions_cleaned_up]} step executions and " \
+      "#{results[:signals_cleaned_up]} signals older than #{cutoff_time}"
     )
   end
 
+  # Deletes a batch of signals belonging to old workflows.
+  #
+  # @param cutoff_time [Time] workflows transitioned before this time are eligible
+  # @param batch_size [Integer] maximum records to delete in this batch
+  # @return [Integer] number of records deleted
+  def delete_signals_batch(cutoff_time, batch_size)
+    GenevaDrive::Signal.connection_pool.with_connection do |conn|
+      signals_table = conn.quote_table_name(GenevaDrive::Signal.table_name)
+      workflows_table = conn.quote_table_name(GenevaDrive::Workflow.table_name)
+      limit = batch_size.to_i
+
+      # MySQL doesn't support LIMIT in subqueries with IN, so we wrap it in another SELECT
+      # Also, MySQL doesn't handle bind parameters for LIMIT properly, so we interpolate directly
+      sql = <<~SQL.squish
+        DELETE FROM #{signals_table}
+        WHERE id IN (
+          SELECT id FROM (
+            SELECT s.id
+            FROM #{signals_table} s
+            INNER JOIN #{workflows_table} w ON w.id = s.workflow_id
+            WHERE w.state IN ('finished', 'canceled')
+            AND w.transitioned_at < ?
+            LIMIT #{limit}
+          ) AS batch_to_delete
+        )
+      SQL
+
+      conn.delete(GenevaDrive::Signal.sanitize_sql([sql, cutoff_time]))
+    end
+  end
+
   # Deletes a batch of step executions belonging to old workflows.
   #
   # @param cutoff_time [Time] workflows transitioned before this time are eligible
@@ -190,6 +293,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
       stuck_executions = GenevaDrive::StepExecution
         .in_progress
         .where("started_at < ?", cutoff_time)
+        .order(started_at: :asc)
         .limit(batch_size)
         .to_a
 
@@ -219,6 +323,7 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
       stuck_executions = GenevaDrive::StepExecution
         .scheduled
         .where("scheduled_for < ?", cutoff_time)
+        .order(scheduled_for: :asc)
         .limit(batch_size)
         .to_a
 
@@ -256,6 +361,9 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
 
   # Reattempts a stuck step execution by marking it as completed and scheduling a retry.
   #
+  # Resumable step executions carry a cursor; those are continued via a
+  # successor execution so recovery does not restart the iteration from zero.
+  #
   # @param step_execution [GenevaDrive::StepExecution]
   # @param workflow [GenevaDrive::Workflow]
   # @return [void]
@@ -273,7 +381,12 @@ class GenevaDrive::HousekeepingJob < ActiveJob::Base
         )
 
         workflow.update!(state: "ready") if workflow.performing?
-        workflow.reschedule_current_step!
+
+        if step_execution.step_definition&.resumable? && GenevaDrive::StepExecution.resumable_columns?
+          workflow.create_successor_execution!(step_execution)
+        else
+          workflow.reschedule_current_step!
+        end
       end
     end
   end