geneva_drive 0.5.0 → 0.7.0

data/lib/geneva_drive.rb

@@ -31,15 +31,23 @@ module GenevaDrive
   autoload :StepConfigurationError, "geneva_drive/flow_control"
   autoload :StepExecutionError, "geneva_drive/flow_control"
   autoload :StepNotDefinedError, "geneva_drive/flow_control"
+  autoload :CursorTooLargeError, "geneva_drive/flow_control"
+  autoload :SignalPayloadTooLargeError, "geneva_drive/flow_control"
+  autoload :WorkflowNotOngoing, "geneva_drive/flow_control"
   autoload :StepFailedError, "geneva_drive/flow_control"
   autoload :PreconditionError, "geneva_drive/flow_control"
   autoload :FlowControl, "geneva_drive/flow_control"
   autoload :StepDefinition, "geneva_drive/step_definition"
+  autoload :ResumableStepDefinition, "geneva_drive/resumable_step_definition"
   autoload :StepCollection, "geneva_drive/step_collection"
   autoload :Workflow, "geneva_drive/workflow"
   autoload :StepExecution, "geneva_drive/step_execution"
+  autoload :Signal, "geneva_drive/signal"
+  autoload :SignalMatcher, "geneva_drive/signal_matcher"
+  autoload :IterableStep, "geneva_drive/iterable_step"
   autoload :Executor, "geneva_drive/executor"
   autoload :ExceptionPolicy, "geneva_drive/exception_policy"
+  autoload :JobOptions, "geneva_drive/job_options"
   autoload :CombinedExceptionPolicy, "geneva_drive/combined_exception_policy"
   autoload :MigrationHelpers, "geneva_drive/migration_helpers"
   autoload :TestHelpers, "geneva_drive/test_helpers"
@@ -74,6 +82,32 @@ module GenevaDrive
     # @return [Symbol]
     attr_accessor :stuck_recovery_action
 
+    # Maximum size in bytes of a resumable step cursor once serialized to
+    # JSON. The cursor is meant to be a compact position marker (an id, a
+    # page number, a pagination token) - not a payload. It is rewritten on
+    # every checkpoint and copied to every successor execution, so an
+    # oversized cursor multiplies write load and row size. Exceeding the
+    # limit raises CursorTooLargeError from set!/checkpoint!.
+    # Set to nil to disable the check.
+    # @return [Integer, nil]
+    attr_accessor :max_cursor_size
+
+    # Maximum size in bytes of a signal payload once serialized to JSON.
+    # A payload describes an event - an order id, an amount, a decision -
+    # not the data the step is going to work on. Exceeding the limit raises
+    # SignalPayloadTooLargeError from Workflow#signal!, before anything is
+    # persisted. Set to nil to disable the check.
+    # @return [Integer, nil]
+    attr_accessor :max_signal_payload_size
+
+    # How long a step execution may sit in the "waiting" state before
+    # housekeeping counts it in the `geneva_drive.waiting_overdue` gauge.
+    # Waiting indefinitely is legitimate, so this is not a timeout - it is
+    # the floor that keeps a stalled rendezvous from being silent.
+    # Set to nil to disable the gauge.
+    # @return [ActiveSupport::Duration, nil]
+    attr_accessor :waiting_visibility_threshold
+
     # Whether to defer job enqueueing to after the database transaction commits.
     # When true (the default in non-test environments), jobs are enqueued inside
     # an `after_all_transactions_commit` callback to ensure the step execution
@@ -160,5 +194,8 @@ module GenevaDrive
   self.stuck_scheduled_threshold = 15.minutes
   self.housekeeping_batch_size = 1000
   self.stuck_recovery_action = :reattempt
+  self.max_cursor_size = 128 * 1024
+  self.max_signal_payload_size = 128 * 1024
+  self.waiting_visibility_threshold = 7.days
   self.enqueue_after_commit = !Rails.env.test?
 end

data/test/dsl/signal_step_definition_test.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+# wait: and wait_for: read alike and take disjoint types, so the step
+# definition enforces the disjointness in both directions at class load.
+class SignalStepDefinitionTest < ActiveSupport::TestCase
+  class NameSetMatcher
+    def initialize(*names)
+      @names = names.map(&:to_s)
+    end
+
+    def matches?(signal)
+      @names.include?(signal.name)
+    end
+  end
+
+  def build_step(**options)
+    GenevaDrive::StepDefinition.new(name: "capture", callable: proc {}, **options)
+  end
+
+  test "wait_for: with a symbol builds a name matcher" do
+    step_def = build_step(wait_for: :payment_confirmed)
+
+    assert step_def.waits_for_signal?
+    assert_equal "payment_confirmed", step_def.signal_matcher.name
+    assert_nil step_def.signal_matcher.condition
+  end
+
+  test "wait_for: with a string builds a name matcher" do
+    step_def = build_step(wait_for: "payment_confirmed")
+
+    assert step_def.waits_for_signal?
+    assert_equal "payment_confirmed", step_def.signal_matcher.name
+  end
+
+  test "wait_for: with a SignalMatcher keeps it as-is" do
+    matcher = GenevaDrive::SignalMatcher.new(:payment_confirmed) { |payload| payload[:order_id] == 1 }
+    step_def = build_step(wait_for: matcher)
+
+    assert_same matcher, step_def.signal_matcher
+    assert_equal "payment_confirmed", step_def.signal_matcher.name
+    assert step_def.signal_matcher.condition
+  end
+
+  test "a SignalMatcher needs a name" do
+    assert_raises(ArgumentError) { GenevaDrive::SignalMatcher.new(nil) }
+    assert_raises(ArgumentError) { GenevaDrive::SignalMatcher.new("") }
+  end
+
+  test "wait_for: with a custom matcher object delegates the whole predicate" do
+    matcher = NameSetMatcher.new(:a, :b)
+    step_def = build_step(wait_for: matcher)
+
+    assert step_def.waits_for_signal?
+    assert_same matcher, step_def.signal_matcher
+  end
+
+  test "steps without wait_for: do not wait for a signal" do
+    assert_not build_step.waits_for_signal?
+    assert_nil build_step.signal_matcher
+  end
+
+  test "wait_for: with a duration points at wait:" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) { build_step(wait_for: 2.days) }
+
+    assert_match(/wait_for: 2 days/, error.message)
+    assert_match(/to delay the step, use wait:/, error.message)
+  end
+
+  test "wait_for: with a number points at wait:" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) { build_step(wait_for: 30) }
+
+    assert_match(/to delay the step, use wait:/, error.message)
+  end
+
+  test "wait_for: with a time points at wait:" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) { build_step(wait_for: Time.current) }
+
+    assert_match(/to delay the step, use wait:/, error.message)
+  end
+
+  test "wait: with a symbol points at wait_for:" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) { build_step(wait: :payment_confirmed) }
+
+    assert_match(/wait: :payment_confirmed/, error.message)
+    assert_match(/to wait for a signal, use wait_for:/, error.message)
+  end
+
+  test "wait: with a string points at wait_for: instead of silently meaning zero seconds" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) { build_step(wait: "payment_confirmed") }
+
+    assert_match(/to wait for a signal, use wait_for:/, error.message)
+  end
+
+  test "wait: with a duration-shaped string is rejected too" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) { build_step(wait: "7200") }
+
+    assert_match(/to wait for a signal, use wait_for:/, error.message)
+  end
+
+  test "wait: with a matcher object points at wait_for:" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) { build_step(wait: NameSetMatcher.new(:a)) }
+
+    assert_match(/to wait for a signal, use wait_for:/, error.message)
+  end
+
+  test "wait: still accepts durations and numbers" do
+    assert_equal 2.days, build_step(wait: 2.days).wait
+    assert_equal 30, build_step(wait: 30).wait
+  end
+
+  test "wait_for: with an unusable value is rejected" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) { build_step(wait_for: Object.new) }
+
+    assert_match(/must be a Symbol, String, GenevaDrive::SignalMatcher, or an object responding to #matches\?/, error.message)
+  end
+
+  test "wait: and wait_for: compose" do
+    step_def = build_step(wait: 2.days, wait_for: :payment_confirmed)
+
+    assert_equal 2.days, step_def.wait
+    assert step_def.waits_for_signal?
+  end
+
+  test "resumable steps accept wait_for:" do
+    step_def = GenevaDrive::ResumableStepDefinition.new(name: "iterate", wait_for: :go) { |iter| }
+
+    assert step_def.resumable?
+    assert step_def.waits_for_signal?
+  end
+end

data/test/dsl/step_definition_test.rb

@@ -140,6 +140,76 @@ class StepDefinitionTest < ActiveSupport::TestCase
     assert_match(/non-negative/, error.message)
   end
 
+  test "rejects job_options that can't be coerced to a Hash" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) do
+      Class.new(GenevaDrive::Workflow) do
+        step(:invalid, job_options: :critical) {}
+      end
+    end
+
+    assert_match(/cannot coerce Symbol to a Hash/, error.message)
+  end
+
+  test "accepts anything responding to to_h as job_options" do
+    struct = Struct.new(:queue, :priority).new(:critical, 5)
+    workflow_class = Class.new(GenevaDrive::Workflow) do
+      step(:from_struct, job_options: struct) {}
+    end
+
+    assert_equal({queue: :critical, priority: 5}, workflow_class.step_definitions.first.job_options)
+  end
+
+  test "rejects unknown job_options keys" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) do
+      Class.new(GenevaDrive::Workflow) do
+        step(:typo, job_options: {piority: 10}) {}
+      end
+    end
+
+    assert_match(/unknown job_options key/, error.message)
+    assert_match(/piority/, error.message)
+  end
+
+  test "rejects non-integer priority" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) do
+      Class.new(GenevaDrive::Workflow) do
+        step(:bad_pri, job_options: {priority: "high"}) {}
+      end
+    end
+
+    assert_match(/job_options\[:priority\]/, error.message)
+  end
+
+  test "rejects non-symbol/string queue" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) do
+      Class.new(GenevaDrive::Workflow) do
+        step(:bad_queue, job_options: {queue: 42}) {}
+      end
+    end
+
+    assert_match(/job_options\[:queue\]/, error.message)
+  end
+
+  test "accepts valid queue and priority" do
+    workflow_class = Class.new(GenevaDrive::Workflow) do
+      step(:ok, job_options: {queue: :critical, priority: -1, wait: 5}) {}
+    end
+
+    assert_equal({queue: :critical, priority: -1, wait: 5}, workflow_class.step_definitions.first.job_options)
+  end
+
+  test "copies, symbolizes, and freezes job options" do
+    job_options = {"queue" => :critical}
+    workflow_class = Class.new(GenevaDrive::Workflow) do
+      step(:urgent, job_options: job_options) {}
+    end
+    job_options["queue"] = :default
+
+    options = workflow_class.step_definitions.first.job_options
+    assert_equal({queue: :critical}, options)
+    assert_predicate options, :frozen?
+  end
+
   # Tests for on_exception validation
   test "accepts valid exception handlers" do
     workflow_class = Class.new(GenevaDrive::Workflow) do

data/test/jobs/housekeeping_job_test.rb

@@ -5,6 +5,16 @@ require "test_helper"
 class HousekeepingJobTest < ActiveSupport::TestCase
   include ActiveJob::TestHelper
 
+  # Captures Measurometer gauge/metric calls for assertions.
+  class GaugeRecorder
+    attr_reader :gauges
+    def initialize = @gauges = []
+    def set_gauge(name, value, tags = {}) = @gauges << {name: name.to_s, value: value, tags: tags}
+    def instrument(*, &blk) = blk&.call
+    def add_distribution_value(*) = nil
+    def increment_counter(*) = nil
+  end
+
   class SimpleWorkflow < GenevaDrive::Workflow
     step :step_one do
       # Step body
@@ -41,6 +51,8 @@ class HousekeepingJobTest < ActiveSupport::TestCase
     @original_stuck_scheduled = GenevaDrive.stuck_scheduled_threshold
     @original_recovery_action = GenevaDrive.stuck_recovery_action
     @original_batch_size = GenevaDrive.housekeeping_batch_size
+    @recorder = GaugeRecorder.new
+    Measurometer.drivers << @recorder
   end
 
   teardown do
@@ -49,6 +61,15 @@ class HousekeepingJobTest < ActiveSupport::TestCase
     GenevaDrive.stuck_scheduled_threshold = @original_stuck_scheduled
     GenevaDrive.stuck_recovery_action = @original_recovery_action
     GenevaDrive.housekeeping_batch_size = @original_batch_size
+    Measurometer.drivers.delete(@recorder)
+  end
+
+  # Returns the paused_ratio gauge value recorded for a given workflow class.
+  def paused_ratio_for(workflow_class)
+    gauge = @recorder.gauges.find do |g|
+      g[:name] == "geneva_drive.paused_ratio" && g[:tags][:workflow] == workflow_class.name
+    end
+    gauge&.fetch(:value)
   end
 
   # Cleanup tests
@@ -314,6 +335,59 @@ class HousekeepingJobTest < ActiveSupport::TestCase
     assert_equal "scheduled", step_execution.state
   end
 
+  test "recover_stuck_in_progress! orders by started_at ascending and terminates" do
+    GenevaDrive.stuck_in_progress_threshold = 1.hour
+    GenevaDrive.stuck_recovery_action = :cancel
+    GenevaDrive.housekeeping_batch_size = 2
+
+    # 3 stuck (old) + 2 fresh (within threshold, must not be touched)
+    stuck = 3.times.map do |i|
+      wf = SimpleWorkflow.create!(hero: @user, allow_multiple: true)
+      wf.update!(state: "performing")
+      se = wf.step_executions.first
+      se.update!(state: "in_progress", started_at: (3 + i).hours.ago)
+      se
+    end
+
+    fresh = 2.times.map do
+      wf = SimpleWorkflow.create!(hero: @user, allow_multiple: true)
+      wf.update!(state: "performing")
+      se = wf.step_executions.first
+      se.update!(state: "in_progress", started_at: 10.minutes.ago)
+      se
+    end
+
+    result = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 3, result[:stuck_in_progress_recovered]
+    stuck.each { |se| assert_equal "canceled", se.reload.state }
+    fresh.each { |se| assert_equal "in_progress", se.reload.state }
+  end
+
+  test "recover_stuck_scheduled! orders by scheduled_for ascending and terminates" do
+    GenevaDrive.stuck_scheduled_threshold = 1.hour
+    GenevaDrive.stuck_recovery_action = :cancel
+    GenevaDrive.housekeeping_batch_size = 2
+
+    stuck = 3.times.map do |i|
+      wf = SimpleWorkflow.create!(hero: @user, allow_multiple: true)
+      se = wf.step_executions.first
+      se.update!(scheduled_for: (2 + i).hours.ago)
+      se
+    end
+
+    fresh = 2.times.map do
+      wf = SimpleWorkflow.create!(hero: @user, allow_multiple: true)
+      wf.step_executions.first
+    end
+
+    result = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 3, result[:stuck_scheduled_recovered]
+    stuck.each { |se| assert_equal "canceled", se.reload.state }
+    fresh.each { |se| assert_equal "scheduled", se.reload.state }
+  end
+
   # Combined tests
 
   test "performs both cleanup and recovery in one run" do
@@ -373,6 +447,51 @@ class HousekeepingJobTest < ActiveSupport::TestCase
     assert result.key?(:stuck_scheduled_recovered)
   end
 
+  # Paused ratio gauge tests
+
+  test "emits per-class paused_ratio gauges for mixed states" do
+    # SimpleWorkflow: 1 paused of 4 total => 0.25
+    4.times do |i|
+      workflow = SimpleWorkflow.create!(hero: @user, allow_multiple: true)
+      workflow.transition_to!("paused") if i.zero?
+    end
+
+    # ThreeStepWorkflow: 2 paused of 2 total => 1.0
+    2.times do
+      workflow = ThreeStepWorkflow.create!(hero: @user, allow_multiple: true)
+      workflow.transition_to!("paused")
+    end
+
+    GenevaDrive::HousekeepingJob.perform_now
+
+    assert_in_delta 0.25, paused_ratio_for(SimpleWorkflow), 1e-9
+    assert_in_delta 1.0, paused_ratio_for(ThreeStepWorkflow), 1e-9
+  end
+
+  test "reports 0.0 paused_ratio for a class with no paused workflows" do
+    3.times do
+      SimpleWorkflow.create!(hero: @user, allow_multiple: true)
+    end
+
+    GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 0.0, paused_ratio_for(SimpleWorkflow)
+  end
+
+  test "denominator is total population, not ongoing-only" do
+    # Only finished/canceled workflows exist for this class - no paused, no ongoing.
+    # The ratio must still be reported as 0.0 (paused ÷ total), not skipped.
+    finished = SimpleWorkflow.create!(hero: @user, allow_multiple: true)
+    finished.transition_to!("finished")
+
+    canceled = SimpleWorkflow.create!(hero: @user, allow_multiple: true)
+    canceled.transition_to!("canceled")
+
+    GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 0.0, paused_ratio_for(SimpleWorkflow)
+  end
+
   test "processes all stuck in_progress executions in multiple batches" do
     GenevaDrive.stuck_in_progress_threshold = 1.hour
     GenevaDrive.stuck_recovery_action = :cancel

data/test/jobs/housekeeping_signals_test.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+# Signals are wiped with their workflows, and parked executions - which the
+# stuck sweeps deliberately ignore - are reported through gauges instead.
+class HousekeepingSignalsTest < ActiveSupport::TestCase
+  include GenevaDrive::TestHelpers
+
+  class GaugeRecorder
+    attr_reader :gauges
+
+    def initialize
+      @gauges = []
+    end
+
+    def set_gauge(name, value, tags = {})
+      @gauges << {name: name.to_s, value: value, tags: tags}
+    end
+
+    def instrument(*, &blk) = blk&.call
+
+    def add_distribution_value(*) = nil
+
+    def increment_counter(*) = nil
+  end
+
+  class WaitingWorkflow < GenevaDrive::Workflow
+    step :await_payment, wait_for: :payment_confirmed do
+    end
+  end
+
+  class OtherWaitingWorkflow < GenevaDrive::Workflow
+    step :await_approval, wait_for: :approved do
+    end
+  end
+
+  setup do
+    @user = create_user
+    @original_delete_after = GenevaDrive.delete_completed_workflows_after
+    @original_batch_size = GenevaDrive.housekeeping_batch_size
+    @original_waiting_threshold = GenevaDrive.waiting_visibility_threshold
+    @recorder = GaugeRecorder.new
+    Measurometer.drivers << @recorder
+  end
+
+  teardown do
+    GenevaDrive.delete_completed_workflows_after = @original_delete_after
+    GenevaDrive.housekeeping_batch_size = @original_batch_size
+    GenevaDrive.waiting_visibility_threshold = @original_waiting_threshold
+    Measurometer.drivers.delete(@recorder)
+  end
+
+  def gauge(name, workflow: nil)
+    found = @recorder.gauges.select { |g| g[:name] == name && g[:tags][:workflow] == workflow }
+    found.last&.fetch(:value)
+  end
+
+  def park_workflow(workflow_class, user)
+    workflow = workflow_class.create!(hero: user)
+    workflow.current_execution.execute!
+    workflow.reload
+  end
+
+  test "signals of old workflows are deleted before the workflows themselves" do
+    GenevaDrive.delete_completed_workflows_after = 30.days
+
+    workflow = WaitingWorkflow.create!(hero: @user)
+    workflow.signal!(:payment_confirmed)
+    workflow.signal!(:unrelated)
+    workflow.cancel!
+    workflow.update!(transitioned_at: 60.days.ago)
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 2, results[:signals_cleaned_up]
+    assert_equal 1, results[:workflows_cleaned_up]
+    assert_equal 0, GenevaDrive::Signal.where(workflow_id: workflow.id).count
+    assert_not GenevaDrive::Workflow.exists?(workflow.id)
+  end
+
+  test "signals of recent workflows are kept" do
+    GenevaDrive.delete_completed_workflows_after = 30.days
+
+    workflow = WaitingWorkflow.create!(hero: @user)
+    workflow.signal!(:payment_confirmed)
+    workflow.cancel!
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 0, results[:signals_cleaned_up]
+    assert_equal 1, GenevaDrive::Signal.where(workflow_id: workflow.id).count
+  end
+
+  test "signal deletion is batched" do
+    GenevaDrive.delete_completed_workflows_after = 30.days
+    GenevaDrive.housekeeping_batch_size = 2
+
+    workflow = WaitingWorkflow.create!(hero: @user)
+    4.times { |n| workflow.signal!(:noise, idempotency_key: "evt_#{n}") }
+    workflow.cancel!
+    workflow.update!(transitioned_at: 60.days.ago)
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 4, results[:signals_cleaned_up]
+    assert_equal 0, GenevaDrive::Signal.where(workflow_id: workflow.id).count
+  end
+
+  test "parked executions are counted per workflow class" do
+    park_workflow(WaitingWorkflow, @user)
+    park_workflow(OtherWaitingWorkflow, create_user(email: "two@example.com"))
+
+    GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 1, gauge("geneva_drive.waiting_step_executions", workflow: WaitingWorkflow.name)
+    assert_equal 1, gauge("geneva_drive.waiting_step_executions", workflow: OtherWaitingWorkflow.name)
+    assert_equal 2, gauge("geneva_drive.waiting_step_executions")
+  end
+
+  test "parked executions past the visibility threshold are reported as overdue" do
+    GenevaDrive.waiting_visibility_threshold = 7.days
+
+    fresh = park_workflow(WaitingWorkflow, @user)
+    stale = park_workflow(OtherWaitingWorkflow, create_user(email: "two@example.com"))
+    stale.current_execution.update!(waiting_since: 30.days.ago)
+
+    GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 1, gauge("geneva_drive.waiting_overdue")
+    assert_equal 1, gauge("geneva_drive.waiting_overdue", workflow: OtherWaitingWorkflow.name)
+    assert_nil gauge("geneva_drive.waiting_overdue", workflow: WaitingWorkflow.name)
+    assert_equal "waiting", fresh.current_execution.state
+  end
+
+  test "the overdue gauge can be disabled" do
+    GenevaDrive.waiting_visibility_threshold = nil
+    park_workflow(WaitingWorkflow, @user)
+
+    GenevaDrive::HousekeepingJob.perform_now
+
+    assert_nil gauge("geneva_drive.waiting_overdue")
+    assert_equal 1, gauge("geneva_drive.waiting_step_executions")
+  end
+
+  test "parked executions are never treated as stuck" do
+    GenevaDrive.delete_completed_workflows_after = nil
+    workflow = park_workflow(WaitingWorkflow, @user)
+    workflow.current_execution.update!(scheduled_for: 30.days.ago, waiting_since: 30.days.ago)
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 0, results[:stuck_scheduled_recovered]
+    assert_equal 0, results[:stuck_in_progress_recovered]
+    assert_equal "waiting", workflow.reload.current_execution.state
+  end
+
+  test "a worker crashing mid-step re-attaches to the same claimed signal" do
+    GenevaDrive.delete_completed_workflows_after = nil
+    workflow = park_workflow(WaitingWorkflow, @user)
+    signal = workflow.signal!(:payment_confirmed, payload: {amount_cents: 7})
+
+    # Pretend the worker died after the gate attached and the step started
+    crashed = workflow.reload.current_execution
+    crashed.update!(state: "in_progress", started_at: 30.days.ago)
+    workflow.update!(state: "performing", current_step_name: "await_payment")
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 1, results[:stuck_in_progress_recovered]
+    assert_equal "claimed", signal.reload.state
+
+    recovered = workflow.reload.current_execution
+    recovered.execute!
+
+    assert_equal signal.id, recovered.reload.signal_id
+    assert_equal "consumed", signal.reload.state
+  end
+
+  test "a dispatched execution whose job got lost is recovered as a stuck scheduled one" do
+    GenevaDrive.delete_completed_workflows_after = nil
+    workflow = park_workflow(WaitingWorkflow, @user)
+    signal = workflow.signal!(:payment_confirmed)
+
+    # Pretend the enqueue evaporated and the threshold has passed
+    workflow.current_execution.update!(scheduled_for: 30.days.ago)
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 1, results[:stuck_scheduled_recovered]
+
+    recovered = workflow.reload.current_execution
+    assert_equal "scheduled", recovered.state
+    # The fresh execution re-attaches to the still-claimed signal at its gate
+    recovered.execute!
+    assert_equal "consumed", signal.reload.state
+  end
+end

data/test/jobs/perform_step_job_test.rb

@@ -11,6 +11,11 @@ class PerformStepJobTest < ActiveSupport::TestCase
     end
   end
 
+  class JobOptionsWorkflow < GenevaDrive::Workflow
+    set_step_job_options queue: :default, priority: 10
+    step(:test_step, job_options: {queue: :critical, priority: -1}) {}
+  end
+
   setup do
     @user = create_user
     Thread.current[:step_executed] = nil
@@ -102,7 +107,7 @@ class PerformStepJobTest < ActiveSupport::TestCase
 
   test "job passes job options from workflow" do
     workflow_class = Class.new(GenevaDrive::Workflow) do
-      set_step_job_options queue: :high_priority
+      set_step_job_options queue: :high_priority, priority: 10
 
       step :test_step do
         # Step body
@@ -119,10 +124,28 @@ class PerformStepJobTest < ActiveSupport::TestCase
 
       assert_enqueued_with(
         job: GenevaDrive::PerformStepJob,
-        queue: "high_priority"
+        queue: "high_priority",
+        priority: 10
       )
     ensure
       Object.send(:remove_const, :TestJobOptionsWorkflow)
     end
   end
+
+  test "step job options override workflow options when scheduled and resumed" do
+    clear_enqueued_jobs
+    workflow = JobOptionsWorkflow.new(hero: @user)
+    workflow.step_job_options = {queue: :instance, priority: 5}
+    workflow.save!
+
+    assert_equal("critical", enqueued_jobs.last[:queue])
+    assert_equal(-1, enqueued_jobs.last[:priority])
+
+    clear_enqueued_jobs
+    workflow.pause!
+    workflow.resume!
+
+    assert_equal("critical", enqueued_jobs.last[:queue])
+    assert_equal(-1, enqueued_jobs.last[:priority])
+  end
 end

data/test/migration_helpers_test.rb

@@ -3,7 +3,7 @@
 require "test_helper"
 
 class MigrationHelpersTest < ActiveSupport::TestCase
-  FakeColumn = Struct.new(:name, :sql_type, :default_function, keyword_init: true)
+  FakeColumn = Struct.new(:name, :sql_type, :default_function)
 
   # Creates a fresh harness with the given fake columns as the app id columns.
   def build_migration(columns = [])