geneva_drive 0.6.0 → 0.8.0

data/test/jobs/housekeeping_signals_test.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+# Signals are wiped with their workflows, and parked executions - which the
+# stuck sweeps deliberately ignore - are reported through gauges instead.
+class HousekeepingSignalsTest < ActiveSupport::TestCase
+  include GenevaDrive::TestHelpers
+
+  class GaugeRecorder
+    attr_reader :gauges
+
+    def initialize
+      @gauges = []
+    end
+
+    def set_gauge(name, value, tags = {})
+      @gauges << {name: name.to_s, value: value, tags: tags}
+    end
+
+    def instrument(*, &blk) = blk&.call
+
+    def add_distribution_value(*) = nil
+
+    def increment_counter(*) = nil
+  end
+
+  class WaitingWorkflow < GenevaDrive::Workflow
+    step :await_payment, wait_for: :payment_confirmed do
+    end
+  end
+
+  class OtherWaitingWorkflow < GenevaDrive::Workflow
+    step :await_approval, wait_for: :approved do
+    end
+  end
+
+  setup do
+    @user = create_user
+    @original_delete_after = GenevaDrive.delete_completed_workflows_after
+    @original_batch_size = GenevaDrive.housekeeping_batch_size
+    @original_waiting_threshold = GenevaDrive.waiting_visibility_threshold
+    @recorder = GaugeRecorder.new
+    Measurometer.drivers << @recorder
+  end
+
+  teardown do
+    GenevaDrive.delete_completed_workflows_after = @original_delete_after
+    GenevaDrive.housekeeping_batch_size = @original_batch_size
+    GenevaDrive.waiting_visibility_threshold = @original_waiting_threshold
+    Measurometer.drivers.delete(@recorder)
+  end
+
+  def gauge(name, workflow: nil)
+    found = @recorder.gauges.select { |g| g[:name] == name && g[:tags][:workflow] == workflow }
+    found.last&.fetch(:value)
+  end
+
+  def park_workflow(workflow_class, user)
+    workflow = workflow_class.create!(hero: user)
+    workflow.current_execution.execute!
+    workflow.reload
+  end
+
+  test "signals of old workflows are deleted before the workflows themselves" do
+    GenevaDrive.delete_completed_workflows_after = 30.days
+
+    workflow = WaitingWorkflow.create!(hero: @user)
+    workflow.signal!(:payment_confirmed)
+    workflow.signal!(:unrelated)
+    workflow.cancel!
+    workflow.update!(transitioned_at: 60.days.ago)
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 2, results[:signals_cleaned_up]
+    assert_equal 1, results[:workflows_cleaned_up]
+    assert_equal 0, GenevaDrive::Signal.where(workflow_id: workflow.id).count
+    assert_not GenevaDrive::Workflow.exists?(workflow.id)
+  end
+
+  test "signals of recent workflows are kept" do
+    GenevaDrive.delete_completed_workflows_after = 30.days
+
+    workflow = WaitingWorkflow.create!(hero: @user)
+    workflow.signal!(:payment_confirmed)
+    workflow.cancel!
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 0, results[:signals_cleaned_up]
+    assert_equal 1, GenevaDrive::Signal.where(workflow_id: workflow.id).count
+  end
+
+  test "signal deletion is batched" do
+    GenevaDrive.delete_completed_workflows_after = 30.days
+    GenevaDrive.housekeeping_batch_size = 2
+
+    workflow = WaitingWorkflow.create!(hero: @user)
+    4.times { |n| workflow.signal!(:noise, idempotency_key: "evt_#{n}") }
+    workflow.cancel!
+    workflow.update!(transitioned_at: 60.days.ago)
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 4, results[:signals_cleaned_up]
+    assert_equal 0, GenevaDrive::Signal.where(workflow_id: workflow.id).count
+  end
+
+  test "parked executions are counted per workflow class" do
+    park_workflow(WaitingWorkflow, @user)
+    park_workflow(OtherWaitingWorkflow, create_user(email: "two@example.com"))
+
+    GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 1, gauge("geneva_drive.waiting_step_executions", workflow: WaitingWorkflow.name)
+    assert_equal 1, gauge("geneva_drive.waiting_step_executions", workflow: OtherWaitingWorkflow.name)
+    assert_equal 2, gauge("geneva_drive.waiting_step_executions")
+  end
+
+  test "parked executions past the visibility threshold are reported as overdue" do
+    GenevaDrive.waiting_visibility_threshold = 7.days
+
+    fresh = park_workflow(WaitingWorkflow, @user)
+    stale = park_workflow(OtherWaitingWorkflow, create_user(email: "two@example.com"))
+    stale.current_execution.update!(waiting_since: 30.days.ago)
+
+    GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 1, gauge("geneva_drive.waiting_overdue")
+    assert_equal 1, gauge("geneva_drive.waiting_overdue", workflow: OtherWaitingWorkflow.name)
+    assert_nil gauge("geneva_drive.waiting_overdue", workflow: WaitingWorkflow.name)
+    assert_equal "waiting", fresh.current_execution.state
+  end
+
+  test "the overdue gauge can be disabled" do
+    GenevaDrive.waiting_visibility_threshold = nil
+    park_workflow(WaitingWorkflow, @user)
+
+    GenevaDrive::HousekeepingJob.perform_now
+
+    assert_nil gauge("geneva_drive.waiting_overdue")
+    assert_equal 1, gauge("geneva_drive.waiting_step_executions")
+  end
+
+  test "parked executions are never treated as stuck" do
+    GenevaDrive.delete_completed_workflows_after = nil
+    workflow = park_workflow(WaitingWorkflow, @user)
+    workflow.current_execution.update!(scheduled_for: 30.days.ago, waiting_since: 30.days.ago)
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 0, results[:stuck_scheduled_recovered]
+    assert_equal 0, results[:stuck_in_progress_recovered]
+    assert_equal "waiting", workflow.reload.current_execution.state
+  end
+
+  test "a worker crashing mid-step re-attaches to the same claimed signal" do
+    GenevaDrive.delete_completed_workflows_after = nil
+    workflow = park_workflow(WaitingWorkflow, @user)
+    signal = workflow.signal!(:payment_confirmed, payload: {amount_cents: 7})
+
+    # Pretend the worker died after the gate attached and the step started
+    crashed = workflow.reload.current_execution
+    crashed.update!(state: "in_progress", started_at: 30.days.ago)
+    workflow.update!(state: "performing", current_step_name: "await_payment")
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 1, results[:stuck_in_progress_recovered]
+    assert_equal "claimed", signal.reload.state
+
+    recovered = workflow.reload.current_execution
+    recovered.execute!
+
+    assert_equal signal.id, recovered.reload.signal_id
+    assert_equal "consumed", signal.reload.state
+  end
+
+  test "a dispatched execution whose job got lost is recovered as a stuck scheduled one" do
+    GenevaDrive.delete_completed_workflows_after = nil
+    workflow = park_workflow(WaitingWorkflow, @user)
+    signal = workflow.signal!(:payment_confirmed)
+
+    # Pretend the enqueue evaporated and the threshold has passed
+    workflow.current_execution.update!(scheduled_for: 30.days.ago)
+
+    results = GenevaDrive::HousekeepingJob.perform_now
+
+    assert_equal 1, results[:stuck_scheduled_recovered]
+
+    recovered = workflow.reload.current_execution
+    assert_equal "scheduled", recovered.state
+    # The fresh execution re-attaches to the still-claimed signal at its gate
+    recovered.execute!
+    assert_equal "consumed", signal.reload.state
+  end
+end

data/test/migration_helpers_test.rb

@@ -3,7 +3,7 @@
 require "test_helper"
 
 class MigrationHelpersTest < ActiveSupport::TestCase
-  FakeColumn = Struct.new(:name, :sql_type, :default_function, keyword_init: true)
+  FakeColumn = Struct.new(:name, :sql_type, :default_function)
 
   # Creates a fresh harness with the given fake columns as the app id columns.
   def build_migration(columns = [])

data/test/test_helper.rb

@@ -7,29 +7,52 @@ require_relative "../test/dummy/config/environment"
 ActiveRecord::Migrator.migrations_paths = [File.expand_path("../test/dummy/db/migrate", __dir__)]
 ActiveRecord::Migrator.migrations_paths << File.expand_path("../db/migrate", __dir__)
 
-# Ensure GenevaDrive migrations exist and database is prepared.
-# The generator is the single source of truth for migrations.
+# Bring the dummy app's database in line with the migrations the install
+# generator produces, recreating it from scratch whenever it has drifted.
+#
+# Rebuilding rather than migrating forward is deliberate. The generated
+# migrations are gitignored and get fresh timestamps every time they are
+# regenerated, so a database left behind by an earlier generation records
+# versions that no longer exist on disk: every migration reads as pending, and
+# replaying them onto the existing tables just fails on "table already exists".
 unless defined?(GENEVA_DRIVE_TEST_DB_PREPARED)
   GENEVA_DRIVE_TEST_DB_PREPARED = true
 
   dummy_root = File.expand_path("../test/dummy", __dir__)
-  migrations = Dir.glob("#{dummy_root}/db/migrate/*geneva_drive*.rb")
-  tables_exist = begin
-    ActiveRecord::Base.connection.table_exists?("geneva_drive_workflows")
-  rescue
-    false
-  end
+  db_config = ActiveRecord::Base.connection_db_config
+  # Adapter-specific, and deliberately so - see the comment in the dummy app's
+  # config/database.yml. Ask Rails for the path rather than rebuilding it here.
+  schema_file = ActiveRecord::Tasks::DatabaseTasks.schema_dump_path(db_config)
 
-  if migrations.empty? && !tables_exist
+  # The generator is the single source of truth for migrations.
+  if Dir.glob("#{dummy_root}/db/migrate/*geneva_drive*.rb").empty?
     puts "Generating GenevaDrive migrations..."
     Dir.chdir(dummy_root) do
       system("bin/rails", "generate", "geneva_drive:install", "--skip") || abort("Failed to generate migrations")
-      system("bin/rails", "db:migrate") || abort("Failed to run migrations")
     end
-  elsif migrations.any? && !tables_exist
-    puts "Running pending migrations..."
+  end
+
+  on_disk = Dir.glob("#{dummy_root}/db/migrate/*.rb").map { |path| File.basename(path)[/\A\d+/] }
+  applied = begin
+    ActiveRecord::Base.connection.select_values("SELECT version FROM schema_migrations").map(&:to_s)
+  rescue
+    [] # No database, or no schema_migrations in it yet
+  end
+
+  if (on_disk - applied).any? || !File.exist?(schema_file)
+    puts "Recreating the #{db_config.adapter} test database..."
+    # db:migrate seeds an empty database from the schema dump before applying
+    # anything, so the outdated dump has to go first - otherwise it recreates
+    # exactly the tables the migrations are about to create.
+    File.delete(schema_file) if File.exist?(schema_file)
+    # bin/rails cannot drop a database this process still holds a connection to.
+    ActiveRecord::Base.connection_handler.clear_all_connections!
     Dir.chdir(dummy_root) do
-      system("bin/rails", "db:prepare") || abort("Failed to prepare database")
+      # One bin/rails per task, deliberately. Asking a single process to drop,
+      # recreate and migrate carries its column cache across the drop, and the
+      # schema it then dumps silently loses column defaults.
+      system("bin/rails", "db:drop") || abort("Failed to drop the test database")
+      system("bin/rails", "db:prepare") || abort("Failed to recreate the test database")
     end
   end
 end
@@ -44,6 +67,21 @@ if ActiveSupport::TestCase.respond_to?(:fixture_paths=)
   ActiveSupport::TestCase.fixtures :all
 end
 
+# Rails 8.1 clears Active Record's connections before forking the parallel test
+# workers. 7.2 and 8.0 fork with them still open, and a libpq connection
+# inherited across the fork takes the child down with a segfault in
+# PG::Connection#connect_start as it opens its own - a race, so it only bites
+# some of the time, which makes it maddening rather than merely broken. Where
+# upstream has no pre-fork hook to register on, wrap the fork point itself.
+unless ActiveSupport::Testing::Parallelization.respond_to?(:before_fork_hook)
+  ActiveSupport::Testing::Parallelization.prepend(Module.new do
+    def start
+      ActiveRecord::Base.connection_handler.clear_all_connections!
+      super
+    end
+  end)
+end
+
 # Test helper methods
 class ActiveSupport::TestCase
   # Run tests in parallel with specified workers
@@ -55,6 +93,8 @@ class ActiveSupport::TestCase
   setup do
     GenevaDrive::StepExecution.reset_metadata_column_cache!
     GenevaDrive::StepExecution.reset_resumable_columns_cache!
+    GenevaDrive::StepExecution.reset_signal_columns_cache!
+    GenevaDrive::Signal.reset_table_available_cache!
     GenevaDrive::Workflow.reset_metadata_column_cache!
   end
 

data/test/workflow/removed_step_test.rb

@@ -0,0 +1,294 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class RemovedStepTest < ActiveSupport::TestCase
+  include GenevaDrive::TestHelpers
+
+  # The shape a workflow has after `capture_payment` was taken out of the
+  # middle of it and a gravestone left behind.
+  class PaymentWorkflow < GenevaDrive::Workflow
+    step :authorize do
+      Thread.current[:ran] << :authorize
+    end
+
+    removed_step :capture_payment
+
+    step :send_receipt do
+      Thread.current[:ran] << :send_receipt
+    end
+  end
+
+  # A gravestone in first position, so nothing runnable precedes it.
+  class LeadingRemovalWorkflow < GenevaDrive::Workflow
+    removed_step :old_preflight
+
+    step :only_step do
+      Thread.current[:ran] << :only_step
+    end
+  end
+
+  # A gravestone in last position, so nothing runnable follows it.
+  class TrailingRemovalWorkflow < GenevaDrive::Workflow
+    step :only_step do
+      Thread.current[:ran] << :only_step
+    end
+
+    removed_step :old_teardown
+  end
+
+  # Every step removed - there is nothing left to run at all.
+  class FullyRemovedWorkflow < GenevaDrive::Workflow
+    removed_step :gone_one
+    removed_step :gone_two
+  end
+
+  # A step deleted outright, with no gravestone. This is the situation
+  # removed_step exists to prevent, kept here to prove the default is
+  # unchanged: the workflow still pauses rather than guessing.
+  class UndeclaredRemovalWorkflow < GenevaDrive::Workflow
+    step :step_one do
+      Thread.current[:ran] << :step_one
+    end
+
+    step :step_two do
+      Thread.current[:ran] << :step_two
+    end
+  end
+
+  setup do
+    @user = create_user
+    Thread.current[:ran] = []
+  end
+
+  teardown do
+    Thread.current[:ran] = nil
+  end
+
+  # ===========================================
+  # The declaration itself
+  # ===========================================
+
+  test "removed_step registers a gravestone that reports itself as removed" do
+    step_def = PaymentWorkflow.steps.named("capture_payment")
+
+    assert_not_nil step_def
+    assert_predicate step_def, :removed?
+    assert_equal "capture_payment", step_def.name
+  end
+
+  test "an ordinary step is not removed" do
+    assert_not_predicate PaymentWorkflow.steps.named("authorize"), :removed?
+  end
+
+  test "a removed step keeps its name resolvable" do
+    assert PaymentWorkflow.steps.key?("capture_payment")
+  end
+
+  test "a removed step holds its position in the declared order" do
+    assert_equal %w[authorize capture_payment send_receipt], PaymentWorkflow.steps.map(&:name)
+  end
+
+  test "runnable omits removed steps and removed lists them" do
+    assert_equal %w[authorize send_receipt], PaymentWorkflow.steps.runnable.map(&:name)
+    assert_equal %w[capture_payment], PaymentWorkflow.steps.removed.map(&:name)
+  end
+
+  test "removed_step refuses a name already taken by a step" do
+    error = assert_raises(GenevaDrive::StepConfigurationError) do
+      Class.new(GenevaDrive::Workflow) do
+        def self.name = "DoubleDeclarationWorkflow"
+
+        step :thing do
+          # no-op
+        end
+
+        removed_step :thing
+      end
+    end
+
+    assert_match(/already defined/, error.message)
+  end
+
+  test "removed_step requires a name" do
+    assert_raises(ArgumentError) do
+      Class.new(GenevaDrive::Workflow) do
+        removed_step nil
+      end
+    end
+  end
+
+  test "before_step can still reference a removed step" do
+    klass = Class.new(GenevaDrive::Workflow) do
+      def self.name = "PositionedAroundRemovalWorkflow"
+
+      removed_step :gone
+
+      step :inserted, before_step: :gone do
+        # no-op
+      end
+    end
+
+    assert_equal %w[inserted gone], klass.steps.map(&:name)
+  end
+
+  # ===========================================
+  # Navigation never lands on a gravestone
+  # ===========================================
+
+  test "next_after steps over a removed step" do
+    assert_equal "send_receipt", PaymentWorkflow.steps.next_after("authorize").name
+  end
+
+  test "next_after from a removed step returns the next runnable one" do
+    assert_equal "send_receipt", PaymentWorkflow.steps.next_after("capture_payment").name
+  end
+
+  test "next_after returns nil when only removed steps remain" do
+    assert_nil TrailingRemovalWorkflow.steps.next_after("only_step")
+  end
+
+  test "next_after from the beginning skips a leading removed step" do
+    assert_equal "only_step", LeadingRemovalWorkflow.steps.next_after(nil).name
+  end
+
+  test "next_after still returns nil for a name the collection never had" do
+    assert_nil PaymentWorkflow.steps.next_after("never_existed")
+  end
+
+  test "previous_before steps over a removed step" do
+    assert_equal "authorize", PaymentWorkflow.steps.previous_before("send_receipt").name
+  end
+
+  test "previous_before returns nil when only removed steps precede" do
+    assert_nil LeadingRemovalWorkflow.steps.previous_before("only_step")
+  end
+
+  # ===========================================
+  # Scheduling never spools a removed step
+  # ===========================================
+
+  test "a new workflow runs straight past a removed middle step" do
+    workflow = PaymentWorkflow.create!(hero: @user)
+    speedrun_workflow(workflow)
+
+    assert_equal %i[authorize send_receipt], Thread.current[:ran]
+    assert_workflow_state(workflow, :finished)
+    assert_empty workflow.step_executions.where(step_name: "capture_payment")
+  end
+
+  test "a new workflow starts at the first runnable step, not a leading gravestone" do
+    workflow = LeadingRemovalWorkflow.create!(hero: @user)
+
+    assert_equal "only_step", workflow.next_step_name
+
+    speedrun_workflow(workflow)
+
+    assert_equal %i[only_step], Thread.current[:ran]
+    assert_workflow_state(workflow, :finished)
+  end
+
+  test "a trailing gravestone finishes the workflow instead of being scheduled" do
+    workflow = TrailingRemovalWorkflow.create!(hero: @user)
+    speedrun_workflow(workflow)
+
+    assert_workflow_state(workflow, :finished)
+    assert_empty workflow.step_executions.where(step_name: "old_teardown")
+  end
+
+  test "a workflow whose every step was removed finishes immediately" do
+    workflow = FullyRemovedWorkflow.create!(hero: @user)
+
+    assert_workflow_state(workflow, :finished)
+    assert_empty workflow.step_executions
+  end
+
+  # ===========================================
+  # Executions scheduled before the removal shipped
+  # ===========================================
+
+  test "an execution scheduled before the removal is skipped and the workflow continues" do
+    workflow = PaymentWorkflow.create!(hero: @user)
+
+    # Stand in for a rolling deploy: this row was written by the old code,
+    # which still had capture_payment as a real step.
+    execution = reschedule_at(workflow, "capture_payment")
+
+    execution.execute!
+
+    assert_equal "skipped", execution.reload.state
+    assert_equal "skipped", execution.outcome
+
+    speedrun_workflow(workflow)
+
+    assert_equal %i[send_receipt], Thread.current[:ran]
+    assert_workflow_state(workflow, :finished)
+  end
+
+  test "an execution for a trailing removed step finishes the workflow" do
+    workflow = TrailingRemovalWorkflow.create!(hero: @user)
+    execution = reschedule_at(workflow, "old_teardown")
+
+    execution.execute!
+
+    assert_equal "skipped", execution.reload.state
+    assert_workflow_state(workflow, :finished)
+  end
+
+  test "a step deleted without a gravestone still pauses the workflow" do
+    workflow = UndeclaredRemovalWorkflow.create!(hero: @user)
+    execution = reschedule_at(workflow, "deleted_without_a_trace")
+
+    assert_raises(GenevaDrive::StepNotDefinedError) { execution.execute! }
+
+    assert_workflow_state(workflow, :paused)
+  end
+
+  # ===========================================
+  # Knowing when the gravestone can go
+  # ===========================================
+
+  test "removed_steps_in_flight counts rows still pointing at a removed step" do
+    workflow = PaymentWorkflow.create!(hero: @user)
+    reschedule_at(workflow, "capture_payment")
+
+    assert_equal({"capture_payment" => 1}, PaymentWorkflow.removed_steps_in_flight)
+  end
+
+  test "removed_steps_in_flight ignores workflows that will never run again" do
+    workflow = PaymentWorkflow.create!(hero: @user)
+    reschedule_at(workflow, "capture_payment")
+    workflow.update!(state: "canceled")
+
+    assert_empty PaymentWorkflow.removed_steps_in_flight
+  end
+
+  test "removed_steps_in_flight is empty once nothing references the name" do
+    PaymentWorkflow.create!(hero: @user)
+
+    assert_empty PaymentWorkflow.removed_steps_in_flight
+  end
+
+  test "removed_steps_in_flight is empty for a workflow with no removals" do
+    assert_empty UndeclaredRemovalWorkflow.removed_steps_in_flight
+  end
+
+  private
+
+  # Replaces whatever is scheduled with an execution for the given step name,
+  # the way a row written by a previous deploy would look.
+  def reschedule_at(workflow, step_name)
+    workflow.step_executions.where(state: %w[scheduled waiting]).update_all(
+      state: "canceled",
+      outcome: "canceled",
+      canceled_at: Time.current
+    )
+    execution = workflow.step_executions.create!(
+      step_name: step_name,
+      state: "scheduled",
+      scheduled_for: Time.current
+    )
+    workflow.update!(next_step_name: step_name, current_step_name: nil)
+    execution
+  end
+end