@axiom-lattice/pg-stores 3.3.12 → 3.4.0

package/src/stores/PostgreSQLAgentRunLedger.ts

@@ -0,0 +1,1210 @@
+/**
+ * PostgreSQL Agent Run Ledger
+ *
+ * Durable run lifecycle and task queue. `source='run_api'` rows are claimed by
+ * the run-task worker; `source='legacy'` rows preserve the existing
+ * observational ledger/message-queue behavior.
+ */
+
+import { Pool, type PoolConfig } from "pg";
+import {
+  AgentRunAlreadyActiveError,
+  AgentRunDuplicateMessageIdError,
+  AgentRunInputCommandConflictError,
+  AgentRunInvalidTaskIdError,
+  AgentRunTenantQuotaExceededError,
+  assertQueueSources,
+  extractAgentRunInputMessageIds,
+  normalizeAgentRunMessageIds,
+  type AgentRunCounts,
+  type AgentRunExpiredOutcome,
+  type AgentRunJsonObject,
+  type AgentRunRecord,
+  type AgentRunRollbackResult,
+  type AgentRunSource,
+  type AgentRunStatus,
+  type BeginAgentRunInput,
+  type ClaimAgentRunInput,
+  type CreatePendingAgentRunInput,
+  type DeleteAgentRunExpected,
+  type IAgentRunLedger,
+  type ListPendingRunsOptions,
+  type ReapedAgentRun,
+  type RollbackFailureResult,
+  type SettleExpiredRunOptions,
+} from "@axiom-lattice/core";
+import { MigrationManager } from "../migrations/migration";
+import { createAgentRunLedgerTable } from "../migrations/agent_run_ledger_migration";
+import { createAgentRunTaskQueueMigration } from "../migrations/agent_run_task_queue_migration";
+import { createAgentRunCapabilityRevisionMigration } from "../migrations/agent_run_capability_revision_migration";
+import { createAgentRunBaseCheckpointCapturedMigration } from "../migrations/agent_run_base_checkpoint_captured_migration";
+import { createAgentRunRollbackFailureMigration } from "../migrations/agent_run_rollback_failure_migration";
+import { createAgentRunRetentionMigration } from "../migrations/agent_run_retention_migration";
+import { agentRunRunningIndexSourceMigration } from "../migrations/agent_run_running_index_source_migration";
+import { createAgentRunProjectTaskMigration } from "../migrations/agent_run_project_task_migration";
+import { agentRunProjectTaskRunningIndexMigration } from "../migrations/agent_run_project_task_running_index_migration";
+
+/** Options accepted by {@link PostgreSQLAgentRunLedger}. */
+export interface PostgreSQLAgentRunLedgerOptions {
+  /** Externally managed pool. When supplied, migrations are not run here. */
+  pool?: Pool;
+  /** Connection string or pool config used to create a pool. */
+  poolConfig?: string | PoolConfig;
+  /** Set to `false` to skip running this store's migration automatically. */
+  autoMigrate?: boolean;
+}
+
+/** Raw `lattice_agent_runs` row shape. */
+interface AgentRunRow {
+  run_id: string;
+  tenant_id: string;
+  workspace_id: string | null;
+  project_id: string | null;
+  task_id: string | null;
+  assistant_id: string;
+  thread_id: string;
+  status: string;
+  source: string | null;
+  owner: string | null;
+  lease_expires_at: Date | null;
+  input_message_ids: string[];
+  attempts: number;
+  error: string | null;
+  cancel_requested: boolean;
+  rollback_requested: boolean;
+  rollback_attempts: number;
+  rollback_error: string | null;
+  input: unknown;
+  command: unknown;
+  config: unknown;
+  metadata: unknown;
+  base_checkpoint_id: string | null;
+  base_checkpoint_captured: boolean;
+  capability_runtime_revision: string | null;
+  available_at: Date;
+  created_at: Date;
+  updated_at: Date;
+  started_at: Date | null;
+  finished_at: Date | null;
+}
+
+const DEFAULT_LIST_LIMIT = 50;
+const MAX_LIST_LIMIT = 200;
+
+const CLAIM_RUN_SQL = `UPDATE lattice_agent_runs AS r
+    SET status = 'running',
+        owner = $2,
+        lease_expires_at = now() + ($3::bigint || ' milliseconds')::interval,
+        started_at = now(),
+        attempts = r.attempts + 1,
+        input_message_ids = COALESCE($4::text[], r.input_message_ids),
+        updated_at = now()
+  WHERE r.run_id = $1
+    AND r.source = ANY($5::text[])
+    AND r.status = 'pending'
+    AND r.available_at <= now()
+    AND NOT EXISTS (
+      SELECT 1 FROM lattice_agent_runs AS o
+       WHERE o.tenant_id = r.tenant_id
+         AND o.thread_id = r.thread_id
+         AND o.source IN ('run_api', 'project_task')
+         AND o.status = 'running'
+    )
+    AND NOT EXISTS (
+      SELECT 1 FROM lattice_agent_runs AS b
+       WHERE b.tenant_id = r.tenant_id
+         AND b.thread_id = r.thread_id
+         AND b.source = 'run_api'
+         AND b.rollback_requested = TRUE
+    )
+  RETURNING r.*`;
+
+/**
+ * PostgreSQL implementation of {@link IAgentRunLedger}.
+ *
+ * @remarks
+ * - Run rows are the durable queue for `run_api` and `project_task` work.
+ *   Claim is an atomic CAS guarded by the same-thread running rule.
+ * - Legacy `beginRun` keeps the historical reconcile-then-insert behavior and
+ *   writes `source='legacy'`.
+ * - `reapExpired` reports expired run-queue rows without mutating them;
+ *   `reapLegacyExpired` preserves the supervisor's legacy settle behavior.
+ */
+export class PostgreSQLAgentRunLedger implements IAgentRunLedger {
+  private pool: Pool;
+  private ownsPool = true;
+  private initialized = false;
+  private initPromise: Promise<void> | null = null;
+  private migrationManager!: MigrationManager;
+
+  /**
+   * @param options An externally managed pool, or a pool config/connection URL.
+   */
+  constructor(options: PostgreSQLAgentRunLedgerOptions) {
+    if (options.pool) {
+      this.pool = options.pool;
+      this.ownsPool = false;
+      this.initialized = true;
+      return;
+    }
+
+    if (typeof options.poolConfig === "string") {
+      this.pool = new Pool({ connectionString: options.poolConfig });
+    } else if (options.poolConfig) {
+      this.pool = new Pool(options.poolConfig as PoolConfig);
+    } else {
+      throw new Error("Either pool or poolConfig must be provided");
+    }
+
+    this.migrationManager = new MigrationManager(this.pool);
+    this.migrationManager.register(createAgentRunLedgerTable);
+    this.migrationManager.register(createAgentRunTaskQueueMigration);
+    this.migrationManager.register(createAgentRunCapabilityRevisionMigration);
+    this.migrationManager.register(createAgentRunBaseCheckpointCapturedMigration);
+    this.migrationManager.register(createAgentRunRollbackFailureMigration);
+    this.migrationManager.register(createAgentRunRetentionMigration);
+    this.migrationManager.register(agentRunRunningIndexSourceMigration);
+    this.migrationManager.register(createAgentRunProjectTaskMigration);
+    this.migrationManager.register(agentRunProjectTaskRunningIndexMigration);
+
+    if (options.autoMigrate !== false) {
+      this.initialize().catch((error) => {
+        console.error("Failed to initialize PostgreSQLAgentRunLedger:", error);
+        throw error;
+      });
+    }
+  }
+
+  /** Runs pending migrations once. */
+  async initialize(): Promise<void> {
+    if (this.initialized) return;
+    if (this.initPromise) return this.initPromise;
+
+    this.initPromise = (async () => {
+      try {
+        await this.migrationManager.migrate();
+        this.initialized = true;
+      } finally {
+        this.initPromise = null;
+      }
+    })();
+    return this.initPromise;
+  }
+
+  /** Ends the pool when this store owns it. */
+  async dispose(): Promise<void> {
+    if (this.ownsPool && this.pool) {
+      await this.pool.end();
+    }
+  }
+
+  /**
+   * Creates a running legacy run, reconciling any existing running row first.
+   *
+   * @remarks The caller holds the `thread-run` lease, so a surviving `running`
+   * row is stale (its owner can no longer hold the lease) even when its ledger
+   * lease has not expired yet.
+   */
+  async beginRun(input: BeginAgentRunInput): Promise<AgentRunRecord> {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      await client.query(
+        `UPDATE lattice_agent_runs
+            SET status = 'interrupted',
+                error = COALESCE(error, 'superseded by a new run on the same thread'),
+                finished_at = now(),
+                updated_at = now()
+          WHERE tenant_id = $1 AND thread_id = $2
+            AND status = 'running'`,
+        [input.tenantId, input.threadId],
+      );
+      const result = await client.query<AgentRunRow>(
+        `INSERT INTO lattice_agent_runs
+           (run_id, tenant_id, workspace_id, project_id, assistant_id, thread_id,
+            status, source, owner, lease_expires_at, input_message_ids, attempts,
+            started_at)
+         VALUES ($1, $2, $3, $4, $5, $6, 'running', 'legacy', $7,
+                 now() + ($8::bigint || ' milliseconds')::interval, $9, 1, now())
+         RETURNING *`,
+        [
+          input.runId,
+          input.tenantId,
+          input.workspaceId ?? null,
+          input.projectId ?? null,
+          input.assistantId,
+          input.threadId,
+          input.owner,
+          input.leaseMs,
+          input.inputMessageIds,
+        ],
+      );
+      await client.query("COMMIT");
+      return rowToRecord(result.rows[0]);
+    } catch (error) {
+      await client.query("ROLLBACK");
+      if (isActiveRunViolation(error)) {
+        throw new AgentRunAlreadyActiveError();
+      }
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Creates an unowned pending run-queue row. */
+  async createPendingRun(
+    input: CreatePendingAgentRunInput,
+  ): Promise<AgentRunRecord> {
+    const source = input.source ?? "run_api";
+    if (source === "project_task") {
+      if (input.taskId === undefined || input.taskId.trim().length === 0) {
+        throw new AgentRunInvalidTaskIdError(
+          "A project_task run requires a non-empty taskId",
+        );
+      }
+    } else if (input.taskId !== undefined) {
+      throw new AgentRunInvalidTaskIdError(
+        `A ${source} run cannot carry a taskId`,
+      );
+    }
+    if (input.input != null && input.command != null) {
+      throw new AgentRunInputCommandConflictError();
+    }
+    const inputMessageIds = input.inputMessageIds
+      ? normalizeAgentRunMessageIds(input.inputMessageIds)
+      : extractAgentRunInputMessageIds(input.input);
+    const availableAt = input.availableAt ?? Date.now();
+    const limit = input.maxActiveRunsForTenant;
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      // Quota admission must serialize every create for one tenant, not just
+      // one thread, or concurrent creates could each observe spare capacity.
+      if (limit !== undefined && limit > 0) {
+        await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [
+          input.tenantId,
+        ]);
+      }
+      // Serialize duplicate-id checks and inserts per thread. The lock is
+      // transaction-scoped and does not block unrelated threads.
+      await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [
+        `agent-run:${input.tenantId}:${input.threadId}`,
+      ]);
+
+      if (inputMessageIds.length > 0) {
+        const duplicate = await client.query<{ input_message_ids: string[] }>(
+          `SELECT input_message_ids
+             FROM lattice_agent_runs
+            WHERE tenant_id = $1 AND thread_id = $2
+              AND status IN ('pending', 'running')
+              AND input_message_ids && $3::text[]
+            LIMIT 1`,
+          [input.tenantId, input.threadId, inputMessageIds],
+        );
+        if ((duplicate.rowCount ?? 0) > 0) {
+          const existing = duplicate.rows[0].input_message_ids;
+          const messageId =
+            existing.find((id) => inputMessageIds.includes(id)) ??
+            inputMessageIds[0];
+          throw new AgentRunDuplicateMessageIdError(messageId);
+        }
+      }
+
+      if (limit !== undefined && limit > 0) {
+        const active = await client.query<{ count: string }>(
+          `SELECT count(*)::text AS count
+             FROM lattice_agent_runs
+            WHERE tenant_id = $1
+              AND source = 'run_api'
+              AND status IN ('pending', 'running')`,
+          [input.tenantId],
+        );
+        if (Number(active.rows[0].count) >= limit) {
+          throw new AgentRunTenantQuotaExceededError(input.tenantId, limit);
+        }
+      }
+
+      const result = await client.query<AgentRunRow>(
+        `INSERT INTO lattice_agent_runs
+           (run_id, tenant_id, workspace_id, project_id, assistant_id, thread_id,
+            status, source, owner, lease_expires_at, input_message_ids, attempts,
+            input, command, config, metadata, capability_runtime_revision,
+            available_at, created_at, updated_at, task_id)
+         VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7, NULL, NULL, $8, 0,
+                 $9::jsonb, $10::jsonb, $11::jsonb, $12::jsonb, $13,
+                 to_timestamp($14::double precision / 1000.0),
+                 date_trunc('milliseconds', now()),
+                 date_trunc('milliseconds', now()), $15)
+         RETURNING *`,
+        [
+          input.runId,
+          input.tenantId,
+          input.workspaceId ?? null,
+          input.projectId ?? null,
+          input.assistantId,
+          input.threadId,
+          source,
+          inputMessageIds,
+          jsonParam(input.input),
+          jsonParam(input.command),
+          jsonParam(input.config),
+          jsonParam(input.metadata),
+          input.capabilityRuntimeRevision ?? null,
+          availableAt,
+          input.taskId ?? null,
+        ],
+      );
+      await client.query("COMMIT");
+      return rowToRecord(result.rows[0]);
+    } catch (error) {
+      await client.query("ROLLBACK");
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Lists claimable run-queue rows, oldest available first. */
+  async listPendingRuns(
+    options?: ListPendingRunsOptions,
+  ): Promise<AgentRunRecord[]> {
+    const limit = clampLimit(options?.limit);
+    const after = options?.after;
+    const fairTenants = options?.fairTenants === true;
+    if (fairTenants && after !== undefined) {
+      throw new RangeError(
+        "listPendingRuns fairTenants does not support the after cursor",
+      );
+    }
+    const maxRunning = positiveLimit(options?.maxRunningPerTenant);
+    const sources: AgentRunSource[] =
+      options?.sources === undefined ? ["run_api"] : [...options.sources];
+    assertQueueSources(sources);
+    const params: unknown[] = [limit];
+    let cursorClause = "";
+    if (after) {
+      // createPendingRun stores millisecond-precision timestamps, so cursor
+      // values round-trip exactly through AgentRunRecord's numeric fields.
+      params.push(
+        new Date(after.availableAt),
+        new Date(after.createdAt),
+        after.runId,
+      );
+      cursorClause = `
+          AND (r.available_at, r.created_at, r.run_id)
+              > ($2::timestamptz, $3::timestamptz, $4::uuid)`;
+    }
+    params.push(maxRunning ?? null);
+    const capIndex = params.length;
+    params.push(sources);
+    const sourcesIndex = params.length;
+    // Exclude a tenant already at its execution cap so the scan can reach
+    // other tenants. This is best-effort fairness, not round-robin scheduling.
+    const capClause = `
+          AND ($${capIndex}::bigint IS NULL OR (
+            SELECT count(*) FROM lattice_agent_runs AS c
+             WHERE c.tenant_id = r.tenant_id
+               AND c.source = ANY($${sourcesIndex}::text[])
+               AND c.status = 'running'
+          ) < $${capIndex}::bigint)`;
+    const filterClause = `WHERE r.source = ANY($${sourcesIndex}::text[]) AND r.status = 'pending'
+          AND r.available_at <= now()
+          AND NOT EXISTS (
+            SELECT 1 FROM lattice_agent_runs AS o
+             WHERE o.tenant_id = r.tenant_id
+               AND o.thread_id = r.thread_id
+               AND o.source IN ('run_api', 'project_task')
+               AND o.status = 'running'
+          )
+          AND NOT EXISTS (
+            SELECT 1 FROM lattice_agent_runs AS b
+             WHERE b.tenant_id = r.tenant_id
+               AND b.thread_id = r.thread_id
+               AND b.source = 'run_api'
+               AND b.rollback_requested = TRUE
+          )${capClause}${cursorClause}`;
+    if (fairTenants) {
+      // Rank each tenant's pending rows, then interleave by rank. This is
+      // best-effort fair ordering at scan time: the first row of every tenant
+      // precedes the second row of any tenant, but a later scan can still see
+      // a different mix as work is claimed.
+      const ranked = await this.pool.query<AgentRunRow>(
+        `WITH ranked AS (
+          SELECT r.*,
+                 row_number() OVER (
+                   PARTITION BY r.tenant_id
+                   ORDER BY r.available_at ASC, r.created_at ASC, r.run_id ASC
+                 ) AS tenant_rank
+            FROM lattice_agent_runs AS r
+            ${filterClause}
+        )
+        SELECT * FROM ranked
+        ORDER BY tenant_rank ASC, available_at ASC, created_at ASC,
+                 run_id ASC, tenant_id ASC
+        LIMIT $1`,
+        params,
+      );
+      return ranked.rows.map(rowToRecord);
+    }
+    const result = await this.pool.query<AgentRunRow>(
+      `SELECT r.* FROM lattice_agent_runs AS r
+        ${filterClause}
+        ORDER BY r.available_at ASC, r.created_at ASC, r.run_id ASC
+        LIMIT $1`,
+      params,
+    );
+    return result.rows.map(rowToRecord);
+  }
+
+  /** Atomically claims a pending run-queue row. */
+  async claimRun(
+    input: ClaimAgentRunInput,
+  ): Promise<AgentRunRecord | undefined> {
+    const maxRunning = positiveLimit(input.maxRunningPerTenant);
+    const sources: AgentRunSource[] =
+      input.sources === undefined ? ["run_api"] : [...input.sources];
+    assertQueueSources(sources);
+    if (maxRunning !== undefined) {
+      return this.claimRunWithTenantCap(input, maxRunning, sources);
+    }
+    try {
+      const result = await this.pool.query<AgentRunRow>(
+        CLAIM_RUN_SQL,
+        [
+          input.runId,
+          input.owner,
+          input.leaseMs,
+          input.inputMessageIds ?? null,
+          sources,
+        ],
+      );
+      return result.rowCount === 0 ? undefined : rowToRecord(result.rows[0]);
+    } catch (error) {
+      // The partial unique index is the safety net for a concurrent claim.
+      if (isActiveRunViolation(error)) return undefined;
+      throw error;
+    }
+  }
+
+  /**
+   * Claims under a per-tenant running cap.
+   *
+   * @remarks Serializes claims for one tenant with a transaction-scoped
+   * advisory lock, counts running rows, then runs the same claim statement.
+   * The lock is shared with admission creates so cap checks cannot race.
+   */
+  private async claimRunWithTenantCap(
+    input: ClaimAgentRunInput,
+    maxRunning: number,
+    sources: AgentRunSource[],
+  ): Promise<AgentRunRecord | undefined> {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      const candidate = await client.query<{ tenant_id: string }>(
+        `SELECT tenant_id FROM lattice_agent_runs
+          WHERE run_id = $1 AND source = ANY($2::text[]) AND status = 'pending'`,
+        [input.runId, sources],
+      );
+      if (candidate.rowCount === 0) {
+        await client.query("COMMIT");
+        return undefined;
+      }
+      const tenantId = candidate.rows[0].tenant_id;
+      await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [
+        tenantId,
+      ]);
+      const running = await client.query<{ count: string }>(
+        `SELECT count(*)::text AS count
+           FROM lattice_agent_runs
+          WHERE tenant_id = $1
+            AND source = ANY($2::text[])
+            AND status = 'running'`,
+        [tenantId, sources],
+      );
+      if (Number(running.rows[0].count) >= maxRunning) {
+        await client.query("COMMIT");
+        return undefined;
+      }
+      const result = await client.query<AgentRunRow>(
+        CLAIM_RUN_SQL,
+        [
+          input.runId,
+          input.owner,
+          input.leaseMs,
+          input.inputMessageIds ?? null,
+          sources,
+        ],
+      );
+      await client.query("COMMIT");
+      return result.rowCount === 0 ? undefined : rowToRecord(result.rows[0]);
+    } catch (error) {
+      await client.query("ROLLBACK");
+      if (isActiveRunViolation(error)) return undefined;
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Captures the checkpoint a running run-queue row started from. */
+  async setBaseCheckpoint(
+    tenantId: string,
+    runId: string,
+    owner: string,
+    baseCheckpointId: string | null,
+  ): Promise<boolean> {
+    const result = await this.pool.query(
+      `UPDATE lattice_agent_runs
+          SET base_checkpoint_id = $4,
+              base_checkpoint_captured = TRUE,
+              updated_at = now()
+        WHERE tenant_id = $1 AND run_id = $2 AND owner = $3
+          AND status = 'running'
+          AND source IN ('run_api', 'project_task')
+        RETURNING run_id`,
+      [tenantId, runId, owner, baseCheckpointId],
+    );
+    return (result.rowCount ?? 0) > 0;
+  }
+
+  /** Records the resolved capability runtime revision for a running run. */
+  async setCapabilityRevision(
+    tenantId: string,
+    runId: string,
+    owner: string,
+    revision: string | null,
+  ): Promise<boolean> {
+    const result = await this.pool.query(
+      `UPDATE lattice_agent_runs
+          SET capability_runtime_revision = $4, updated_at = now()
+        WHERE tenant_id = $1 AND run_id = $2 AND owner = $3
+          AND status = 'running'
+          AND source IN ('run_api', 'project_task')
+        RETURNING run_id`,
+      [tenantId, runId, owner, revision],
+    );
+    return (result.rowCount ?? 0) > 0;
+  }
+
+  /** Deletes a run row immediately, optionally guarded by status/owner. */
+  async deleteRun(
+    tenantId: string,
+    runId: string,
+    expected?: DeleteAgentRunExpected,
+  ): Promise<boolean> {
+    const params: unknown[] = [tenantId, runId];
+    let predicate = "tenant_id = $1 AND run_id = $2";
+    if (expected?.status !== undefined) {
+      params.push(expected.status);
+      predicate += ` AND status = $${params.length}`;
+    }
+    if (expected?.owner !== undefined) {
+      params.push(expected.owner);
+      predicate += ` AND owner = $${params.length}`;
+    }
+    const result = await this.pool.query(
+      `DELETE FROM lattice_agent_runs WHERE ${predicate}`,
+      params,
+    );
+    return (result.rowCount ?? 0) > 0;
+  }
+
+  /** Deletes terminal rows finished before `cutoffMs`, oldest first. */
+  async deleteTerminalBefore(
+    cutoffMs: number,
+    limit: number,
+  ): Promise<number> {
+    if (!Number.isFinite(cutoffMs)) return 0;
+    const max = Number.isFinite(limit) ? Math.floor(limit) : 0;
+    if (max <= 0) return 0;
+
+    const result = await this.pool.query(
+      `DELETE FROM lattice_agent_runs
+        WHERE run_id IN (
+          SELECT run_id FROM lattice_agent_runs
+           WHERE status IN ('success', 'error', 'interrupted', 'cancelled', 'timeout')
+             AND finished_at < to_timestamp($1::double precision / 1000.0)
+           ORDER BY finished_at ASC
+           LIMIT $2
+        )`,
+      [cutoffMs, max],
+    );
+    return result.rowCount ?? 0;
+  }
+
+  /** Extends a running run's lease when `owner` still holds it. */
+  async heartbeatRun(
+    runId: string,
+    owner: string,
+    leaseMs: number,
+  ): Promise<boolean> {
+    const result = await this.pool.query(
+      `UPDATE lattice_agent_runs
+          SET lease_expires_at = now() + ($3::bigint || ' milliseconds')::interval,
+              updated_at = now()
+        WHERE run_id = $1 AND owner = $2 AND status = 'running'
+        RETURNING run_id`,
+      [runId, owner, leaseMs],
+    );
+    return (result.rowCount ?? 0) > 0;
+  }
+
+  /** Settles a running run when `owner` still holds it. */
+  async settleRun(
+    runId: string,
+    owner: string,
+    status: Exclude<AgentRunStatus, "pending" | "running">,
+    error?: string,
+  ): Promise<boolean> {
+    const result = await this.pool.query(
+      `UPDATE lattice_agent_runs
+          SET status = $3, error = $4, finished_at = now(), updated_at = now()
+        WHERE run_id = $1 AND owner = $2 AND status = 'running'
+        RETURNING run_id`,
+      [runId, owner, status, error ?? null],
+    );
+    return (result.rowCount ?? 0) > 0;
+  }
+
+  /** Settles a running run only when no rollback is pending. */
+  async settleRunUnlessRollbackRequested(
+    runId: string,
+    owner: string,
+    status: Exclude<AgentRunStatus, "pending" | "running">,
+    error?: string,
+  ): Promise<boolean> {
+    const result = await this.pool.query(
+      `UPDATE lattice_agent_runs
+          SET status = $3, error = $4, finished_at = now(), updated_at = now()
+        WHERE run_id = $1 AND owner = $2 AND status = 'running'
+          AND rollback_requested = FALSE
+        RETURNING run_id`,
+      [runId, owner, status, error ?? null],
+    );
+    return (result.rowCount ?? 0) > 0;
+  }
+
+  /** Terminates an uncaptured rollback run with a repair signal. */
+  async settleRollbackFailure(
+    tenantId: string,
+    runId: string,
+    error: string,
+  ): Promise<boolean> {
+    const result = await this.pool.query(
+      `UPDATE lattice_agent_runs
+          SET status = 'error',
+              error = $3,
+              rollback_requested = FALSE,
+              rollback_error = $3,
+              finished_at = now(),
+              updated_at = now()
+        WHERE tenant_id = $1 AND run_id = $2
+          AND source = 'run_api'
+          AND rollback_requested = TRUE
+          AND base_checkpoint_captured = FALSE
+          AND status IN ('running', 'interrupted')
+        RETURNING run_id`,
+      [tenantId, runId, error],
+    );
+    return (result.rowCount ?? 0) > 0;
+  }
+
+  /** Records one bounded rollback failure and optionally converges. */
+  async recordRollbackFailure(
+    tenantId: string,
+    runId: string,
+    error: string,
+    maxAttempts: number,
+  ): Promise<RollbackFailureResult | undefined> {
+    const result = await this.pool.query<{
+      rollback_attempts: number;
+      status: string;
+    }>(
+      `UPDATE lattice_agent_runs
+          SET rollback_attempts = rollback_attempts + 1,
+              rollback_error = $3,
+              status = CASE
+                WHEN rollback_attempts + 1 >= $4 THEN 'error'
+                ELSE status
+              END,
+              error = CASE
+                WHEN rollback_attempts + 1 >= $4 THEN $3
+                ELSE error
+              END,
+              rollback_requested = CASE
+                WHEN rollback_attempts + 1 >= $4 THEN FALSE
+                ELSE rollback_requested
+              END,
+              finished_at = CASE
+                WHEN rollback_attempts + 1 >= $4 THEN now()
+                ELSE finished_at
+              END,
+              updated_at = now()
+        WHERE tenant_id = $1
+          AND run_id = $2
+          AND source = 'run_api'
+          AND rollback_requested = TRUE
+          AND status IN ('running', 'interrupted')
+        RETURNING rollback_attempts, status`,
+      [tenantId, runId, error, maxAttempts],
+    );
+    if (result.rowCount === 0) return undefined;
+    const row = result.rows[0];
+    return {
+      attempts: row.rollback_attempts,
+      terminal: row.status === "error",
+    };
+  }
+
+  /** Reads a run within a tenant. */
+  async getRun(
+    tenantId: string,
+    runId: string,
+  ): Promise<AgentRunRecord | undefined> {
+    const result = await this.pool.query<AgentRunRow>(
+      `SELECT * FROM lattice_agent_runs WHERE tenant_id = $1 AND run_id = $2`,
+      [tenantId, runId],
+    );
+    return result.rowCount === 0 ? undefined : rowToRecord(result.rows[0]);
+  }
+
+  /** Finds the active (pending/running) run for a ProjectTask. */
+  async getActiveRunForTask(
+    tenantId: string,
+    taskId: string,
+  ): Promise<AgentRunRecord | undefined> {
+    const result = await this.pool.query<AgentRunRow>(
+      `SELECT * FROM lattice_agent_runs
+        WHERE tenant_id = $1 AND task_id = $2
+          AND status IN ('pending', 'running')
+        ORDER BY created_at DESC, run_id DESC
+        LIMIT 1`,
+      [tenantId, taskId],
+    );
+    return result.rowCount === 0 ? undefined : rowToRecord(result.rows[0]);
+  }
+
+  /** Lists a thread's runs, newest first (including pending rows). */
+  async listRuns(
+    tenantId: string,
+    threadId: string,
+    options?: { limit?: number; offset?: number },
+  ): Promise<AgentRunRecord[]> {
+    const limit = clampLimit(options?.limit);
+    const offset = Math.max(0, Math.floor(options?.offset ?? 0));
+    const result = await this.pool.query<AgentRunRow>(
+      `SELECT * FROM lattice_agent_runs
+        WHERE tenant_id = $1 AND thread_id = $2
+        ORDER BY created_at DESC
+        LIMIT $3 OFFSET $4`,
+      [tenantId, threadId, limit, offset],
+    );
+    return result.rows.map(rowToRecord);
+  }
+
+  /** Finds the thread's running run, if any. */
+  async findRunning(
+    tenantId: string,
+    threadId: string,
+  ): Promise<AgentRunRecord | undefined> {
+    const result = await this.pool.query<AgentRunRow>(
+      `SELECT * FROM lattice_agent_runs
+        WHERE tenant_id = $1 AND thread_id = $2 AND status = 'running'
+        LIMIT 1`,
+      [tenantId, threadId],
+    );
+    return result.rowCount === 0 ? undefined : rowToRecord(result.rows[0]);
+  }
+
+  /** Requests cancellation, settling pending/expired rows interrupted. */
+  async requestCancel(
+    tenantId: string,
+    runId: string,
+  ): Promise<"cancelled" | "requested" | "not_found" | "terminal"> {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      const current = await client.query<{
+        status: string;
+        expired: boolean;
+      }>(
+        `SELECT status,
+                COALESCE(lease_expires_at < now(), TRUE) AS expired
+           FROM lattice_agent_runs
+          WHERE tenant_id = $1 AND run_id = $2
+          FOR UPDATE`,
+        [tenantId, runId],
+      );
+
+      if (current.rowCount === 0) {
+        await client.query("COMMIT");
+        return "not_found";
+      }
+
+      const row = current.rows[0];
+      if (row.status === "pending") {
+        await client.query(
+          `UPDATE lattice_agent_runs
+              SET status = 'interrupted', cancel_requested = TRUE,
+                  finished_at = now(), updated_at = now()
+            WHERE tenant_id = $1 AND run_id = $2`,
+          [tenantId, runId],
+        );
+        await client.query("COMMIT");
+        return "cancelled";
+      }
+      if (row.status !== "running") {
+        await client.query("COMMIT");
+        return "terminal";
+      }
+
+      if (row.expired) {
+        await client.query(
+          `UPDATE lattice_agent_runs
+              SET status = 'interrupted', cancel_requested = TRUE,
+                  finished_at = now(), updated_at = now()
+            WHERE tenant_id = $1 AND run_id = $2`,
+          [tenantId, runId],
+        );
+        await client.query("COMMIT");
+        return "cancelled";
+      }
+
+      await client.query(
+        `UPDATE lattice_agent_runs
+            SET cancel_requested = TRUE, updated_at = now()
+          WHERE tenant_id = $1 AND run_id = $2`,
+        [tenantId, runId],
+      );
+      await client.query("COMMIT");
+      return "requested";
+    } catch (error) {
+      await client.query("ROLLBACK");
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Deletes a pending run or flags a running run for rollback. */
+  async requestRollback(
+    tenantId: string,
+    runId: string,
+  ): Promise<AgentRunRollbackResult> {
+    const client = await this.pool.connect();
+    try {
+      await client.query("BEGIN");
+      const current = await client.query<{ status: string; source: string }>(
+        `SELECT status, source FROM lattice_agent_runs
+          WHERE tenant_id = $1 AND run_id = $2
+          FOR UPDATE`,
+        [tenantId, runId],
+      );
+      if (current.rowCount === 0) {
+        await client.query("COMMIT");
+        return "not_found";
+      }
+
+      const { status, source } = current.rows[0];
+      if (source !== "run_api") {
+        await client.query("COMMIT");
+        return "terminal";
+      }
+      if (status === "pending") {
+        await client.query(
+          `DELETE FROM lattice_agent_runs WHERE tenant_id = $1 AND run_id = $2`,
+          [tenantId, runId],
+        );
+        await client.query("COMMIT");
+        return "deleted";
+      }
+      if (status !== "running") {
+        await client.query("COMMIT");
+        return "terminal";
+      }
+
+      await client.query(
+        `UPDATE lattice_agent_runs
+            SET rollback_requested = TRUE, cancel_requested = TRUE,
+                updated_at = now()
+          WHERE tenant_id = $1 AND run_id = $2`,
+        [tenantId, runId],
+      );
+      await client.query("COMMIT");
+      return "requested";
+    } catch (error) {
+      await client.query("ROLLBACK");
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  /** Atomically reclaims an expired running row without requeueing it. */
+  async recoverExpiredRun(
+    runId: string,
+    owner: string,
+    leaseMs: number,
+  ): Promise<AgentRunRecord | undefined> {
+    const result = await this.pool.query<AgentRunRow>(
+      `UPDATE lattice_agent_runs
+          SET owner = $2,
+              lease_expires_at = now() + ($3::bigint || ' milliseconds')::interval,
+              attempts = attempts + 1,
+              updated_at = now()
+        WHERE run_id = $1
+          AND source IN ('run_api', 'project_task')
+          AND status = 'running'
+          AND lease_expires_at < now()
+        RETURNING *`,
+      [runId, owner, leaseMs],
+    );
+    return result.rowCount === 0 ? undefined : rowToRecord(result.rows[0]);
+  }
+
+  /** Atomically requeues or interrupts an expired run-queue row. */
+  async settleExpiredRun(
+    runId: string,
+    outcome: AgentRunExpiredOutcome,
+    options?: SettleExpiredRunOptions,
+  ): Promise<boolean> {
+    if (outcome === "pending") {
+      const availableAt = options?.availableAt ?? Date.now();
+      const result = await this.pool.query(
+        `UPDATE lattice_agent_runs
+            SET status = 'pending',
+                owner = NULL,
+                lease_expires_at = NULL,
+                started_at = NULL,
+                available_at = to_timestamp($2::double precision / 1000.0),
+                finished_at = NULL,
+                updated_at = now()
+          WHERE run_id = $1
+            AND source IN ('run_api', 'project_task')
+            AND status = 'running'
+            AND lease_expires_at < now()
+          RETURNING run_id`,
+        [runId, availableAt],
+      );
+      return (result.rowCount ?? 0) > 0;
+    }
+
+    const result = await this.pool.query(
+      `UPDATE lattice_agent_runs
+          SET status = $3,
+              error = COALESCE($2, error),
+              finished_at = now(),
+              updated_at = now()
+        WHERE run_id = $1
+          AND source IN ('run_api', 'project_task')
+          AND status = 'running'
+          AND lease_expires_at < now()
+        RETURNING run_id`,
+      [
+        runId,
+        options?.error ?? null,
+        outcome === "timeout" ? "timeout" : "interrupted",
+      ],
+    );
+    return (result.rowCount ?? 0) > 0;
+  }
+
+  /** Counts runs grouped by lifecycle status. */
+  async counts(): Promise<AgentRunCounts> {
+    const counts: AgentRunCounts = {
+      pending: 0,
+      running: 0,
+      success: 0,
+      error: 0,
+      interrupted: 0,
+      cancelled: 0,
+      timeout: 0,
+    };
+    const result = await this.pool.query<{ status: string; count: number }>(
+      `SELECT status, count(*)::int AS count
+         FROM lattice_agent_runs
+        GROUP BY status`,
+    );
+    for (const row of result.rows) {
+      const status = parseStatus(row.status);
+      if (status) counts[status] = row.count;
+    }
+    return counts;
+  }
+
+  /** Reports expired run-queue rows for the requested sources. */
+  async reapExpired(
+    sources: readonly AgentRunSource[] = ["run_api"],
+  ): Promise<ReapedAgentRun[]> {
+    assertQueueSources(sources);
+    const result = await this.pool.query<AgentRunRow>(
+      `SELECT * FROM lattice_agent_runs
+        WHERE source = ANY($1::text[]) AND status = 'running'
+          AND lease_expires_at < now()
+        ORDER BY lease_expires_at ASC`,
+      [[...sources]],
+    );
+    return result.rows.map(reapedFromRow);
+  }
+
+  /** Lists rollback work that still needs a checkpoint rewind. */
+  async listRollbackRuns(): Promise<ReapedAgentRun[]> {
+    const result = await this.pool.query<AgentRunRow>(
+      `SELECT * FROM lattice_agent_runs
+        WHERE source = 'run_api'
+          AND rollback_requested = TRUE
+          AND (
+            status = 'interrupted'
+            OR (status = 'running' AND lease_expires_at < now())
+          )
+        ORDER BY updated_at ASC`,
+    );
+    return result.rows.map(reapedFromRow);
+  }
+
+  /** Settles expired legacy rows, preserving the historical supervisor behavior. */
+  async reapLegacyExpired(): Promise<ReapedAgentRun[]> {
+    const result = await this.pool.query<{
+      run_id: string;
+      tenant_id: string;
+      assistant_id: string;
+      thread_id: string;
+      workspace_id: string | null;
+      project_id: string | null;
+    }>(
+      `UPDATE lattice_agent_runs
+          SET status = 'interrupted',
+              error = COALESCE(error, 'run owner lost'),
+              finished_at = now(),
+              updated_at = now()
+        WHERE source = 'legacy' AND status = 'running'
+          AND lease_expires_at < now()
+        RETURNING run_id, tenant_id, assistant_id, thread_id, workspace_id, project_id`,
+    );
+    return result.rows.map((row) => ({
+      runId: row.run_id,
+      tenantId: row.tenant_id,
+      assistantId: row.assistant_id,
+      threadId: row.thread_id,
+      workspaceId: row.workspace_id ?? undefined,
+      projectId: row.project_id ?? undefined,
+    }));
+  }
+}
+
+function positiveLimit(limit: number | undefined): number | undefined {
+  return limit !== undefined && Number.isSafeInteger(limit) && limit > 0
+    ? limit
+    : undefined;
+}
+
+function clampLimit(limit: number | undefined): number {
+  if (limit === undefined) return DEFAULT_LIST_LIMIT;
+  const floored = Math.floor(limit);
+  if (!Number.isFinite(floored)) return DEFAULT_LIST_LIMIT;
+  return Math.min(MAX_LIST_LIMIT, Math.max(1, floored));
+}
+
+function jsonParam(value: AgentRunJsonObject | null | undefined): string | null {
+  return value === undefined || value === null ? null : JSON.stringify(value);
+}
+
+function parseStatus(value: string): AgentRunStatus | undefined {
+  switch (value) {
+    case "pending":
+    case "running":
+    case "success":
+    case "error":
+    case "interrupted":
+    case "cancelled":
+    case "timeout":
+      return value;
+    default:
+      return undefined;
+  }
+}
+
+function parseSource(value: string | null): AgentRunSource {
+  switch (value) {
+    case "run_api":
+      return "run_api";
+    case "project_task":
+      return "project_task";
+    default:
+      return "legacy";
+  }
+}
+
+function toJsonObject(value: unknown): AgentRunJsonObject | null {
+  if (value === null || value === undefined) return null;
+  if (typeof value !== "object" || Array.isArray(value)) return null;
+  return value as AgentRunJsonObject;
+}
+
+function rowToRecord(row: AgentRunRow): AgentRunRecord {
+  return {
+    runId: row.run_id,
+    tenantId: row.tenant_id,
+    workspaceId: row.workspace_id ?? undefined,
+    projectId: row.project_id ?? undefined,
+    assistantId: row.assistant_id,
+    threadId: row.thread_id,
+    status: parseStatus(row.status) ?? "error",
+    source: parseSource(row.source),
+    taskId: row.task_id ?? undefined,
+    owner: row.owner,
+    leaseExpiresAt:
+      row.lease_expires_at === null ? null : row.lease_expires_at.getTime(),
+    inputMessageIds: row.input_message_ids,
+    attempts: row.attempts,
+    error: row.error,
+    cancelRequested: row.cancel_requested,
+    rollbackRequested: row.rollback_requested,
+    rollbackAttempts: row.rollback_attempts,
+    rollbackError: row.rollback_error,
+    input: toJsonObject(row.input),
+    command: toJsonObject(row.command),
+    config: toJsonObject(row.config),
+    metadata: toJsonObject(row.metadata),
+    baseCheckpointId: row.base_checkpoint_id,
+    baseCheckpointCaptured: row.base_checkpoint_captured,
+    capabilityRuntimeRevision: row.capability_runtime_revision,
+    availableAt: row.available_at.getTime(),
+    createdAt: row.created_at.getTime(),
+    updatedAt: row.updated_at.getTime(),
+    startedAt: row.started_at === null ? null : row.started_at.getTime(),
+    finishedAt: row.finished_at === null ? null : row.finished_at.getTime(),
+  };
+}
+
+function reapedFromRow(row: AgentRunRow): ReapedAgentRun {
+  return {
+    runId: row.run_id,
+    tenantId: row.tenant_id,
+    assistantId: row.assistant_id,
+    threadId: row.thread_id,
+    workspaceId: row.workspace_id ?? undefined,
+    projectId: row.project_id ?? undefined,
+    source: parseSource(row.source),
+    taskId: row.task_id ?? undefined,
+    attempts: row.attempts,
+    cancelRequested: row.cancel_requested,
+    rollbackRequested: row.rollback_requested,
+    baseCheckpointId: row.base_checkpoint_id,
+    createdAt: row.created_at.getTime(),
+    startedAt: row.started_at === null ? null : row.started_at.getTime(),
+    leaseExpiresAt:
+      row.lease_expires_at === null ? null : row.lease_expires_at.getTime(),
+  };
+}
+
+function isActiveRunViolation(error: unknown): boolean {
+  if (typeof error !== "object" || error === null) return false;
+  const { code, constraint } = error as {
+    code?: unknown;
+    constraint?: unknown;
+  };
+  return code === "23505" && constraint === "uq_lattice_agent_runs_one_running";
+}