@opsee/cli 0.11.9

package/src/foreman/core/process-table.ts

@@ -0,0 +1,535 @@
+/**
+ * The Process Table (see ../../../CONTEXT.md; ADR-0009): the Foreman's local record of live
+ * Workers, OS process, vendor session id, Account, last output time, kept in SQLite on the machine
+ * that runs them, with an outbox for Run Record writes that could not reach Opsee. Nothing here is
+ * a fact about the work: those go to the Run Record first (ADR-0009), and deleting this file loses
+ * nothing about a Run except the ability to reattach to processes that are already gone (story 52).
+ *
+ * Three tables:
+ *
+ * - `workers`: one row per Worker the Foreman has running or believes it has (a crash leaves rows
+ *   behind; Reconcile is what settles them). `session_id` is written the moment the adapter's
+ *   `started` event arrives, before any further output, so a crash right after launch is resumable.
+ * - `outbox`: Run Record event batches that could not be appended, in id order, which is the order
+ *   they must reach Opsee in. `drain` stops at the first failure so order is never broken.
+ * - `runs`: Run requests `foreman run` hands the daemon (story 13); the daemon serves the oldest
+ *   pending one per tick, skipping those of a paused Initiative, and puts a request back to
+ *   pending when its Initiative is paused mid-Run. One row per request, whatever the Slots the Run
+ *   it starts then fills.
+ * - `pauses`: the Initiatives whose Run a human has paused (story 55): no new dispatch until
+ *   resumed, in-flight Workers finish. One row per paused Initiative.
+ *
+ * A Worker row also carries a `control` mark, the machine-local side of the human's steering
+ * (stories 53-56; the Run Record gets a `control` event): `attached` while a human has the session
+ * as an attended turn (Reconcile leaves the row alone), `released` once handed back (the next tick
+ * resumes it unattended), `cancelled` once `foreman cancel` stopped it (whoever owns the turn, or
+ * the next tick once the pid is dead, closes the attempt). The command that sets the mark is the
+ * one that stops the process and clears the pid it stopped; the process that owns the turn reads
+ * the mark when the stream ends and settles accordingly, and Reconcile closes a cancelled row only
+ * when nothing is alive to own it, so the attempt is recorded once.
+ *
+ * The driver is `node:sqlite`, built into the Node the CLI runs on (see the decision on Initiative
+ * 17): no native build, no dependency. One connection per process; the daemon and a `foreman run`
+ * from another terminal share the file through SQLite's own locking, and every statement here is
+ * short.
+ */
+import { chmodSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { fromJson, toJson, type JsonValue } from "@bufbuild/protobuf";
+import { RunEventInputSchema, type RunEventInput } from "@opsee/mcp-server/gen/api/v1/initiative_pb.js";
+import type { Vendor } from "../vendor.js";
+
+/** The human's steering mark on a Worker row (see the module comment). */
+export type WorkerControl = "attached" | "released" | "cancelled";
+
+export interface WorkerRow {
+  taskId: number;
+  initiativeId: number;
+  /** Human identifier (OPS-267), for the transcript file name and the log. */
+  identifier: string;
+  account: string;
+  vendor: Vendor;
+  /** Unset until the adapter's `started` event; a row without one cannot be resumed, only restarted. */
+  sessionId?: string;
+  /** The Worker's OS pid, when the adapter exposes one; unset for adapters that do not (the fake).
+   * During a Verifier round it is the Verifier turn's pid (core/verifier.ts). */
+  pid?: number;
+  /** The pid (and, spawned detached, the process group) of the app a Verifier round has running
+   * for this Task; unset outside a round. `foreman cancel` stops it with the Worker, and
+   * Reconcile stops one a dead Foreman left running. */
+  appPid?: number;
+  workspace: string;
+  branch: string;
+  /** Epoch milliseconds. */
+  startedAt: number;
+  lastOutputAt: number;
+  attempt: number;
+  /** Set by `foreman attach`, `release` and `cancel`; unset for a Worker nobody has steered. */
+  control?: WorkerControl;
+  /** Epoch milliseconds of the last control change. */
+  controlAt?: number;
+}
+
+export interface OutboxRow {
+  id: number;
+  initiativeId: number;
+  events: RunEventInput[];
+  createdAt: number;
+  /** How many drains have failed on this batch. */
+  attempts: number;
+}
+
+export type RunRequestStatus = "pending" | "running" | "done" | "failed";
+
+export interface RunRequestRow {
+  id: number;
+  initiativeId: number;
+  account?: string;
+  once: boolean;
+  taskId?: number;
+  /** The checkout the request was made from; the daemon serves only its own. */
+  repoRoot?: string;
+  maxTurns?: number;
+  stallTimeoutMs?: number;
+  requestedAt: number;
+  startedAt?: number;
+  finishedAt?: number;
+  status: RunRequestStatus;
+  error?: string;
+}
+
+export interface NewRunRequest {
+  initiativeId: number;
+  account?: string;
+  once?: boolean;
+  taskId?: number;
+  repoRoot?: string;
+  maxTurns?: number;
+  stallTimeoutMs?: number;
+}
+
+/** The seam the Run loop and Reconcile use; `ProcessTable` is the SQLite implementation. */
+export interface ProcessTableApi {
+  upsertWorker(row: WorkerRow): void;
+  /** Records the vendor session id (and pid) the moment `started` arrives. */
+  setSession(taskId: number, sessionId: string, pid?: number): void;
+  /** Records the OS pid the moment the launch returns one, before the vendor has said anything:
+   * a row with a pid and no session is a Worker that may still be running. */
+  setPid(taskId: number, pid: number): void;
+  /** Forgets the pid once the process it named is known to be gone (`foreman attach` and `cancel`
+   * stopped it), so a later check never mistakes a recycled pid for the Worker. */
+  clearPid(taskId: number): void;
+  /** Records the pid of the app a Verifier round started for the Task, or forgets it (undefined)
+   * once the app is stopped. */
+  setAppPid(taskId: number, pid: number | undefined): void;
+  /** Bumps `lastOutputAt`; callers throttle. */
+  touch(taskId: number, at: number): void;
+  removeWorker(taskId: number): void;
+  liveWorkers(filter?: { initiativeId?: number; account?: string }): WorkerRow[];
+  worker(taskId: number): WorkerRow | undefined;
+  /** Sets or clears (undefined) the control mark on a row; false when there is no such row. */
+  setControl(taskId: number, control: WorkerControl | undefined, at: number): boolean;
+  /** Whether a human has paused new dispatch for the Initiative (story 55). */
+  isPaused(initiativeId: number): boolean;
+  enqueue(initiativeId: number, events: RunEventInput[]): number;
+  pending(filter?: { initiativeId?: number }): OutboxRow[];
+  /** Sends pending batches in id order through `sender`; stops at the first failure (the batch
+   * stays, its attempt count grows) and reports how many went and what stopped it. A failure
+   * `discard` accepts (the backend refused the batch and would refuse it forever) drops the batch
+   * instead, so one dead batch does not hold every later one; `discarded` counts those. */
+  drain(sender: (initiativeId: number, events: RunEventInput[]) => Promise<unknown>, options?: DrainOptions): Promise<DrainResult>;
+}
+
+export interface DrainOptions {
+  initiativeId?: number;
+  discard?: (error: unknown) => boolean;
+}
+
+export interface DrainResult {
+  sent: number;
+  remaining: number;
+  /** Batches dropped because the backend refused them; each is logged by the caller. */
+  discarded: OutboxRow[];
+  /** The error that stopped the drain, when one did. */
+  error?: Error;
+}
+
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS workers (
+  task_id INTEGER PRIMARY KEY,
+  initiative_id INTEGER NOT NULL,
+  identifier TEXT NOT NULL,
+  account TEXT NOT NULL,
+  vendor TEXT NOT NULL,
+  session_id TEXT,
+  pid INTEGER,
+  workspace TEXT NOT NULL,
+  branch TEXT NOT NULL,
+  started_at INTEGER NOT NULL,
+  last_output_at INTEGER NOT NULL,
+  attempt INTEGER NOT NULL,
+  settled_at INTEGER
+);
+CREATE TABLE IF NOT EXISTS outbox (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  initiative_id INTEGER NOT NULL,
+  events_json TEXT NOT NULL,
+  created_at INTEGER NOT NULL,
+  attempts INTEGER NOT NULL DEFAULT 0
+);
+CREATE TABLE IF NOT EXISTS runs (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  initiative_id INTEGER NOT NULL,
+  account TEXT,
+  once INTEGER NOT NULL DEFAULT 0,
+  task_id INTEGER,
+  repo_root TEXT,
+  max_turns INTEGER,
+  stall_timeout_ms INTEGER,
+  requested_at INTEGER NOT NULL,
+  started_at INTEGER,
+  finished_at INTEGER,
+  status TEXT NOT NULL DEFAULT 'pending',
+  error TEXT
+);
+CREATE TABLE IF NOT EXISTS pauses (
+  initiative_id INTEGER PRIMARY KEY,
+  paused_at INTEGER NOT NULL
+);
+`;
+
+/** Columns added after the first release; a table created before them gets them on open. */
+const WORKER_COLUMNS_ADDED: ReadonlyArray<{ name: string; ddl: string }> = [
+  { name: "control", ddl: "control TEXT" },
+  { name: "control_at", ddl: "control_at INTEGER" },
+  { name: "app_pid", ddl: "app_pid INTEGER" },
+];
+
+type Row = Record<string, string | number | bigint | null | Uint8Array>;
+
+const num = (v: Row[string]): number => Number(v);
+const opt = <T>(v: Row[string], map: (x: Exclude<Row[string], null>) => T): T | undefined => (v === null || v === undefined ? undefined : map(v));
+
+function workerOf(r: Row): WorkerRow {
+  return {
+    taskId: num(r.task_id),
+    initiativeId: num(r.initiative_id),
+    identifier: String(r.identifier),
+    account: String(r.account),
+    vendor: String(r.vendor) as Vendor,
+    sessionId: opt(r.session_id, String),
+    pid: opt(r.pid, num),
+    appPid: opt(r.app_pid, num),
+    workspace: String(r.workspace),
+    branch: String(r.branch),
+    startedAt: num(r.started_at),
+    lastOutputAt: num(r.last_output_at),
+    attempt: num(r.attempt),
+    control: opt(r.control, (v) => String(v) as WorkerControl),
+    controlAt: opt(r.control_at, num),
+  };
+}
+
+function runOf(r: Row): RunRequestRow {
+  return {
+    id: num(r.id),
+    initiativeId: num(r.initiative_id),
+    account: opt(r.account, String),
+    once: num(r.once) === 1,
+    taskId: opt(r.task_id, num),
+    repoRoot: opt(r.repo_root, String),
+    maxTurns: opt(r.max_turns, num),
+    stallTimeoutMs: opt(r.stall_timeout_ms, num),
+    requestedAt: num(r.requested_at),
+    startedAt: opt(r.started_at, num),
+    finishedAt: opt(r.finished_at, num),
+    status: String(r.status) as RunRequestStatus,
+    error: opt(r.error, String),
+  };
+}
+
+export function serializeEvents(events: RunEventInput[]): string {
+  return JSON.stringify(events.map((e) => toJson(RunEventInputSchema, e)));
+}
+
+export function deserializeEvents(json: string): RunEventInput[] {
+  return (JSON.parse(json) as JsonValue[]).map((j) => fromJson(RunEventInputSchema, j));
+}
+
+export class ProcessTable implements ProcessTableApi {
+  private readonly db: DatabaseSync;
+
+  /** The clock every row's timestamp is taken from. Injectable so a test can put worker rows and
+   * Run requests on one timeline: mixing a frozen clock for the first with the real one for the
+   * second makes an ordering assertion depend on the wall-clock date it is run on. */
+  private readonly now: () => number;
+
+  private constructor(readonly path: string, now: () => number = Date.now) {
+    this.now = now;
+    if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+    this.db = new DatabaseSync(path);
+    // Owner-only, like the credentials and Accounts files beside it; the driver takes no mode.
+    if (path !== ":memory:") chmodSync(path, 0o600);
+    this.db.exec("PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;");
+    this.db.exec(SCHEMA);
+    const have = new Set((this.db.prepare("PRAGMA table_info(workers)").all() as Row[]).map((c) => String(c.name)));
+    for (const column of WORKER_COLUMNS_ADDED) if (!have.has(column.name)) this.db.exec(`ALTER TABLE workers ADD COLUMN ${column.ddl}`);
+  }
+
+  /** Opens (creating when missing) the table at `path`; `:memory:` for tests that need no file. */
+  static open(path: string, options: { now?: () => number } = {}): ProcessTable {
+    return new ProcessTable(path, options.now);
+  }
+
+  close(): void {
+    this.db.close();
+  }
+
+  // --- workers ---
+
+  upsertWorker(row: WorkerRow): void {
+    this.db
+      .prepare(
+        `INSERT INTO workers (task_id, initiative_id, identifier, account, vendor, session_id, pid, app_pid, workspace, branch, started_at, last_output_at, attempt, control, control_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(task_id) DO UPDATE SET initiative_id = excluded.initiative_id, identifier = excluded.identifier, account = excluded.account,
+           vendor = excluded.vendor, session_id = excluded.session_id, pid = excluded.pid, app_pid = excluded.app_pid, workspace = excluded.workspace, branch = excluded.branch,
+           started_at = excluded.started_at, last_output_at = excluded.last_output_at, attempt = excluded.attempt, control = excluded.control, control_at = excluded.control_at`,
+      )
+      .run(
+        row.taskId,
+        row.initiativeId,
+        row.identifier,
+        row.account,
+        row.vendor,
+        row.sessionId ?? null,
+        row.pid ?? null,
+        row.appPid ?? null,
+        row.workspace,
+        row.branch,
+        row.startedAt,
+        row.lastOutputAt,
+        row.attempt,
+        row.control ?? null,
+        row.controlAt ?? null,
+      );
+  }
+
+  setControl(taskId: number, control: WorkerControl | undefined, at: number): boolean {
+    return Number(this.db.prepare("UPDATE workers SET control = ?, control_at = ? WHERE task_id = ?").run(control ?? null, at, taskId).changes) > 0;
+  }
+
+  setSession(taskId: number, sessionId: string, pid?: number): void {
+    this.db.prepare("UPDATE workers SET session_id = ?, pid = COALESCE(?, pid) WHERE task_id = ?").run(sessionId, pid ?? null, taskId);
+  }
+
+  setPid(taskId: number, pid: number): void {
+    this.db.prepare("UPDATE workers SET pid = ? WHERE task_id = ?").run(pid, taskId);
+  }
+
+  clearPid(taskId: number): void {
+    this.db.prepare("UPDATE workers SET pid = NULL WHERE task_id = ?").run(taskId);
+  }
+
+  setAppPid(taskId: number, pid: number | undefined): void {
+    this.db.prepare("UPDATE workers SET app_pid = ? WHERE task_id = ?").run(pid ?? null, taskId);
+  }
+
+  touch(taskId: number, at: number): void {
+    this.db.prepare("UPDATE workers SET last_output_at = ? WHERE task_id = ?").run(at, taskId);
+  }
+
+  removeWorker(taskId: number): void {
+    this.db.prepare("DELETE FROM workers WHERE task_id = ?").run(taskId);
+  }
+
+  worker(taskId: number): WorkerRow | undefined {
+    const r = this.db.prepare("SELECT * FROM workers WHERE task_id = ?").get(taskId) as Row | undefined;
+    return r ? workerOf(r) : undefined;
+  }
+
+  liveWorkers(filter: { initiativeId?: number; account?: string } = {}): WorkerRow[] {
+    const where: string[] = [];
+    const args: (number | string)[] = [];
+    if (filter.initiativeId !== undefined) {
+      where.push("initiative_id = ?");
+      args.push(filter.initiativeId);
+    }
+    if (filter.account !== undefined) {
+      where.push("account = ?");
+      args.push(filter.account);
+    }
+    const sql = `SELECT * FROM workers${where.length ? ` WHERE ${where.join(" AND ")}` : ""} ORDER BY started_at, task_id`;
+    return (this.db.prepare(sql).all(...args) as Row[]).map(workerOf);
+  }
+
+  // --- outbox ---
+
+  enqueue(initiativeId: number, events: RunEventInput[]): number {
+    const result = this.db.prepare("INSERT INTO outbox (initiative_id, events_json, created_at) VALUES (?, ?, ?)").run(initiativeId, serializeEvents(events), this.now());
+    return Number(result.lastInsertRowid);
+  }
+
+  pending(filter: { initiativeId?: number } = {}): OutboxRow[] {
+    const rows =
+      filter.initiativeId === undefined
+        ? (this.db.prepare("SELECT * FROM outbox ORDER BY id").all() as Row[])
+        : (this.db.prepare("SELECT * FROM outbox WHERE initiative_id = ? ORDER BY id").all(filter.initiativeId) as Row[]);
+    return rows.map((r) => ({ id: num(r.id), initiativeId: num(r.initiative_id), events: deserializeEvents(String(r.events_json)), createdAt: num(r.created_at), attempts: num(r.attempts) }));
+  }
+
+  async drain(sender: (initiativeId: number, events: RunEventInput[]) => Promise<unknown>, options: DrainOptions = {}): Promise<DrainResult> {
+    const filter = options.initiativeId === undefined ? {} : { initiativeId: options.initiativeId };
+    let sent = 0;
+    const discarded: OutboxRow[] = [];
+    for (const batch of this.pending(filter)) {
+      try {
+        await sender(batch.initiativeId, batch.events);
+      } catch (error) {
+        if (options.discard?.(error)) {
+          this.db.prepare("DELETE FROM outbox WHERE id = ?").run(batch.id);
+          discarded.push(batch);
+          continue;
+        }
+        this.db.prepare("UPDATE outbox SET attempts = attempts + 1 WHERE id = ?").run(batch.id);
+        return { sent, remaining: this.pending(filter).length, discarded, error: error instanceof Error ? error : new Error(String(error)) };
+      }
+      this.db.prepare("DELETE FROM outbox WHERE id = ?").run(batch.id);
+      sent++;
+    }
+    return { sent, remaining: 0, discarded };
+  }
+
+  // --- runs ---
+
+  requestRun(request: NewRunRequest): RunRequestRow {
+    const result = this.db
+      .prepare("INSERT INTO runs (initiative_id, account, once, task_id, repo_root, max_turns, stall_timeout_ms, requested_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
+      .run(request.initiativeId, request.account ?? null, request.once ? 1 : 0, request.taskId ?? null, request.repoRoot ?? null, request.maxTurns ?? null, request.stallTimeoutMs ?? null, this.now());
+    return this.run(Number(result.lastInsertRowid))!;
+  }
+
+  run(id: number): RunRequestRow | undefined {
+    const r = this.db.prepare("SELECT * FROM runs WHERE id = ?").get(id) as Row | undefined;
+    return r ? runOf(r) : undefined;
+  }
+
+  /** The oldest pending Run request, marked running, or undefined when none waits. A request for
+   * a paused Initiative (story 55) is left pending: serving it would only find the pause again, so
+   * it waits for `foreman resume` while the daemon serves the others. */
+  takeRun(): RunRequestRow | undefined {
+    const r = this.db
+      .prepare("SELECT * FROM runs WHERE status = 'pending' AND initiative_id NOT IN (SELECT initiative_id FROM pauses) ORDER BY id LIMIT 1")
+      .get() as Row | undefined;
+    if (!r) return undefined;
+    this.db.prepare("UPDATE runs SET status = 'running', started_at = ? WHERE id = ?").run(this.now(), num(r.id));
+    return this.run(num(r.id));
+  }
+
+  finishRun(id: number, status: "done" | "failed", error?: string): void {
+    this.db.prepare("UPDATE runs SET status = ?, finished_at = ?, error = ? WHERE id = ?").run(status, this.now(), error ?? null, id);
+  }
+
+  /** Puts a request the daemon started back to pending, with a note saying why (its Initiative was
+   * paused mid-Run); the daemon takes it again once the pause is lifted. */
+  requeueRun(id: number, note: string): void {
+    this.db.prepare("UPDATE runs SET status = 'pending', started_at = NULL, finished_at = NULL, error = ? WHERE id = ?").run(note, id);
+  }
+
+  /** Requests left `running` by a daemon that died; the next daemon puts them back to pending. */
+  requeueRunning(): number {
+    return Number(this.db.prepare("UPDATE runs SET status = 'pending', started_at = NULL WHERE status = 'running'").run().changes);
+  }
+
+  // --- pauses ---
+
+  /** Pauses new dispatch for the Initiative; idempotent, keeps the first pause time. */
+  pause(initiativeId: number, at: number): void {
+    this.db.prepare("INSERT INTO pauses (initiative_id, paused_at) VALUES (?, ?) ON CONFLICT(initiative_id) DO NOTHING").run(initiativeId, at);
+  }
+
+  /** Lifts the pause; false when the Initiative was not paused. */
+  resume(initiativeId: number): boolean {
+    return Number(this.db.prepare("DELETE FROM pauses WHERE initiative_id = ?").run(initiativeId).changes) > 0;
+  }
+
+  isPaused(initiativeId: number): boolean {
+    return this.pausedAt(initiativeId) !== undefined;
+  }
+
+  /** Every Initiative a human has paused, oldest pause first: what `foreman status` shows so a
+   * fleet that has stopped dispatching says why. */
+  pausedInitiatives(): Array<{ initiativeId: number; pausedAt: number }> {
+    return (this.db.prepare("SELECT initiative_id, paused_at FROM pauses ORDER BY paused_at, initiative_id").all() as Row[]).map((r) => ({
+      initiativeId: num(r.initiative_id),
+      pausedAt: num(r.paused_at),
+    }));
+  }
+
+  /** The Initiatives this machine has anything of, most recently touched first: one with a live
+   * Worker row, or one a Run request named. What `foreman review` defaults to when exactly one
+   * comes back, so the usual "what happened last night" needs no id. */
+  knownInitiatives(): number[] {
+    const rows = this.db
+      .prepare(
+        `SELECT initiative_id, MAX(at) AS at FROM (
+           SELECT initiative_id, last_output_at AS at FROM workers
+           UNION ALL
+           SELECT initiative_id, COALESCE(finished_at, started_at, requested_at) AS at FROM runs
+         ) GROUP BY initiative_id ORDER BY at DESC, initiative_id DESC`,
+      )
+      .all() as Row[];
+    return rows.map((r) => num(r.initiative_id));
+  }
+
+  /** When the Initiative was paused, epoch milliseconds; undefined when it is not. */
+  pausedAt(initiativeId: number): number | undefined {
+    const r = this.db.prepare("SELECT paused_at FROM pauses WHERE initiative_id = ?").get(initiativeId) as Row | undefined;
+    return r ? num(r.paused_at) : undefined;
+  }
+}
+
+/** How long a row with no pid yet is read as a Foreman mid-dispatch rather than as leftover state.
+ * The row exists from the moment the dispatch is committed and gets its pid when the adapter
+ * launches (core/run.ts), so there is a window in which a live Foreman on an Account is invisible
+ * to a pid check. Short, because the other reading of a pid-less row is a Foreman that died in that
+ * same window, and one of those must not cost an Account its Failover for the rest of the day. */
+export const DISPATCH_PID_GRACE_MS = 60_000;
+
+/** The Workers another Foreman has on an Account right now (ADR-0013: the cap is one vendor
+ * identity's, and Slots to enforce it are a Run's, so a second Run opening its own Slots here would
+ * double it).
+ *
+ * A row counts when its pid is alive, and also when it has no pid but was written inside
+ * `DISPATCH_PID_GRACE_MS` — that is the dispatch-commit-to-launch window, where a Foreman really is
+ * starting a Worker on this Account and no pid check can see it. Rows the caller owns are its own
+ * turns, not another Foreman's, and rows whose Worker is provably gone are what Reconcile exists to
+ * settle: neither is held against the Account.
+ */
+export function otherForemanWorkersOn(
+  table: Pick<ProcessTableApi, "liveWorkers">,
+  account: string,
+  at: number,
+  options: { isAlive?: (pid: number | undefined) => boolean; owns?: (taskId: number) => boolean } = {},
+): WorkerRow[] {
+  const isAlive = options.isAlive ?? pidIsAlive;
+  return table.liveWorkers({ account }).filter((row) => {
+    if (options.owns?.(row.taskId)) return false;
+    if (row.pid !== undefined) return isAlive(row.pid);
+    return at - row.startedAt < DISPATCH_PID_GRACE_MS;
+  });
+}
+
+/** True when a process with this pid exists and this user may signal it. Unset means unknown,
+ * which Reconcile treats as gone: a row without a pid can only come from an adapter that never
+ * gave one, and then there is nothing to wait for. */
+export function pidIsAlive(pid: number | undefined): boolean {
+  if (pid === undefined) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return (error as NodeJS.ErrnoException).code === "EPERM";
+  }
+}