gsd-pi 2.78.1-dev.d8826a445 → 2.78.1-dev.eccf86e27

package/src/resources/extensions/gsd/crash-recovery.ts

@@ -1,21 +1,43 @@
 /**
- * GSD Crash Recovery
+ * GSD Crash Recovery (Phase C pt 2 — DB-backed)
  *
- * Detects interrupted auto-mode sessions via a lock file.
- * Written on auto-start, updated on each unit dispatch, deleted on clean stop.
- * If the lock file exists on next startup, the previous session crashed.
+ * Detects interrupted auto-mode sessions via the DB-backed workers +
+ * unit_dispatches + runtime_kv tables. The auto.lock file is gone; the
+ * `LockData` shape is preserved for backward compatibility with callers
+ * (auto.ts, doctor checks, interrupted-session.ts), but the contents are
+ * now synthesized from:
  *
- * The lock records the pi session file path so crash recovery can read the
- * surviving JSONL (pi appends entries incrementally via appendFileSync,
- * so the file on disk reflects every tool call up to the crash point).
+ *   - workers.pid / .started_at / .last_heartbeat_at  → liveness + age
+ *   - unit_dispatches.unit_type / .unit_id / .started_at  → what was running
+ *   - runtime_kv("worker", workerId, "session_file")  → pi session JSONL path
+ *
+ * "Crashed" is detected via workers.status='active' + heartbeat past TTL,
+ * cross-checked with the OS PID via isLockProcessAlive(). When the DB is
+ * unavailable (fresh project before init), all readers return null and
+ * writers no-op — preserving the historical "no lock means no prior
+ * crash" semantics.
+ *
+ * The journal-based emitCrashRecoveredUnitEnd is unchanged from the file
+ * era — it queries the journal independently of the lock mechanism.
  */
 
+import {
+  emitJournalEvent,
+  queryJournal,
+} from "./journal.js";
 import { readFileSync, unlinkSync, existsSync } from "node:fs";
 import { join } from "node:path";
-import { gsdRoot } from "./paths.js";
+import {
+  findStaleWorkerForProject,
+  getAllAutoWorkers,
+  type AutoWorkerRow,
+} from "./db/auto-workers.js";
+import { getLatestForUnit, type DispatchStatus } from "./db/unit-dispatches.js";
+import { getRuntimeKv, setRuntimeKv, deleteRuntimeKv } from "./db/runtime-kv.js";
+import { _getAdapter, isDbAvailable } from "./gsd-db.js";
+import { gsdRoot, normalizeRealPath } from "./paths.js";
 import { atomicWriteSync } from "./atomic-write.js";
 import { effectiveLockFile } from "./session-lock.js";
-import { emitJournalEvent, queryJournal } from "./journal.js";
 
 export interface LockData {
   pid: number;
@@ -27,11 +49,91 @@ export interface LockData {
   sessionFile?: string;
 }
 
+const SESSION_FILE_KV_KEY = "session_file";
+
 function lockPath(basePath: string): string {
   return join(gsdRoot(basePath), effectiveLockFile());
 }
 
-/** Write or update the lock file with current auto-mode state. */
+function readLegacyLock(basePath: string): LockData | null {
+  try {
+    const p = lockPath(basePath);
+    if (!existsSync(p)) return null;
+    return JSON.parse(readFileSync(p, "utf-8")) as LockData;
+  } catch {
+    return null;
+  }
+}
+
+function findActiveWorkerForCurrentProcess(
+  projectRootRealpath: string,
+): AutoWorkerRow | null {
+  if (!isDbAvailable()) return null;
+  const workers = getAllAutoWorkers();
+  for (const worker of workers) {
+    if (
+      worker.pid === process.pid
+      && worker.project_root_realpath === projectRootRealpath
+    ) {
+      return worker;
+    }
+  }
+  return null;
+}
+
+/**
+ * Look up the most recent dispatch row for a worker, regardless of status.
+ * Returns null if the worker has no dispatch history yet (e.g. crashed
+ * during bootstrap before claiming the first unit).
+ */
+function getLatestDispatchForWorker(workerId: string):
+  | { unit_type: string; unit_id: string; started_at: string; status: DispatchStatus }
+  | null {
+  if (!isDbAvailable()) return null;
+  const db = _getAdapter()!;
+  const row = db.prepare(
+    `SELECT unit_type, unit_id, started_at, status
+     FROM unit_dispatches
+     WHERE worker_id = :worker_id
+     ORDER BY id DESC
+     LIMIT 1`,
+  ).get({ ":worker_id": workerId }) as
+    | { unit_type: string; unit_id: string; started_at: string; status: DispatchStatus }
+    | undefined;
+  return row ?? null;
+}
+
+function workerToLockData(worker: AutoWorkerRow): LockData {
+  const dispatch = getLatestDispatchForWorker(worker.worker_id);
+  const sessionFile =
+    getRuntimeKv<string>("worker", worker.worker_id, SESSION_FILE_KV_KEY) ?? undefined;
+  return {
+    pid: worker.pid,
+    startedAt: worker.started_at,
+    // Pre-Phase-C-pt-2 default: when no dispatch row exists yet (bootstrap
+    // crash), report unitType="starting", unitId="bootstrap" — same shape
+    // the file-based writer used to produce.
+    unitType: dispatch?.unit_type ?? "starting",
+    unitId: dispatch?.unit_id ?? "bootstrap",
+    unitStartedAt: dispatch?.started_at ?? worker.started_at,
+    sessionFile,
+  };
+}
+
+/**
+ * Write or update the lock state for the current auto-mode session.
+ *
+ * Phase C pt 2: the only persistent state this function adds beyond what
+ * the workers + unit_dispatches tables already track is the pi session
+ * JSONL path, which lands in runtime_kv (worker scope, key
+ * "session_file"). The pid/startedAt/unitType/unitId/unitStartedAt are
+ * recorded by registerAutoWorker / heartbeatAutoWorker / recordDispatchClaim
+ * already.
+ *
+ * basePath is unused by the new implementation (kept as a parameter for
+ * back-compat with the 15+ call sites) — the worker is identified by
+ * pid + project_root_realpath in the workers table.
+ */
 export function writeLock(
   basePath: string,
   unitType: string,
@@ -47,51 +149,84 @@ export function writeLock(
       unitStartedAt: new Date().toISOString(),
       sessionFile,
     };
-    const lp = lockPath(basePath);
-    atomicWriteSync(lp, JSON.stringify(data, null, 2));
-  } catch (e) { /* non-fatal: lock write failure */ void e; }
+    atomicWriteSync(lockPath(basePath), JSON.stringify(data, null, 2));
+  } catch {
+    // Best-effort — never throw from the lock writer.
+  }
+
+  if (!isDbAvailable() || !sessionFile) return;
+  try {
+    const projectRoot = normalizeRealPath(basePath);
+    const worker = findActiveWorkerForCurrentProcess(projectRoot);
+    if (!worker) return;
+    setRuntimeKv("worker", worker.worker_id, SESSION_FILE_KV_KEY, sessionFile);
+  } catch {
+    // Best-effort — never throw from the lock writer.
+  }
 }
 
-/** Remove the lock file on clean stop. */
+/**
+ * Phase C pt 2: clearLock no longer deletes a file. The cleanup path
+ * (markWorkerStopping in stopAuto) flips the workers row to 'stopping'.
+ * This function additionally drops the session_file runtime_kv row for
+ * the current worker so a follow-up crash detection doesn't pick up a
+ * stale session-file pointer.
+ */
 export function clearLock(basePath: string): void {
   try {
     const p = lockPath(basePath);
     if (existsSync(p)) unlinkSync(p);
-  } catch (e) { /* non-fatal: lock clear failure */ void e; }
+  } catch {
+    // Best-effort.
+  }
+
+  if (!isDbAvailable()) return;
+  try {
+    const projectRoot = normalizeRealPath(basePath);
+    const worker = findActiveWorkerForCurrentProcess(projectRoot);
+    if (!worker) return;
+    deleteRuntimeKv("worker", worker.worker_id, SESSION_FILE_KV_KEY);
+  } catch {
+    // Best-effort.
+  }
 }
 
-/** Check if a crash lock exists and return its data. */
+/**
+ * Detect a previous crashed auto-mode session.
+ *
+ * Phase C pt 2: synthesized from workers (status='active' + lapsed
+ * heartbeat) + unit_dispatches (most recent for that worker) +
+ * runtime_kv (session_file). Returns null when no stale worker exists
+ * or the DB is unavailable.
+ */
 export function readCrashLock(basePath: string): LockData | null {
-  try {
-    const p = lockPath(basePath);
-    if (!existsSync(p)) return null;
-    const raw = readFileSync(p, "utf-8");
-    return JSON.parse(raw) as LockData;
-  } catch (e) {
-    /* non-fatal: corrupt or unreadable lock file */ void e;
-    return null;
+  if (isDbAvailable()) {
+    try {
+      const projectRoot = normalizeRealPath(basePath);
+      const stale = findStaleWorkerForProject(projectRoot);
+      if (stale) return workerToLockData(stale);
+    } catch {
+      // Fall through to the legacy lock-file compatibility path.
+    }
   }
+  return readLegacyLock(basePath);
 }
 
 /**
  * Check whether the process that wrote the lock is still running.
  * Uses `process.kill(pid, 0)` which sends no signal but checks liveness.
  * Returns true if the PID matches our own — we are the lock holder (#2470).
+ *
+ * Unchanged from the file-based era — pure stateless OS check.
  */
 export function isLockProcessAlive(lock: LockData): boolean {
   const pid = lock.pid;
   if (!Number.isInteger(pid) || pid <= 0) return false;
-  // Our own PID means WE hold this lock — we are alive. (#2470)
-  // Callers that need to distinguish "our lock" from "someone else's lock"
-  // (e.g. startAuto checking for a prior crashed session with a recycled PID)
-  // already guard with `crashLock.pid !== process.pid` before calling us.
   if (pid === process.pid) return true;
   try {
     process.kill(pid, 0);
     return true;
   } catch (err) {
-    // EPERM means the process exists but we lack permission — treat as alive.
-    // ESRCH means the process does not exist — treat as dead (stale lock).
     if ((err as NodeJS.ErrnoException).code === "EPERM") return true;
     return false;
   }
@@ -106,7 +241,6 @@ export function formatCrashInfo(lock: LockData): string {
     `  PID: ${lock.pid}`,
   ];
 
-  // Add recovery guidance based on what was happening when it crashed
   if (lock.unitType === "starting" && lock.unitId === "bootstrap") {
     lines.push(`No work was lost. Run /gsd auto to restart.`);
   } else if (lock.unitType.includes("research") || lock.unitType.includes("plan")) {
@@ -122,22 +256,14 @@ export function formatCrashInfo(lock: LockData): string {
 
 /**
  * Emit a synthetic unit-end event for a unit that crashed without emitting its own.
- *
- * Queries the journal to find the most recent unit-start for the crashed unit.
- * If a matching unit-end already exists (e.g. the hard timeout fired), this is a
- * no-op. Called during crash recovery, before clearing the stale lock.
- *
- * Addresses the gap reported in #3348 where `unit-start` was emitted but no
- * `unit-end` followed — side effects landed but the worker died before closeout.
+ * Unchanged from the file era — operates on the journal, not the lock.
  */
 export function emitCrashRecoveredUnitEnd(basePath: string, lock: LockData): void {
-  // Skip bootstrap / starting pseudo-units — they have no meaningful unit-start event.
   if (!lock.unitType || !lock.unitId || lock.unitType === "starting") return;
 
   try {
     const all = queryJournal(basePath);
 
-    // Find the most recent unit-start for this unitId
     const starts = all.filter(
       (e) => e.eventType === "unit-start" && e.data?.unitId === lock.unitId,
     );
@@ -145,7 +271,6 @@ export function emitCrashRecoveredUnitEnd(basePath: string, lock: LockData): voi
 
     const lastStart = starts[starts.length - 1];
 
-    // Check if a unit-end was already emitted (e.g. hard timeout fired after the crash)
     const alreadyClosed = all.some(
       (e) =>
         e.eventType === "unit-end" &&
@@ -155,7 +280,6 @@ export function emitCrashRecoveredUnitEnd(basePath: string, lock: LockData): voi
     );
     if (alreadyClosed) return;
 
-    // Find the highest seq in this flow for monotonic ordering
     const maxSeq = all
       .filter((e) => e.flowId === lastStart.flowId)
       .reduce((max, e) => Math.max(max, e.seq), lastStart.seq);
@@ -174,6 +298,16 @@ export function emitCrashRecoveredUnitEnd(basePath: string, lock: LockData): voi
       causedBy: { flowId: lastStart.flowId, seq: lastStart.seq },
     });
   } catch {
-    // Never throw from crash recovery path — journal failure must not block recovery
+    // Never throw from crash recovery path.
   }
 }
+
+/**
+ * Used by the doctor checks (doctor-runtime-checks.ts, doctor-proactive.ts)
+ * to enumerate stale workers across all projects this DB knows about.
+ * Phase C pt 2 export — surface for the same diagnostics that previously
+ * iterated `auto.lock` files.
+ */
+export function findStaleAutoWorker(basePath: string): LockData | null {
+  return readCrashLock(basePath);
+}

package/src/resources/extensions/gsd/db/auto-workers.ts

@@ -0,0 +1,273 @@
+// gsd-2 + Auto-mode worker process registry (DB-backed coordination, Phase B)
+//
+// IMPORTANT — naming clarification (codex review LOW N1):
+// This module is the AUTO-MODE PROCESS REGISTRY. It tracks long-running
+// `gsd auto` worker processes for cross-process coordination via the shared
+// SQLite WAL. It is NOT the in-process subagent registry, which lives at
+// `src/resources/extensions/subagent/worker-registry.ts` and tracks dispatched
+// subagent threads within a single process.
+//
+// Both modules use the word "worker" but they are unrelated:
+//   - subagent/worker-registry.ts → ephemeral in-process subagent threads
+//   - db/auto-workers.ts          → durable cross-process auto-mode sessions
+//
+// Single-host invariant: SQLite WAL coordination only works on local disk.
+// NFS / network filesystems break heartbeat semantics. Multi-host execution
+// needs a real coordinator (etcd, Postgres) — out of scope for Phase B.
+
+import { randomUUID } from "node:crypto";
+import { hostname } from "node:os";
+
+import {
+  _getAdapter,
+  isDbAvailable,
+  transaction,
+  insertAuditEvent,
+} from "../gsd-db.js";
+import { normalizeRealPath } from "../paths.js";
+
+const HEARTBEAT_TTL_SECONDS = 60;
+// Version label is for diagnostics only — embedded in audit_events and
+// workers.version. Bumping this manually on protocol changes is fine; we
+// don't pull it from package.json to avoid module-load filesystem I/O.
+const WORKER_REGISTRY_VERSION = "1";
+
+export type WorkerStatus = "active" | "stopping" | "crashed";
+
+export interface AutoWorkerRow {
+  worker_id: string;
+  host: string;
+  pid: number;
+  started_at: string;
+  version: string;
+  last_heartbeat_at: string;
+  status: WorkerStatus;
+  project_root_realpath: string;
+}
+
+/**
+ * Register a new auto-mode worker process. Returns the generated worker_id
+ * for the session to store on its AutoSession.
+ *
+ * The worker is created with `status='active'` and an initial heartbeat
+ * stamp; callers must invoke heartbeatAutoWorker() periodically (e.g. once
+ * per loop iteration) to refresh the TTL.
+ */
+export function registerAutoWorker(opts: {
+  projectRootRealpath: string;
+}): string {
+  if (!isDbAvailable()) {
+    throw new Error("registerAutoWorker: DB unavailable");
+  }
+  const workerId = `auto-${hostname()}-${process.pid}-${randomUUID().slice(0, 8)}`;
+  const now = new Date().toISOString();
+
+  transaction(() => {
+    const db = _getAdapter()!;
+    db.prepare(
+      `INSERT INTO workers (
+        worker_id, host, pid, started_at, version,
+        last_heartbeat_at, status, project_root_realpath
+      ) VALUES (
+        :worker_id, :host, :pid, :started_at, :version,
+        :last_heartbeat_at, 'active', :project_root_realpath
+      )`,
+    ).run({
+      ":worker_id": workerId,
+      ":host": hostname(),
+      ":pid": process.pid,
+      ":started_at": now,
+      ":version": WORKER_REGISTRY_VERSION,
+      ":last_heartbeat_at": now,
+      ":project_root_realpath": opts.projectRootRealpath,
+    });
+  });
+
+  insertAuditEvent({
+    eventId: randomUUID(),
+    traceId: workerId,
+    category: "orchestration",
+    type: "worker-registered",
+    ts: now,
+    payload: {
+      workerId,
+      host: hostname(),
+      pid: process.pid,
+      version: WORKER_REGISTRY_VERSION,
+      projectRootRealpath: opts.projectRootRealpath,
+    },
+  });
+
+  return workerId;
+}
+
+/**
+ * Refresh the worker's heartbeat. Call once per auto-loop iteration.
+ * Idempotent — silently no-ops if the worker no longer exists (e.g. row was
+ * cleaned up by a janitor).
+ */
+export function heartbeatAutoWorker(workerId: string): void {
+  if (!isDbAvailable()) return;
+  const now = new Date().toISOString();
+  const db = _getAdapter()!;
+  db.prepare(
+    `UPDATE workers SET last_heartbeat_at = :now WHERE worker_id = :worker_id AND status = 'active'`,
+  ).run({ ":now": now, ":worker_id": workerId });
+}
+
+/**
+ * Mark the worker as crashed. Used by janitors / doctor commands when a
+ * worker's heartbeat has expired beyond the TTL window.
+ */
+export function markWorkerCrashed(workerId: string): void {
+  if (!isDbAvailable()) return;
+  const db = _getAdapter()!;
+  let changes = 0;
+  transaction(() => {
+    const result = db.prepare(
+      `UPDATE workers SET status = 'crashed' WHERE worker_id = :worker_id AND status = 'active'`,
+    ).run({ ":worker_id": workerId });
+    changes =
+      typeof (result as { changes?: unknown }).changes === "number"
+        ? (result as { changes: number }).changes
+        : 0;
+  });
+  if (changes < 1) return;
+  insertAuditEvent({
+    eventId: randomUUID(),
+    traceId: workerId,
+    category: "orchestration",
+    type: "worker-crashed",
+    ts: new Date().toISOString(),
+    payload: { workerId },
+  });
+}
+
+/**
+ * Mark the worker as stopping. Called from the stopAuto path when the user
+ * cleanly shuts down auto-mode.
+ */
+export function markWorkerStopping(workerId: string): void {
+  if (!isDbAvailable()) return;
+  const db = _getAdapter()!;
+  transaction(() => {
+    db.prepare(
+      `UPDATE workers SET status = 'stopping' WHERE worker_id = :worker_id`,
+    ).run({ ":worker_id": workerId });
+  });
+}
+
+/**
+ * Return all workers whose status is 'active' AND whose heartbeat is within
+ * the TTL window. Workers older than the TTL are NOT auto-marked crashed
+ * here — that's a separate janitor responsibility — but they are filtered
+ * out of the active set so callers see a fresh view.
+ */
+export function getActiveAutoWorkers(): readonly AutoWorkerRow[] {
+  if (!isDbAvailable()) return [];
+  const db = _getAdapter()!;
+  const cutoffMs = Date.now() - HEARTBEAT_TTL_SECONDS * 1000;
+  const cutoffIso = new Date(cutoffMs).toISOString();
+  const rows = db.prepare(
+    `SELECT worker_id, host, pid, started_at, version,
+            last_heartbeat_at, status, project_root_realpath
+     FROM workers
+     WHERE status = 'active' AND last_heartbeat_at >= :cutoff
+     ORDER BY started_at`,
+  ).all({ ":cutoff": cutoffIso }) as unknown as AutoWorkerRow[];
+  return rows;
+}
+
+/** Return all worker rows regardless of status or TTL. */
+export function getAllAutoWorkers(): readonly AutoWorkerRow[] {
+  if (!isDbAvailable()) return [];
+  const db = _getAdapter()!;
+  const rows = db.prepare(
+    `SELECT worker_id, host, pid, started_at, version,
+            last_heartbeat_at, status, project_root_realpath
+     FROM workers
+     ORDER BY started_at`,
+  ).all() as unknown as AutoWorkerRow[];
+  return rows;
+}
+
+/**
+ * Look up a single worker row. Returns null if no row exists.
+ */
+export function getAutoWorker(workerId: string): AutoWorkerRow | null {
+  if (!isDbAvailable()) return null;
+  const db = _getAdapter()!;
+  const row = db.prepare(
+    `SELECT worker_id, host, pid, started_at, version,
+            last_heartbeat_at, status, project_root_realpath
+     FROM workers WHERE worker_id = :worker_id`,
+  ).get({ ":worker_id": workerId }) as AutoWorkerRow | undefined;
+  return row ?? null;
+}
+
+/** Test/janitor helper: TTL constant exported for callers to compute expirations. */
+export function autoWorkerHeartbeatTtlSeconds(): number {
+  return HEARTBEAT_TTL_SECONDS;
+}
+
+function isWorkerProcessAlive(candidate: Pick<AutoWorkerRow, "host" | "pid">): boolean {
+  const pid = candidate.pid;
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  if (candidate.host !== hostname()) return false;
+  if (pid === process.pid) return true;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "EPERM") return true;
+    return false;
+  }
+}
+
+/**
+ * Phase C pt 2 — find the most recently active worker for a project root
+ * whose heartbeat has lapsed (the "previous crashed session" indicator).
+ *
+ * Used by crash-recovery.ts:readCrashLock to detect when a prior auto-mode
+ * session ended without cleanup. Workers are only treated as stale after
+ * their heartbeat has lapsed and the OS PID liveness check says the process
+ * is no longer alive.
+ *
+ * Returns null if no stale worker exists for this project root.
+ */
+export function findStaleWorkerForProject(
+  projectRootRealpath: string,
+): AutoWorkerRow | null {
+  if (!isDbAvailable()) return null;
+  const db = _getAdapter()!;
+  const cutoffMs = Date.now() - HEARTBEAT_TTL_SECONDS * 1000;
+  const cutoffIso = new Date(cutoffMs).toISOString();
+  const row = db.prepare(
+    `SELECT worker_id, host, pid, started_at, version,
+            last_heartbeat_at, status, project_root_realpath
+     FROM workers
+     WHERE project_root_realpath = :project_root
+       AND status = 'active'
+       AND last_heartbeat_at < :cutoff
+     ORDER BY started_at DESC
+     LIMIT 1`,
+  ).get({ ":project_root": projectRootRealpath, ":cutoff": cutoffIso }) as AutoWorkerRow | undefined;
+  if (row && !isWorkerProcessAlive(row)) return row;
+
+  // Older rows and external fixtures may have captured a non-realpath spelling
+  // of the same project root, e.g. /var/... vs /private/var/... on macOS.
+  const canonicalProjectRoot = normalizeRealPath(projectRootRealpath);
+  const staleRows = db.prepare(
+    `SELECT worker_id, host, pid, started_at, version,
+            last_heartbeat_at, status, project_root_realpath
+     FROM workers
+     WHERE status = 'active'
+       AND last_heartbeat_at < :cutoff
+     ORDER BY started_at DESC`,
+  ).all({ ":cutoff": cutoffIso }) as unknown as AutoWorkerRow[];
+  return staleRows.find(
+    (candidate) =>
+      normalizeRealPath(candidate.project_root_realpath) === canonicalProjectRoot
+      && !isWorkerProcessAlive(candidate),
+  ) ?? null;
+}