gsd-pi 2.78.1-dev.e9d88a536 → 2.78.1-dev.eccf86e27

package/src/resources/extensions/gsd/db/runtime-kv.ts

@@ -0,0 +1,127 @@
+// gsd-2 + Non-correctness-critical key-value storage (Phase C — file-state migration)
+//
+// STRICT INVARIANT (re-stated from gsd-db.ts createRuntimeKvTableV25):
+// runtime_kv is for SOFT state only. UI cursors, dashboard caches,
+// last-seen-version markers, resume cursors, and similar values that
+// can be lost without breaking auto-mode correctness.
+//
+// Anything that drives the auto-loop's control flow MUST get typed
+// columns in unit_dispatches / workers / milestone_leases — never a
+// bag of JSON in runtime_kv. The reviewer's smell test: if losing the
+// row would cause the loop to reorder, double-execute, or stuck-loop,
+// it does NOT belong here.
+//
+// Single-host invariant: SQLite WAL coordination, local disk only.
+// See db/auto-workers.ts for the same constraint applied to coordination.
+
+import {
+  _getAdapter,
+  isDbAvailable,
+  transaction,
+} from "../gsd-db.js";
+
+export type RuntimeKvScope = "global" | "worker" | "milestone";
+
+export interface RuntimeKvRow {
+  scope: RuntimeKvScope;
+  scope_id: string;
+  key: string;
+  value_json: string;
+  updated_at: string;
+}
+
+/**
+ * Set or update a runtime_kv row. The value is JSON-stringified before
+ * storage. Best-effort — silently no-ops when the DB is unavailable.
+ */
+export function setRuntimeKv(
+  scope: RuntimeKvScope,
+  scopeId: string,
+  key: string,
+  value: unknown,
+): void {
+  if (!isDbAvailable()) return;
+  const now = new Date().toISOString();
+  const db = _getAdapter()!;
+  let valueJson: string;
+  try {
+    valueJson = JSON.stringify(value);
+  } catch {
+    valueJson = JSON.stringify(String(value));
+  }
+  if (valueJson === undefined) {
+    valueJson = JSON.stringify(null);
+  }
+  transaction(() => {
+    db.prepare(
+      `INSERT INTO runtime_kv (scope, scope_id, key, value_json, updated_at)
+       VALUES (:scope, :scope_id, :key, :value_json, :updated_at)
+       ON CONFLICT (scope, scope_id, key) DO UPDATE SET
+         value_json = excluded.value_json,
+         updated_at = excluded.updated_at`,
+    ).run({
+      ":scope": scope,
+      ":scope_id": scopeId,
+      ":key": key,
+      ":value_json": valueJson,
+      ":updated_at": now,
+    });
+  });
+}
+
+/**
+ * Read a runtime_kv value, parsed from JSON. Returns null if the row
+ * doesn't exist or the DB is unavailable.
+ */
+export function getRuntimeKv<T = unknown>(
+  scope: RuntimeKvScope,
+  scopeId: string,
+  key: string,
+): T | null {
+  if (!isDbAvailable()) return null;
+  const db = _getAdapter()!;
+  const row = db.prepare(
+    `SELECT value_json FROM runtime_kv
+     WHERE scope = :scope AND scope_id = :scope_id AND key = :key`,
+  ).get({ ":scope": scope, ":scope_id": scopeId, ":key": key }) as { value_json: string } | undefined;
+  if (!row) return null;
+  try {
+    return JSON.parse(row.value_json) as T;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Delete a runtime_kv row. Idempotent — silently no-ops when the row
+ * doesn't exist or the DB is unavailable.
+ */
+export function deleteRuntimeKv(
+  scope: RuntimeKvScope,
+  scopeId: string,
+  key: string,
+): void {
+  if (!isDbAvailable()) return;
+  const db = _getAdapter()!;
+  db.prepare(
+    `DELETE FROM runtime_kv WHERE scope = :scope AND scope_id = :scope_id AND key = :key`,
+  ).run({ ":scope": scope, ":scope_id": scopeId, ":key": key });
+}
+
+/**
+ * List all rows within a (scope, scopeId) bucket. Useful for diagnostics
+ * and bulk migrations.
+ */
+export function listRuntimeKv(
+  scope: RuntimeKvScope,
+  scopeId: string,
+): readonly RuntimeKvRow[] {
+  if (!isDbAvailable()) return [];
+  const db = _getAdapter()!;
+  return db.prepare(
+    `SELECT scope, scope_id, key, value_json, updated_at
+     FROM runtime_kv
+     WHERE scope = :scope AND scope_id = :scope_id
+     ORDER BY key`,
+  ).all({ ":scope": scope, ":scope_id": scopeId }) as unknown as RuntimeKvRow[];
+}

package/src/resources/extensions/gsd/db/unit-dispatches.ts

@@ -0,0 +1,446 @@
+// gsd-2 + Unit dispatch ledger (DB-backed coordination, Phase B)
+//
+// Records every auto-mode unit dispatch (plan-slice, run-task, summarize, …)
+// with worker_id, fencing token, status lifecycle, and retry metadata. The
+// ledger is the substrate Phase C will consume to migrate stuck-state.json
+// and paused-session.json out of the runtime/ directory.
+//
+// Codex review MEDIUM B2: partial unique index
+//   idx_unit_dispatches_active_per_unit ON unit_dispatches(unit_id)
+//   WHERE status IN ('claimed','running')
+// enforces that two workers cannot simultaneously claim the same unit.
+// recordDispatchClaim relies on the index to fail fast at INSERT time
+// rather than racing in application code.
+
+import { randomUUID } from "node:crypto";
+
+import {
+  _getAdapter,
+  isDbAvailable,
+  transaction,
+  insertAuditEvent,
+} from "../gsd-db.js";
+
+export type DispatchStatus =
+  | "pending"
+  | "claimed"
+  | "running"
+  | "completed"
+  | "failed"
+  | "stuck"
+  | "canceled"
+  | "paused";
+
+export interface UnitDispatchRow {
+  id: number;
+  trace_id: string;
+  turn_id: string | null;
+  worker_id: string;
+  milestone_lease_token: number;
+  milestone_id: string;
+  slice_id: string | null;
+  task_id: string | null;
+  unit_type: string;
+  unit_id: string;
+  status: DispatchStatus;
+  attempt_n: number;
+  started_at: string;
+  ended_at: string | null;
+  exit_reason: string | null;
+  error_summary: string | null;
+  verification_evidence_id: number | null;
+  next_run_at: string | null;
+  retry_after_ms: number | null;
+  max_attempts: number;
+  last_error_code: string | null;
+  last_error_at: string | null;
+}
+
+export interface RecordClaimInput {
+  traceId: string;
+  turnId?: string | null;
+  workerId: string;
+  milestoneLeaseToken: number;
+  milestoneId: string;
+  sliceId?: string | null;
+  taskId?: string | null;
+  unitType: string;
+  unitId: string;
+  /**
+   * Attempt number for this unit. Callers should compute this from the
+   * most recent prior dispatch for the same unit_id (use
+   * getRecentForUnit() then add 1). Defaults to 1 for fresh claims.
+   */
+  attemptN?: number;
+  /** Per-attempt cap; defaults to 3. */
+  maxAttempts?: number;
+}
+
+export type RecordClaimResult =
+  | { ok: true; dispatchId: number }
+  | { ok: false; error: "already_active"; existingId: number; existingStatus: DispatchStatus; existingWorker: string }
+  | { ok: false; error: "stale_lease"; milestoneId: string; workerId: string; milestoneLeaseToken: number };
+
+function isAlreadyActiveConstraintError(err: unknown): boolean {
+  const code =
+    err && typeof err === "object" && "code" in err
+      ? String((err as { code?: unknown }).code ?? "")
+      : "";
+  const msg = err instanceof Error ? err.message : String(err);
+  if (/\bFOREIGN KEY\b/i.test(msg)) {
+    return false;
+  }
+
+  if (code === "SQLITE_CONSTRAINT" || code === "SQLITE_CONSTRAINT_UNIQUE") {
+    return true;
+  }
+
+  return /\bUNIQUE\b|\bconstraint failed\b/i.test(msg);
+}
+
+/**
+ * Insert a new dispatch row in `claimed` state. Atomic guard against
+ * double-claim (B2): the partial unique index
+ * idx_unit_dispatches_active_per_unit refuses the INSERT if any row for
+ * the same unit_id already has status IN ('claimed','running').
+ */
+export function recordDispatchClaim(input: RecordClaimInput): RecordClaimResult {
+  if (!isDbAvailable()) {
+    throw new Error("recordDispatchClaim: DB unavailable");
+  }
+  const now = new Date().toISOString();
+
+  return transaction((): RecordClaimResult => {
+    const db = _getAdapter()!;
+
+    const lease = db.prepare(
+      `SELECT fencing_token
+       FROM milestone_leases
+       WHERE milestone_id = :milestone_id
+         AND worker_id = :worker_id
+         AND fencing_token = :token
+         AND status = 'held'`,
+    ).get({
+      ":milestone_id": input.milestoneId,
+      ":worker_id": input.workerId,
+      ":token": input.milestoneLeaseToken,
+    }) as { fencing_token: number } | undefined;
+    if (!lease) {
+      return {
+        ok: false,
+        error: "stale_lease",
+        milestoneId: input.milestoneId,
+        workerId: input.workerId,
+        milestoneLeaseToken: input.milestoneLeaseToken,
+      };
+    }
+
+    try {
+      const result = db.prepare(
+        `INSERT INTO unit_dispatches (
+          trace_id, turn_id, worker_id, milestone_lease_token,
+          milestone_id, slice_id, task_id,
+          unit_type, unit_id, status, attempt_n,
+          started_at, max_attempts
+        ) VALUES (
+          :trace_id, :turn_id, :worker_id, :milestone_lease_token,
+          :milestone_id, :slice_id, :task_id,
+          :unit_type, :unit_id, 'claimed', :attempt_n,
+          :started_at, :max_attempts
+        )`,
+      ).run({
+        ":trace_id": input.traceId,
+        ":turn_id": input.turnId ?? null,
+        ":worker_id": input.workerId,
+        ":milestone_lease_token": input.milestoneLeaseToken,
+        ":milestone_id": input.milestoneId,
+        ":slice_id": input.sliceId ?? null,
+        ":task_id": input.taskId ?? null,
+        ":unit_type": input.unitType,
+        ":unit_id": input.unitId,
+        ":attempt_n": input.attemptN ?? 1,
+        ":started_at": now,
+        ":max_attempts": input.maxAttempts ?? 3,
+      });
+      const id = Number((result as { lastInsertRowid?: number | bigint }).lastInsertRowid ?? 0);
+
+      insertAuditEvent({
+        eventId: randomUUID(),
+        traceId: input.traceId,
+        turnId: input.turnId ?? undefined,
+        category: "orchestration",
+        type: "dispatch-claimed",
+        ts: now,
+        payload: {
+          dispatchId: id,
+          unitId: input.unitId,
+          unitType: input.unitType,
+          workerId: input.workerId,
+          attemptN: input.attemptN ?? 1,
+        },
+      });
+
+      return { ok: true, dispatchId: id };
+    } catch (err) {
+      if (!isAlreadyActiveConstraintError(err)) throw err;
+
+      // Partial unique index rejected the INSERT — surface the existing
+      // active dispatch so callers can decide what to do.
+      const existing = db.prepare(
+        `SELECT id, status, worker_id FROM unit_dispatches
+         WHERE unit_id = :unit_id AND status IN ('claimed','running')
+         ORDER BY id DESC LIMIT 1`,
+      ).get({ ":unit_id": input.unitId }) as { id: number; status: DispatchStatus; worker_id: string } | undefined;
+
+      return {
+        ok: false,
+        error: "already_active",
+        existingId: existing?.id ?? 0,
+        existingStatus: existing?.status ?? "claimed",
+        existingWorker: existing?.worker_id ?? "unknown",
+      };
+    }
+  });
+}
+
+/** Transition a `claimed` dispatch into `running`. */
+export function markRunning(dispatchId: number): void {
+  if (!isDbAvailable()) return;
+  const db = _getAdapter()!;
+  db.prepare(
+    `UPDATE unit_dispatches SET status = 'running'
+     WHERE id = :id AND status = 'claimed'`,
+  ).run({ ":id": dispatchId });
+}
+
+export interface CompleteOpts {
+  verificationEvidenceId?: number | null;
+  exitReason?: string;
+}
+
+/** Transition a dispatch into `completed`. */
+export function markCompleted(dispatchId: number, opts?: CompleteOpts): void {
+  if (!isDbAvailable()) return;
+  const now = new Date().toISOString();
+  const db = _getAdapter()!;
+  let changes = 0;
+  transaction(() => {
+    const result = db.prepare(
+      `UPDATE unit_dispatches
+       SET status = 'completed', ended_at = :ended_at,
+           exit_reason = :exit_reason,
+           verification_evidence_id = :evidence_id
+       WHERE id = :id
+         AND status IN ('claimed','running')`,
+    ).run({
+      ":id": dispatchId,
+      ":ended_at": now,
+      ":exit_reason": opts?.exitReason ?? null,
+      ":evidence_id": opts?.verificationEvidenceId ?? null,
+    });
+    changes =
+      typeof (result as { changes?: unknown }).changes === "number"
+        ? (result as { changes: number }).changes
+        : 0;
+  });
+  if (changes < 1) return;
+  insertAuditEvent({
+    eventId: randomUUID(),
+    traceId: dispatchId.toString(),
+    category: "orchestration",
+    type: "dispatch-completed",
+    ts: now,
+    payload: { dispatchId },
+  });
+}
+
+export interface FailureOpts {
+  errorSummary: string;
+  errorCode?: string;
+  /** Backoff before next attempt (used by stuck-detector retry suppression). */
+  retryAfterMs?: number;
+}
+
+/** Transition a dispatch into `failed`, optionally scheduling a retry. */
+export function markFailed(dispatchId: number, opts: FailureOpts): void {
+  if (!isDbAvailable()) return;
+  const now = new Date();
+  const nowIso = now.toISOString();
+  const nextRunIso = opts.retryAfterMs
+    ? new Date(now.getTime() + opts.retryAfterMs).toISOString()
+    : null;
+  const db = _getAdapter()!;
+  let changes = 0;
+  transaction(() => {
+    const result = db.prepare(
+      `UPDATE unit_dispatches
+       SET status = 'failed', ended_at = :ended_at,
+           error_summary = :error_summary,
+           last_error_code = :last_error_code,
+           last_error_at = :last_error_at,
+           retry_after_ms = :retry_after_ms,
+           next_run_at = :next_run_at
+       WHERE id = :id
+         AND status IN ('claimed','running')`,
+    ).run({
+      ":id": dispatchId,
+      ":ended_at": nowIso,
+      ":error_summary": opts.errorSummary,
+      ":last_error_code": opts.errorCode ?? null,
+      ":last_error_at": nowIso,
+      ":retry_after_ms": opts.retryAfterMs ?? null,
+      ":next_run_at": nextRunIso,
+    });
+    changes =
+      typeof (result as { changes?: unknown }).changes === "number"
+        ? (result as { changes: number }).changes
+        : 0;
+  });
+  if (changes < 1) return;
+  insertAuditEvent({
+    eventId: randomUUID(),
+    traceId: dispatchId.toString(),
+    category: "orchestration",
+    type: "dispatch-failed",
+    ts: nowIso,
+    payload: { dispatchId, errorSummary: opts.errorSummary, retryAfterMs: opts.retryAfterMs ?? null },
+  });
+}
+
+/** Transition a dispatch into `stuck`. */
+export function markStuck(dispatchId: number, reason: string): void {
+  if (!isDbAvailable()) return;
+  const now = new Date().toISOString();
+  const db = _getAdapter()!;
+  const result = transaction(() => {
+    return db.prepare(
+      `UPDATE unit_dispatches
+       SET status = 'stuck', ended_at = :ended_at, exit_reason = :reason
+       WHERE id = :id
+         AND status IN ('claimed','running')`,
+    ).run({ ":id": dispatchId, ":ended_at": now, ":reason": reason });
+  });
+  const changes =
+    typeof (result as { changes?: unknown }).changes === "number"
+      ? (result as { changes: number }).changes
+      : 0;
+  if (changes <= 0) return;
+  insertAuditEvent({
+    eventId: randomUUID(),
+    traceId: dispatchId.toString(),
+    category: "orchestration",
+    type: "dispatch-stuck",
+    ts: now,
+    payload: { dispatchId, reason },
+  });
+}
+
+/** Transition a dispatch into `paused`. */
+export function markPaused(dispatchId: number): void {
+  if (!isDbAvailable()) return;
+  const now = new Date().toISOString();
+  const db = _getAdapter()!;
+  db.prepare(
+    `UPDATE unit_dispatches
+     SET status = 'paused', ended_at = :ended_at
+     WHERE id = :id AND status IN ('claimed','running')`,
+  ).run({ ":id": dispatchId, ":ended_at": now });
+}
+
+/** Transition a dispatch into `canceled`. */
+export function markCanceled(dispatchId: number, reason: string): void {
+  if (!isDbAvailable()) return;
+  const now = new Date().toISOString();
+  const db = _getAdapter()!;
+  db.prepare(
+    `UPDATE unit_dispatches
+     SET status = 'canceled', ended_at = :ended_at, exit_reason = :reason
+     WHERE id = :id AND status IN ('pending','claimed','running')`,
+  ).run({ ":id": dispatchId, ":ended_at": now, ":reason": reason });
+}
+
+/**
+ * Fetch the most recent N dispatches for a unit. Used by recordDispatchClaim
+ * callers to compute attempt_n and by detect-stuck.ts (B3) to consult
+ * retry budget before tripping the stuck verdict.
+ */
+export function getRecentForUnit(unitId: string, limit = 10): UnitDispatchRow[] {
+  if (!isDbAvailable()) return [];
+  const db = _getAdapter()!;
+  return db.prepare(
+    `SELECT * FROM unit_dispatches WHERE unit_id = :unit_id ORDER BY id DESC LIMIT :limit`,
+  ).all({ ":unit_id": unitId, ":limit": limit }) as unknown as UnitDispatchRow[];
+}
+
+/**
+ * Fetch the latest dispatch for a unit, regardless of status. Returns null
+ * if the unit has never been dispatched.
+ */
+export function getLatestForUnit(unitId: string): UnitDispatchRow | null {
+  if (!isDbAvailable()) return null;
+  const db = _getAdapter()!;
+  const row = db.prepare(
+    `SELECT * FROM unit_dispatches WHERE unit_id = :unit_id ORDER BY id DESC LIMIT 1`,
+  ).get({ ":unit_id": unitId }) as UnitDispatchRow | undefined;
+  return row ?? null;
+}
+
+/**
+ * Phase C — return the most recent unit_id values for a worker, oldest-first.
+ *
+ * Drop-in replacement for the persistence side of stuck-state.json's
+ * `recentUnits` field. The auto-loop uses this to seed loopState.recentUnits
+ * on session start so the stuck-detector window survives a session restart
+ * (#3704). Returned in oldest-first order to match the in-memory window
+ * shape that detect-stuck.ts expects.
+ */
+export function getRecentUnitKeysForWorker(
+  workerId: string,
+  limit = 20,
+): Array<{ key: string }> {
+  if (!isDbAvailable()) return [];
+  const db = _getAdapter()!;
+  const rows = db.prepare(
+    `SELECT unit_id FROM unit_dispatches
+     WHERE worker_id = :worker_id
+     ORDER BY started_at DESC, id DESC
+     LIMIT :limit`,
+  ).all({ ":worker_id": workerId, ":limit": limit }) as Array<{ unit_id: string }>;
+  // Reverse so callers consume oldest-first (sliding-window semantics).
+  return rows.reverse().map((r) => ({ key: r.unit_id }));
+}
+
+export function getRecentUnitKeysForProjectRoot(
+  projectRootRealpath: string,
+  limit = 20,
+): Array<{ key: string }> {
+  if (!isDbAvailable()) return [];
+  const db = _getAdapter()!;
+  const rows = db.prepare(
+    `SELECT ud.unit_id
+     FROM unit_dispatches ud
+     INNER JOIN workers w ON w.worker_id = ud.worker_id
+     WHERE w.project_root_realpath = :project_root_realpath
+     ORDER BY ud.started_at DESC, ud.id DESC
+     LIMIT :limit`,
+  ).all({
+    ":project_root_realpath": projectRootRealpath,
+    ":limit": limit,
+  }) as Array<{ unit_id: string }>;
+  return rows.reverse().map((r) => ({ key: r.unit_id }));
+}
+
+/**
+ * Fetch dispatches for a milestone filtered by status. Useful for janitors
+ * + dashboards.
+ */
+export function getDispatchesByStatus(
+  milestoneId: string,
+  status: DispatchStatus,
+): UnitDispatchRow[] {
+  if (!isDbAvailable()) return [];
+  const db = _getAdapter()!;
+  return db.prepare(
+    `SELECT * FROM unit_dispatches WHERE milestone_id = :mid AND status = :status ORDER BY id`,
+  ).all({ ":mid": milestoneId, ":status": status }) as unknown as UnitDispatchRow[];
+}