gsd-pi 2.77.0-dev.eaa4973bc → 2.78.0-dev.aeeb2ca00

package/src/resources/extensions/gsd/slice-parallel-orchestrator.ts

@@ -13,13 +13,16 @@
  * - Conflict check: file overlap between slice plans (slice-parallel-conflict.ts)
  */
 
-import { spawn, type ChildProcess } from "node:child_process";
+import { execFileSync, spawn, type ChildProcess } from "node:child_process";
+import { randomUUID } from "node:crypto";
 import {
   appendFileSync,
   existsSync,
   writeFileSync,
   readFileSync,
   mkdirSync,
+  renameSync,
+  unlinkSync,
 } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -40,6 +43,8 @@ export interface SliceWorkerInfo {
   milestoneId: string;
   sliceId: string;
   pid: number;
+  workerToken: string;
+  processStartFingerprint: string | null;
   process: ChildProcess | null;
   worktreePath: string;
   startedAt: number;
@@ -69,13 +74,276 @@ export interface StartSliceParallelOpts {
 
 let sliceState: SliceOrchestratorState | null = null;
 
+// ─── Persisted State (crash recovery) ──────────────────────────────────────
+//
+// Mirrors parallel-orchestrator.ts. Without persistence, a coordinator crash
+// leaves orphaned worktrees on disk with no way to detect or clean them up
+// on next session start. (Issue #4980 HIGH-8)
+
+const SLICE_ORCHESTRATOR_STATE_FILE = "slice-orchestrator.json";
+const TMP_SUFFIX = ".tmp";
+
+interface PersistedSliceWorker {
+  milestoneId: string;
+  sliceId: string;
+  pid: number;
+  workerToken?: string;
+  processStartFingerprint?: string | null;
+  worktreePath: string;
+  startedAt: number;
+  state: "running" | "stopped" | "error";
+  completedUnits: number;
+  cost: number;
+}
+
+interface PersistedSliceState {
+  active: boolean;
+  workers: PersistedSliceWorker[];
+  totalCost: number;
+  budgetCeiling?: number;
+  maxWorkers: number;
+  startedAt: number;
+  basePath: string;
+}
+
+function sliceStateFilePath(basePath: string): string {
+  return join(gsdRoot(basePath), SLICE_ORCHESTRATOR_STATE_FILE);
+}
+
+function isPidAlive(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "EPERM") return true;
+    return false;
+  }
+}
+
+function readLinuxProcessStartFingerprint(pid: number): string | null {
+  try {
+    const stat = readFileSync(`/proc/${pid}/stat`, "utf-8");
+    const afterCommand = stat.slice(stat.lastIndexOf(")") + 2).trim();
+    const fields = afterCommand.split(/\s+/);
+    const startTimeTicks = fields[19];
+    return startTimeTicks ? `linux-stat:${startTimeTicks}` : null;
+  } catch {
+    return null;
+  }
+}
+
+function readPsProcessStartFingerprint(pid: number): string | null {
+  try {
+    const raw = execFileSync("ps", ["-p", String(pid), "-o", "lstart="], {
+      stdio: ["ignore", "pipe", "ignore"],
+      encoding: "utf-8",
+    }).trim().replace(/\s+/g, " ");
+    return raw ? `ps-lstart:${raw}` : null;
+  } catch {
+    return null;
+  }
+}
+
+function readProcessStartFingerprint(pid: number): string | null {
+  if (!Number.isInteger(pid) || pid <= 0) return null;
+  return readLinuxProcessStartFingerprint(pid) ?? readPsProcessStartFingerprint(pid);
+}
+
+function linuxProcessEnvContains(pid: number, key: string, value: string): boolean | null {
+  if (process.platform !== "linux") return null;
+  try {
+    const env = readFileSync(`/proc/${pid}/environ`, "utf-8");
+    return env.split("\0").includes(`${key}=${value}`);
+  } catch {
+    return null;
+  }
+}
+
+function createWorkerToken(milestoneId: string, sliceId: string): string {
+  return `slice:${milestoneId}:${sliceId}:${Date.now()}:${randomUUID()}`;
+}
+
+function isRecoveredSliceWorkerAlive(worker: {
+  pid: number;
+  workerToken?: string;
+  processStartFingerprint?: string | null;
+}): boolean {
+  if (!isPidAlive(worker.pid)) return false;
+  if (!worker.processStartFingerprint) return false;
+
+  const currentFingerprint = readProcessStartFingerprint(worker.pid);
+  if (!currentFingerprint || currentFingerprint !== worker.processStartFingerprint) {
+    return false;
+  }
+
+  if (worker.workerToken) {
+    const envMatches = linuxProcessEnvContains(
+      worker.pid,
+      "GSD_SLICE_WORKER_TOKEN",
+      worker.workerToken,
+    );
+    if (envMatches === false) return false;
+  }
+
+  return true;
+}
+
+/**
+ * Persist current slice orchestrator state. Atomic write (tmp + rename) to
+ * prevent partial reads if the coordinator dies mid-write.
+ */
+function persistSliceState(): void {
+  if (!sliceState) return;
+  try {
+    const dir = gsdRoot(sliceState.basePath);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+
+    const persisted: PersistedSliceState = {
+      active: sliceState.active,
+      workers: [...sliceState.workers.values()].map((w) => ({
+        milestoneId: w.milestoneId,
+        sliceId: w.sliceId,
+        pid: w.pid,
+        workerToken: w.workerToken,
+        processStartFingerprint: w.processStartFingerprint,
+        worktreePath: w.worktreePath,
+        startedAt: w.startedAt,
+        state: w.state,
+        completedUnits: w.completedUnits,
+        cost: w.cost,
+      })),
+      totalCost: sliceState.totalCost,
+      budgetCeiling: sliceState.budgetCeiling,
+      maxWorkers: sliceState.maxWorkers,
+      startedAt: sliceState.startedAt,
+      basePath: sliceState.basePath,
+    };
+
+    const dest = sliceStateFilePath(sliceState.basePath);
+    const tmp = dest + TMP_SUFFIX;
+    writeFileSync(tmp, JSON.stringify(persisted, null, 2), "utf-8");
+    renameSync(tmp, dest);
+    lastPersistTs = Date.now();
+  } catch {
+    /* non-fatal: persistence is best-effort */
+  }
+}
+
+/**
+ * Throttled wrapper around `persistSliceState`. Skips if the last successful
+ * persist was less than `PERSIST_THROTTLE_MS` ago; otherwise persists
+ * immediately. Use this on hot paths (e.g. `message_end` events) where we
+ * receive many events per second per worker. Terminal events (worker exit,
+ * crash, stop) should call `persistSliceState()` directly to guarantee the
+ * final state hits disk regardless of timing.
+ */
+const PERSIST_THROTTLE_MS = 1000;
+let lastPersistTs = 0;
+
+function persistSliceStateThrottled(): void {
+  if (Date.now() - lastPersistTs < PERSIST_THROTTLE_MS) return;
+  persistSliceState();
+}
+
+function removeSliceStateFile(basePath: string): void {
+  try {
+    const p = sliceStateFilePath(basePath);
+    if (existsSync(p)) unlinkSync(p);
+  } catch {
+    /* non-fatal */
+  }
+}
+
+/**
+ * Restore slice orchestrator state from disk. Filters dead-PID workers and
+ * removes their orphaned worktrees so a clean restart is possible.
+ *
+ * Returns null if no state file exists or no workers survive.
+ */
+export function restoreSliceState(basePath: string): PersistedSliceState | null {
+  try {
+    const p = sliceStateFilePath(basePath);
+    if (!existsSync(p)) return null;
+    const persisted = JSON.parse(readFileSync(p, "utf-8")) as PersistedSliceState;
+
+    const survivors: PersistedSliceWorker[] = [];
+    const dead: PersistedSliceWorker[] = [];
+    for (const w of persisted.workers) {
+      if (w.state === "running" && isRecoveredSliceWorkerAlive(w)) {
+        survivors.push(w);
+      } else if (w.state === "running") {
+        dead.push(w);
+      } else {
+        survivors.push(w);
+      }
+    }
+
+    // Best-effort cleanup of orphaned worktrees from dead workers.
+    for (const w of dead) {
+      const wtName = `${w.milestoneId}-${w.sliceId}`;
+      try {
+        removeWorktree(persisted.basePath, wtName, { deleteBranch: true, force: true });
+      } catch {
+        /* worktree may already be gone */
+      }
+    }
+
+    persisted.workers = survivors;
+
+    if (survivors.length === 0) {
+      removeSliceStateFile(basePath);
+      return null;
+    }
+
+    return persisted;
+  } catch {
+    return null;
+  }
+}
+
 // ─── Public API ────────────────────────────────────────────────────────────
 
 /**
  * Check whether slice-level parallel is currently active.
+ *
+ * If in-memory state is unset but a persisted state file exists with at
+ * least one live-PID worker, treat as active and rehydrate so a coordinator
+ * crash followed by a fresh process is detectable. (Issue #4980 HIGH-8)
  */
-export function isSliceParallelActive(): boolean {
-  return sliceState?.active === true;
+export function isSliceParallelActive(basePath?: string): boolean {
+  if (sliceState?.active === true) return true;
+  if (!basePath) return false;
+  const restored = restoreSliceState(basePath);
+  if (!restored || restored.workers.length === 0) return false;
+
+  // Rehydrate in-memory state from disk; processes are detached so we have
+  // no ChildProcess handles, only PIDs.
+  sliceState = {
+    active: restored.active,
+    workers: new Map(),
+    totalCost: restored.totalCost,
+    budgetCeiling: restored.budgetCeiling,
+    maxWorkers: restored.maxWorkers,
+    startedAt: restored.startedAt,
+    basePath: restored.basePath,
+  };
+  for (const w of restored.workers) {
+    sliceState.workers.set(w.sliceId, {
+      milestoneId: w.milestoneId,
+      sliceId: w.sliceId,
+      pid: w.pid,
+      process: null,
+      worktreePath: w.worktreePath,
+      workerToken: w.workerToken ?? "",
+      processStartFingerprint: w.processStartFingerprint ?? null,
+      startedAt: w.startedAt,
+      state: w.state,
+      completedUnits: w.completedUnits,
+      cost: w.cost,
+    });
+  }
+  return true;
 }
 
 /**
@@ -146,6 +414,8 @@ export async function startSliceParallel(
         milestoneId,
         sliceId: slice.id,
         pid: 0,
+        workerToken: createWorkerToken(milestoneId, slice.id),
+        processStartFingerprint: null,
         process: null,
         worktreePath: wtPath,
         startedAt: Date.now(),
@@ -162,12 +432,16 @@ export async function startSliceParallel(
         started.push(slice.id);
       } else {
         errors.push({ sid: slice.id, error: "Failed to spawn worker process" });
-        worker.state = "error";
+        sliceState.workers.delete(slice.id);
+        try {
+          removeWorktree(basePath, wtName, { deleteBranch: true, force: true });
+        } catch { /* ignore cleanup failures */ }
       }
     } catch (err) {
       errors.push({ sid: slice.id, error: getErrorMessage(err) });
       // Best-effort cleanup of partially created worktree
       const wtName = `${milestoneId}-${slice.id}`;
+      sliceState.workers.delete(slice.id);
       try {
         removeWorktree(basePath, wtName, { deleteBranch: true, force: true });
       } catch { /* ignore cleanup failures */ }
@@ -177,6 +451,10 @@ export async function startSliceParallel(
   // If nothing started, deactivate
   if (started.length === 0) {
     sliceState.active = false;
+    removeSliceStateFile(basePath);
+  } else {
+    // Persist state for crash recovery (Issue #4980 HIGH-8).
+    persistSliceState();
   }
 
   return { started, errors };
@@ -187,13 +465,16 @@ export async function startSliceParallel(
  */
 export function stopSliceParallel(): void {
   if (!sliceState) return;
+  const basePath = sliceState.basePath;
 
   for (const worker of sliceState.workers.values()) {
-    if (worker.process) {
-      try {
+    try {
+      if (worker.process) {
         worker.process.kill("SIGTERM");
-      } catch { /* already dead */ }
-    }
+      } else if (worker.state === "running" && isRecoveredSliceWorkerAlive(worker)) {
+        process.kill(worker.pid, "SIGTERM");
+      }
+    } catch { /* already dead */ }
     worker.cleanup?.();
     worker.cleanup = undefined;
     worker.process = null;
@@ -207,6 +488,9 @@ export function stopSliceParallel(): void {
   }
 
   sliceState.active = false;
+  // Clear persisted state — clean shutdown means no recovery on next start.
+  // (Issue #4980 HIGH-8)
+  removeSliceStateFile(basePath);
 }
 
 /**
@@ -239,6 +523,7 @@ export function resetSliceOrchestrator(): void {
     }
   }
   sliceState = null;
+  lastPersistTs = 0;
 }
 
 // ─── Internal: Conflict Filtering ──────────────────────────────────────────
@@ -339,6 +624,7 @@ function spawnSliceWorker(
         GSD_MILESTONE_LOCK: milestoneId,
         GSD_PROJECT_ROOT: basePath,
         GSD_PARALLEL_WORKER: "1",
+        GSD_SLICE_WORKER_TOKEN: worker.workerToken,
       },
       stdio: ["ignore", "pipe", "pipe"],
       detached: false,
@@ -357,9 +643,15 @@ function spawnSliceWorker(
 
   worker.process = child;
   worker.pid = child.pid ?? 0;
+  worker.processStartFingerprint = worker.pid > 0
+    ? readProcessStartFingerprint(worker.pid)
+    : null;
 
   if (!child.pid) {
     worker.process = null;
+    worker.pid = 0;
+    worker.processStartFingerprint = null;
+    try { child.kill("SIGTERM"); } catch { /* best-effort */ }
     return false;
   }
 
@@ -438,6 +730,10 @@ function spawnSliceWorker(
       startedAt: w.startedAt,
       worktreePath: w.worktreePath,
     });
+
+    // Persist worker terminal state for crash recovery.
+    // (Issue #4980 HIGH-8)
+    persistSliceState();
   });
 
   return true;
@@ -474,6 +770,11 @@ function processSliceWorkerLine(
         sliceState.totalCost += usage.cost;
       }
       worker.completedUnits++;
+      // Persist cost / progress updates so a crash mid-run preserves them.
+      // Throttled (~1/s per process) so high-frequency message_end traffic
+      // does not saturate disk I/O. Worker exit / start / stop paths persist
+      // unthrottled to guarantee the terminal state lands. (Issue #4980 HIGH-8)
+      persistSliceStateThrottled();
     }
   }
 }

package/src/resources/extensions/gsd/state-transition-matrix.ts

@@ -0,0 +1,152 @@
+import type { Phase } from "./types.js";
+
+export type StateTransitionReasonCode =
+  | "state"
+  | "manual"
+  | "recovery"
+  | "dependency"
+  | "policy"
+  | "retry";
+
+export interface StateTransitionEntry {
+  from: Phase | "*";
+  event: string;
+  guard: string;
+  to: Phase;
+  onFail: Phase | "manual-attention" | "blocked" | "no-transition";
+  reasonCode: StateTransitionReasonCode;
+}
+
+export const STATE_TRANSITION_MATRIX: readonly StateTransitionEntry[] = [
+  {
+    from: "needs-discussion",
+    event: "context-ready",
+    guard: "CONTEXT artifact exists or PRD/context express path produced context",
+    to: "researching",
+    onFail: "needs-discussion",
+    reasonCode: "state",
+  },
+  {
+    from: "researching",
+    event: "research-ready",
+    guard: "RESEARCH artifact exists or research is explicitly skipped",
+    to: "planning",
+    onFail: "researching",
+    reasonCode: "state",
+  },
+  {
+    from: "planning",
+    event: "plan-ready",
+    guard: "ROADMAP/PLAN artifacts exist and plan gate passes",
+    to: "executing",
+    onFail: "replanning-slice",
+    reasonCode: "state",
+  },
+  {
+    from: "executing",
+    event: "task-dispatched",
+    guard: "task inputs are ready and dependencies are closed",
+    to: "executing",
+    onFail: "blocked",
+    reasonCode: "dependency",
+  },
+  {
+    from: "executing",
+    event: "slice-complete",
+    guard: "all slice tasks are closed and verification gate passes",
+    to: "summarizing",
+    onFail: "validating-milestone",
+    reasonCode: "state",
+  },
+  {
+    from: "summarizing",
+    event: "summary-ready",
+    guard: "SUMMARY artifact exists for the completed work unit",
+    to: "validating-milestone",
+    onFail: "summarizing",
+    reasonCode: "state",
+  },
+  {
+    from: "validating-milestone",
+    event: "validation-pass",
+    guard: "validation verdict is terminal and not remediation-required",
+    to: "completing-milestone",
+    onFail: "blocked",
+    reasonCode: "state",
+  },
+  {
+    from: "blocked",
+    event: "recovery-plan-ready",
+    guard: "reassessment produced an executable next action",
+    to: "executing",
+    onFail: "blocked",
+    reasonCode: "recovery",
+  },
+  {
+    from: "replanning-slice",
+    event: "replan-ready",
+    guard: "replacement slice/task plan exists and plan gate passes",
+    to: "executing",
+    onFail: "blocked",
+    reasonCode: "recovery",
+  },
+  {
+    from: "completing-milestone",
+    event: "closeout-complete",
+    guard: "closeout gate passes and git transaction succeeds",
+    to: "complete",
+    onFail: "blocked",
+    reasonCode: "state",
+  },
+  {
+    from: "*",
+    event: "manual-block",
+    guard: "operator or hard gate requested manual attention",
+    to: "blocked",
+    onFail: "manual-attention",
+    reasonCode: "manual",
+  },
+  {
+    from: "*",
+    event: "retryable-failure",
+    guard: "retry budget remains for failure class",
+    to: "executing",
+    onFail: "blocked",
+    reasonCode: "retry",
+  },
+] as const;
+
+export interface MatrixValidationResult {
+  ok: boolean;
+  missingEvents: string[];
+  duplicateKeys: string[];
+}
+
+export function findTransition(
+  from: Phase,
+  event: string,
+): StateTransitionEntry | undefined {
+  return STATE_TRANSITION_MATRIX.find((entry) =>
+    (entry.from === from || entry.from === "*") && entry.event === event,
+  );
+}
+
+export function validateTransitionMatrix(requiredEvents: readonly string[]): MatrixValidationResult {
+  const seen = new Set<string>();
+  const duplicateKeys: string[] = [];
+
+  for (const entry of STATE_TRANSITION_MATRIX) {
+    const key = `${entry.from}:${entry.event}`;
+    if (seen.has(key)) duplicateKeys.push(key);
+    seen.add(key);
+  }
+
+  const availableEvents = new Set(STATE_TRANSITION_MATRIX.map((entry) => entry.event));
+  const missingEvents = requiredEvents.filter((event) => !availableEvents.has(event));
+
+  return {
+    ok: missingEvents.length === 0 && duplicateKeys.length === 0,
+    missingEvents,
+    duplicateKeys,
+  };
+}