gsd-pi 2.77.0-dev.58d3d4d6c → 2.77.0-dev.cfd69e714

package/src/resources/extensions/gsd/slice-parallel-orchestrator.ts

@@ -13,13 +13,16 @@
  * - Conflict check: file overlap between slice plans (slice-parallel-conflict.ts)
  */
 
-import { spawn, type ChildProcess } from "node:child_process";
+import { execFileSync, spawn, type ChildProcess } from "node:child_process";
+import { randomUUID } from "node:crypto";
 import {
   appendFileSync,
   existsSync,
   writeFileSync,
   readFileSync,
   mkdirSync,
+  renameSync,
+  unlinkSync,
 } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -40,6 +43,8 @@ export interface SliceWorkerInfo {
   milestoneId: string;
   sliceId: string;
   pid: number;
+  workerToken: string;
+  processStartFingerprint: string | null;
   process: ChildProcess | null;
   worktreePath: string;
   startedAt: number;
@@ -69,13 +74,276 @@ export interface StartSliceParallelOpts {
 
 let sliceState: SliceOrchestratorState | null = null;
 
+// ─── Persisted State (crash recovery) ──────────────────────────────────────
+//
+// Mirrors parallel-orchestrator.ts. Without persistence, a coordinator crash
+// leaves orphaned worktrees on disk with no way to detect or clean them up
+// on next session start. (Issue #4980 HIGH-8)
+
+const SLICE_ORCHESTRATOR_STATE_FILE = "slice-orchestrator.json";
+const TMP_SUFFIX = ".tmp";
+
+interface PersistedSliceWorker {
+  milestoneId: string;
+  sliceId: string;
+  pid: number;
+  workerToken?: string;
+  processStartFingerprint?: string | null;
+  worktreePath: string;
+  startedAt: number;
+  state: "running" | "stopped" | "error";
+  completedUnits: number;
+  cost: number;
+}
+
+interface PersistedSliceState {
+  active: boolean;
+  workers: PersistedSliceWorker[];
+  totalCost: number;
+  budgetCeiling?: number;
+  maxWorkers: number;
+  startedAt: number;
+  basePath: string;
+}
+
+function sliceStateFilePath(basePath: string): string {
+  return join(gsdRoot(basePath), SLICE_ORCHESTRATOR_STATE_FILE);
+}
+
+function isPidAlive(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "EPERM") return true;
+    return false;
+  }
+}
+
+function readLinuxProcessStartFingerprint(pid: number): string | null {
+  try {
+    const stat = readFileSync(`/proc/${pid}/stat`, "utf-8");
+    const afterCommand = stat.slice(stat.lastIndexOf(")") + 2).trim();
+    const fields = afterCommand.split(/\s+/);
+    const startTimeTicks = fields[19];
+    return startTimeTicks ? `linux-stat:${startTimeTicks}` : null;
+  } catch {
+    return null;
+  }
+}
+
+function readPsProcessStartFingerprint(pid: number): string | null {
+  try {
+    const raw = execFileSync("ps", ["-p", String(pid), "-o", "lstart="], {
+      stdio: ["ignore", "pipe", "ignore"],
+      encoding: "utf-8",
+    }).trim().replace(/\s+/g, " ");
+    return raw ? `ps-lstart:${raw}` : null;
+  } catch {
+    return null;
+  }
+}
+
+function readProcessStartFingerprint(pid: number): string | null {
+  if (!Number.isInteger(pid) || pid <= 0) return null;
+  return readLinuxProcessStartFingerprint(pid) ?? readPsProcessStartFingerprint(pid);
+}
+
+function linuxProcessEnvContains(pid: number, key: string, value: string): boolean | null {
+  if (process.platform !== "linux") return null;
+  try {
+    const env = readFileSync(`/proc/${pid}/environ`, "utf-8");
+    return env.split("\0").includes(`${key}=${value}`);
+  } catch {
+    return null;
+  }
+}
+
+function createWorkerToken(milestoneId: string, sliceId: string): string {
+  return `slice:${milestoneId}:${sliceId}:${Date.now()}:${randomUUID()}`;
+}
+
+function isRecoveredSliceWorkerAlive(worker: {
+  pid: number;
+  workerToken?: string;
+  processStartFingerprint?: string | null;
+}): boolean {
+  if (!isPidAlive(worker.pid)) return false;
+  if (!worker.processStartFingerprint) return false;
+
+  const currentFingerprint = readProcessStartFingerprint(worker.pid);
+  if (!currentFingerprint || currentFingerprint !== worker.processStartFingerprint) {
+    return false;
+  }
+
+  if (worker.workerToken) {
+    const envMatches = linuxProcessEnvContains(
+      worker.pid,
+      "GSD_SLICE_WORKER_TOKEN",
+      worker.workerToken,
+    );
+    if (envMatches === false) return false;
+  }
+
+  return true;
+}
+
+/**
+ * Persist current slice orchestrator state. Atomic write (tmp + rename) to
+ * prevent partial reads if the coordinator dies mid-write.
+ */
+function persistSliceState(): void {
+  if (!sliceState) return;
+  try {
+    const dir = gsdRoot(sliceState.basePath);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+
+    const persisted: PersistedSliceState = {
+      active: sliceState.active,
+      workers: [...sliceState.workers.values()].map((w) => ({
+        milestoneId: w.milestoneId,
+        sliceId: w.sliceId,
+        pid: w.pid,
+        workerToken: w.workerToken,
+        processStartFingerprint: w.processStartFingerprint,
+        worktreePath: w.worktreePath,
+        startedAt: w.startedAt,
+        state: w.state,
+        completedUnits: w.completedUnits,
+        cost: w.cost,
+      })),
+      totalCost: sliceState.totalCost,
+      budgetCeiling: sliceState.budgetCeiling,
+      maxWorkers: sliceState.maxWorkers,
+      startedAt: sliceState.startedAt,
+      basePath: sliceState.basePath,
+    };
+
+    const dest = sliceStateFilePath(sliceState.basePath);
+    const tmp = dest + TMP_SUFFIX;
+    writeFileSync(tmp, JSON.stringify(persisted, null, 2), "utf-8");
+    renameSync(tmp, dest);
+    lastPersistTs = Date.now();
+  } catch {
+    /* non-fatal: persistence is best-effort */
+  }
+}
+
+/**
+ * Throttled wrapper around `persistSliceState`. Skips if the last successful
+ * persist was less than `PERSIST_THROTTLE_MS` ago; otherwise persists
+ * immediately. Use this on hot paths (e.g. `message_end` events) where we
+ * receive many events per second per worker. Terminal events (worker exit,
+ * crash, stop) should call `persistSliceState()` directly to guarantee the
+ * final state hits disk regardless of timing.
+ */
+const PERSIST_THROTTLE_MS = 1000;
+let lastPersistTs = 0;
+
+function persistSliceStateThrottled(): void {
+  if (Date.now() - lastPersistTs < PERSIST_THROTTLE_MS) return;
+  persistSliceState();
+}
+
+function removeSliceStateFile(basePath: string): void {
+  try {
+    const p = sliceStateFilePath(basePath);
+    if (existsSync(p)) unlinkSync(p);
+  } catch {
+    /* non-fatal */
+  }
+}
+
+/**
+ * Restore slice orchestrator state from disk. Filters dead-PID workers and
+ * removes their orphaned worktrees so a clean restart is possible.
+ *
+ * Returns null if no state file exists or no workers survive.
+ */
+export function restoreSliceState(basePath: string): PersistedSliceState | null {
+  try {
+    const p = sliceStateFilePath(basePath);
+    if (!existsSync(p)) return null;
+    const persisted = JSON.parse(readFileSync(p, "utf-8")) as PersistedSliceState;
+
+    const survivors: PersistedSliceWorker[] = [];
+    const dead: PersistedSliceWorker[] = [];
+    for (const w of persisted.workers) {
+      if (w.state === "running" && isRecoveredSliceWorkerAlive(w)) {
+        survivors.push(w);
+      } else if (w.state === "running") {
+        dead.push(w);
+      } else {
+        survivors.push(w);
+      }
+    }
+
+    // Best-effort cleanup of orphaned worktrees from dead workers.
+    for (const w of dead) {
+      const wtName = `${w.milestoneId}-${w.sliceId}`;
+      try {
+        removeWorktree(persisted.basePath, wtName, { deleteBranch: true, force: true });
+      } catch {
+        /* worktree may already be gone */
+      }
+    }
+
+    persisted.workers = survivors;
+
+    if (survivors.length === 0) {
+      removeSliceStateFile(basePath);
+      return null;
+    }
+
+    return persisted;
+  } catch {
+    return null;
+  }
+}
+
 // ─── Public API ────────────────────────────────────────────────────────────
 
 /**
  * Check whether slice-level parallel is currently active.
+ *
+ * If in-memory state is unset but a persisted state file exists with at
+ * least one live-PID worker, treat as active and rehydrate so a coordinator
+ * crash followed by a fresh process is detectable. (Issue #4980 HIGH-8)
  */
-export function isSliceParallelActive(): boolean {
-  return sliceState?.active === true;
+export function isSliceParallelActive(basePath?: string): boolean {
+  if (sliceState?.active === true) return true;
+  if (!basePath) return false;
+  const restored = restoreSliceState(basePath);
+  if (!restored || restored.workers.length === 0) return false;
+
+  // Rehydrate in-memory state from disk; processes are detached so we have
+  // no ChildProcess handles, only PIDs.
+  sliceState = {
+    active: restored.active,
+    workers: new Map(),
+    totalCost: restored.totalCost,
+    budgetCeiling: restored.budgetCeiling,
+    maxWorkers: restored.maxWorkers,
+    startedAt: restored.startedAt,
+    basePath: restored.basePath,
+  };
+  for (const w of restored.workers) {
+    sliceState.workers.set(w.sliceId, {
+      milestoneId: w.milestoneId,
+      sliceId: w.sliceId,
+      pid: w.pid,
+      process: null,
+      worktreePath: w.worktreePath,
+      workerToken: w.workerToken ?? "",
+      processStartFingerprint: w.processStartFingerprint ?? null,
+      startedAt: w.startedAt,
+      state: w.state,
+      completedUnits: w.completedUnits,
+      cost: w.cost,
+    });
+  }
+  return true;
 }
 
 /**
@@ -146,6 +414,8 @@ export async function startSliceParallel(
         milestoneId,
         sliceId: slice.id,
         pid: 0,
+        workerToken: createWorkerToken(milestoneId, slice.id),
+        processStartFingerprint: null,
         process: null,
         worktreePath: wtPath,
         startedAt: Date.now(),
@@ -162,12 +432,16 @@ export async function startSliceParallel(
         started.push(slice.id);
       } else {
         errors.push({ sid: slice.id, error: "Failed to spawn worker process" });
-        worker.state = "error";
+        sliceState.workers.delete(slice.id);
+        try {
+          removeWorktree(basePath, wtName, { deleteBranch: true, force: true });
+        } catch { /* ignore cleanup failures */ }
       }
     } catch (err) {
       errors.push({ sid: slice.id, error: getErrorMessage(err) });
       // Best-effort cleanup of partially created worktree
       const wtName = `${milestoneId}-${slice.id}`;
+      sliceState.workers.delete(slice.id);
       try {
         removeWorktree(basePath, wtName, { deleteBranch: true, force: true });
       } catch { /* ignore cleanup failures */ }
@@ -177,6 +451,10 @@ export async function startSliceParallel(
   // If nothing started, deactivate
   if (started.length === 0) {
     sliceState.active = false;
+    removeSliceStateFile(basePath);
+  } else {
+    // Persist state for crash recovery (Issue #4980 HIGH-8).
+    persistSliceState();
   }
 
   return { started, errors };
@@ -187,13 +465,16 @@ export async function startSliceParallel(
  */
 export function stopSliceParallel(): void {
   if (!sliceState) return;
+  const basePath = sliceState.basePath;
 
   for (const worker of sliceState.workers.values()) {
-    if (worker.process) {
-      try {
+    try {
+      if (worker.process) {
         worker.process.kill("SIGTERM");
-      } catch { /* already dead */ }
-    }
+      } else if (worker.state === "running" && isRecoveredSliceWorkerAlive(worker)) {
+        process.kill(worker.pid, "SIGTERM");
+      }
+    } catch { /* already dead */ }
     worker.cleanup?.();
     worker.cleanup = undefined;
     worker.process = null;
@@ -207,6 +488,9 @@ export function stopSliceParallel(): void {
   }
 
   sliceState.active = false;
+  // Clear persisted state — clean shutdown means no recovery on next start.
+  // (Issue #4980 HIGH-8)
+  removeSliceStateFile(basePath);
 }
 
 /**
@@ -239,6 +523,7 @@ export function resetSliceOrchestrator(): void {
     }
   }
   sliceState = null;
+  lastPersistTs = 0;
 }
 
 // ─── Internal: Conflict Filtering ──────────────────────────────────────────
@@ -339,6 +624,7 @@ function spawnSliceWorker(
         GSD_MILESTONE_LOCK: milestoneId,
         GSD_PROJECT_ROOT: basePath,
         GSD_PARALLEL_WORKER: "1",
+        GSD_SLICE_WORKER_TOKEN: worker.workerToken,
       },
       stdio: ["ignore", "pipe", "pipe"],
       detached: false,
@@ -357,9 +643,15 @@ function spawnSliceWorker(
 
   worker.process = child;
   worker.pid = child.pid ?? 0;
+  worker.processStartFingerprint = worker.pid > 0
+    ? readProcessStartFingerprint(worker.pid)
+    : null;
 
   if (!child.pid) {
     worker.process = null;
+    worker.pid = 0;
+    worker.processStartFingerprint = null;
+    try { child.kill("SIGTERM"); } catch { /* best-effort */ }
     return false;
   }
 
@@ -438,6 +730,10 @@ function spawnSliceWorker(
       startedAt: w.startedAt,
       worktreePath: w.worktreePath,
     });
+
+    // Persist worker terminal state for crash recovery.
+    // (Issue #4980 HIGH-8)
+    persistSliceState();
   });
 
   return true;
@@ -474,6 +770,11 @@ function processSliceWorkerLine(
         sliceState.totalCost += usage.cost;
       }
       worker.completedUnits++;
+      // Persist cost / progress updates so a crash mid-run preserves them.
+      // Throttled (~1/s per process) so high-frequency message_end traffic
+      // does not saturate disk I/O. Worker exit / start / stop paths persist
+      // unthrottled to guarantee the terminal state lands. (Issue #4980 HIGH-8)
+      persistSliceStateThrottled();
     }
   }
 }

package/src/resources/extensions/gsd/state.ts

@@ -267,7 +267,7 @@ export async function deriveState(basePath: string): Promise<GSDState> {
       let synced = false;
       for (const diskId of diskIds) {
         if (!isGhostMilestone(basePath, diskId)) {
-          insertMilestone({ id: diskId, status: 'active' });
+          insertMilestone(diskMilestoneInsert(basePath, diskId));
           synced = true;
         }
       }
@@ -327,6 +327,42 @@ function extractContextTitle(content: string | null, fallback: string): string {
 // Alias kept for backward compatibility within this file.
 const isStatusDone = isClosedStatus;
 
+function loadSync(path: string | null): string | null {
+  if (!path) return null;
+  try {
+    return readFileSync(path, "utf-8");
+  } catch {
+    return null;
+  }
+}
+
+function diskMilestoneInsert(basePath: string, mid: string): {
+  id: string;
+  title?: string;
+  status: string;
+  depends_on: string[];
+} {
+  const contextContent = loadSync(resolveMilestoneFile(basePath, mid, "CONTEXT"));
+  const draftContent = !contextContent ? loadSync(resolveMilestoneFile(basePath, mid, "CONTEXT-DRAFT")) : null;
+  const roadmapContent = loadSync(resolveMilestoneFile(basePath, mid, "ROADMAP"));
+  const summaryContent = loadSync(resolveMilestoneFile(basePath, mid, "SUMMARY"));
+  const roadmap = roadmapContent ? parseRoadmap(roadmapContent) : null;
+  const summary = summaryContent ? parseSummary(summaryContent) : null;
+  const summaryTerminal = summaryContent != null && isTerminalMilestoneSummaryContent(summaryContent);
+  const parked = resolveMilestoneFile(basePath, mid, "PARKED") !== null;
+
+  return {
+    id: mid,
+    title: roadmap
+      ? stripMilestonePrefix(roadmap.title)
+      : (contextContent || draftContent)
+        ? extractContextTitle(contextContent || draftContent, mid)
+        : (summary?.title || mid),
+    status: parked ? "parked" : summaryTerminal ? "complete" : "active",
+    depends_on: parseContextDependsOn(contextContent ?? draftContent),
+  };
+}
+
 /**
  * Derive GSD state from the milestones/slices/tasks DB tables.
  * Flag files (PARKED, VALIDATION, CONTINUE, REPLAN, REPLAN-TRIGGER, CONTEXT-DRAFT)
@@ -342,7 +378,7 @@ function reconcileDiskToDb(basePath: string): MilestoneRow[] {
   let synced = false;
   for (const diskId of diskIds) {
     if (!dbIdSet.has(diskId) && !isGhostMilestone(basePath, diskId)) {
-      insertMilestone({ id: diskId, status: 'active' });
+      insertMilestone(diskMilestoneInsert(basePath, diskId));
       synced = true;
     }
   }
@@ -677,32 +713,12 @@ function resolveSliceDependencies(activeMilestoneSlices: SliceRow[]): { activeSl
     }
   }
 
-  let bestFallback: SliceRow | null = null;
-  let bestFallbackSatisfied = -1;
-
   for (const s of activeMilestoneSlices) {
     if (isStatusDone(s.status)) continue;
     if (isDeferredStatus(s.status)) continue;
     if (s.depends.every(dep => doneSliceIds.has(dep))) {
       return { activeSlice: { id: s.id, title: s.title }, activeSliceRow: s };
     }
-    const satisfied = s.depends.filter(dep => doneSliceIds.has(dep)).length;
-    if (satisfied > bestFallbackSatisfied || (satisfied === bestFallbackSatisfied && !bestFallback)) {
-      bestFallback = s;
-      bestFallbackSatisfied = satisfied;
-    }
-  }
-
-  if (bestFallback) {
-    const unmet = bestFallback.depends.filter(dep => !doneSliceIds.has(dep));
-    logWarning(
-      "state",
-      `No slice has all deps satisfied — falling back to ${bestFallback.id} ` +
-        `(${bestFallbackSatisfied}/${bestFallback.depends.length} deps met, ` +
-        `unmet: ${unmet.join(", ")})`,
-      { mid: activeMilestoneSlices[0]?.milestone_id, sid: bestFallback.id },
-    );
-    return { activeSlice: { id: bestFallback.id, title: bestFallback.title }, activeSliceRow: bestFallback };
   }
 
   return { activeSlice: null, activeSliceRow: null };
@@ -716,14 +732,19 @@ async function reconcileSliceTasks(
 ): Promise<TaskRow[]> {
   let tasks = getSliceTasks(milestoneId, sliceId);
 
-  if (tasks.length === 0 && planFile) {
+  // #3600/#4974: import missing plan-file tasks even when the DB already has
+  // a partial task set. Existing DB task statuses stay authoritative.
+  if (planFile) {
     try {
       const planContent = await loadFile(planFile);
       if (planContent) {
         const diskPlan = parsePlan(planContent);
         if (diskPlan.tasks.length > 0) {
+          const dbTaskIds = new Set(tasks.map(t => t.id));
+          let inserted = 0;
           for (let i = 0; i < diskPlan.tasks.length; i++) {
             const t = diskPlan.tasks[i];
+            if (dbTaskIds.has(t.id)) continue;
             try {
               insertTask({
                 id: t.id,
@@ -733,12 +754,15 @@ async function reconcileSliceTasks(
                 status: t.done ? 'complete' : 'pending',
                 sequence: i + 1,
               });
+              inserted++;
             } catch (insertErr) {
               logWarning("reconcile", `failed to insert task ${t.id} from plan file: ${insertErr instanceof Error ? insertErr.message : String(insertErr)}`);
             }
           }
-          tasks = getSliceTasks(milestoneId, sliceId);
-          logWarning("reconcile", `imported ${tasks.length} tasks from plan file for ${milestoneId}/${sliceId} — DB was empty (#3600)`, { mid: milestoneId, sid: sliceId });
+          if (inserted > 0) {
+            tasks = getSliceTasks(milestoneId, sliceId);
+            logWarning("reconcile", `imported ${inserted} missing task(s) from plan file for ${milestoneId}/${sliceId}`, { mid: milestoneId, sid: sliceId });
+          }
         }
       }
     } catch (err) {
@@ -1569,31 +1593,12 @@ export async function _deriveStateImpl(basePath: string): Promise<GSDState> {
       };
     }
   } else {
-    let bestFallbackLegacy: { id: string; title: string; depends: string[] } | null = null;
-    let bestFallbackLegacySatisfied = -1;
-
     for (const s of activeRoadmap.slices) {
       if (s.done) continue;
       if (s.depends.every(dep => doneSliceIds.has(dep))) {
         activeSlice = { id: s.id, title: s.title };
         break;
       }
-      const satisfied = s.depends.filter(dep => doneSliceIds.has(dep)).length;
-      if (satisfied > bestFallbackLegacySatisfied) {
-        bestFallbackLegacy = s;
-        bestFallbackLegacySatisfied = satisfied;
-      }
-    }
-
-    if (!activeSlice && bestFallbackLegacy) {
-      const unmet = bestFallbackLegacy.depends.filter(dep => !doneSliceIds.has(dep));
-      logWarning(
-        "state",
-        `No slice has all deps satisfied — falling back to ${bestFallbackLegacy.id} ` +
-          `(${bestFallbackLegacySatisfied}/${bestFallbackLegacy.depends.length} deps met, ` +
-          `unmet: ${unmet.join(", ")})`,
-      );
-      activeSlice = { id: bestFallbackLegacy.id, title: bestFallbackLegacy.title };
     }
   }