gsd-pi 2.77.0-dev.eaa4973bc → 2.78.0

package/dist/resources/extensions/gsd/slice-parallel-orchestrator.js

@@ -12,8 +12,9 @@
  * - Lock env: GSD_SLICE_LOCK (in addition to GSD_MILESTONE_LOCK)
  * - Conflict check: file overlap between slice plans (slice-parallel-conflict.ts)
  */
-import { spawn } from "node:child_process";
-import { appendFileSync, existsSync, mkdirSync, } from "node:fs";
+import { execFileSync, spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { appendFileSync, existsSync, writeFileSync, readFileSync, mkdirSync, renameSync, unlinkSync, } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { gsdRoot } from "./paths.js";
@@ -24,12 +25,243 @@ import { getErrorMessage } from "./error-utils.js";
 import { selectConflictFreeBatch } from "./uok/execution-graph.js";
 // ─── Module State ──────────────────────────────────────────────────────────
 let sliceState = null;
+// ─── Persisted State (crash recovery) ──────────────────────────────────────
+//
+// Mirrors parallel-orchestrator.ts. Without persistence, a coordinator crash
+// leaves orphaned worktrees on disk with no way to detect or clean them up
+// on next session start. (Issue #4980 HIGH-8)
+const SLICE_ORCHESTRATOR_STATE_FILE = "slice-orchestrator.json";
+const TMP_SUFFIX = ".tmp";
+function sliceStateFilePath(basePath) {
+    return join(gsdRoot(basePath), SLICE_ORCHESTRATOR_STATE_FILE);
+}
+function isPidAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        if (err.code === "EPERM")
+            return true;
+        return false;
+    }
+}
+function readLinuxProcessStartFingerprint(pid) {
+    try {
+        const stat = readFileSync(`/proc/${pid}/stat`, "utf-8");
+        const afterCommand = stat.slice(stat.lastIndexOf(")") + 2).trim();
+        const fields = afterCommand.split(/\s+/);
+        const startTimeTicks = fields[19];
+        return startTimeTicks ? `linux-stat:${startTimeTicks}` : null;
+    }
+    catch {
+        return null;
+    }
+}
+function readPsProcessStartFingerprint(pid) {
+    try {
+        const raw = execFileSync("ps", ["-p", String(pid), "-o", "lstart="], {
+            stdio: ["ignore", "pipe", "ignore"],
+            encoding: "utf-8",
+        }).trim().replace(/\s+/g, " ");
+        return raw ? `ps-lstart:${raw}` : null;
+    }
+    catch {
+        return null;
+    }
+}
+function readProcessStartFingerprint(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return null;
+    return readLinuxProcessStartFingerprint(pid) ?? readPsProcessStartFingerprint(pid);
+}
+function linuxProcessEnvContains(pid, key, value) {
+    if (process.platform !== "linux")
+        return null;
+    try {
+        const env = readFileSync(`/proc/${pid}/environ`, "utf-8");
+        return env.split("\0").includes(`${key}=${value}`);
+    }
+    catch {
+        return null;
+    }
+}
+function createWorkerToken(milestoneId, sliceId) {
+    return `slice:${milestoneId}:${sliceId}:${Date.now()}:${randomUUID()}`;
+}
+function isRecoveredSliceWorkerAlive(worker) {
+    if (!isPidAlive(worker.pid))
+        return false;
+    if (!worker.processStartFingerprint)
+        return false;
+    const currentFingerprint = readProcessStartFingerprint(worker.pid);
+    if (!currentFingerprint || currentFingerprint !== worker.processStartFingerprint) {
+        return false;
+    }
+    if (worker.workerToken) {
+        const envMatches = linuxProcessEnvContains(worker.pid, "GSD_SLICE_WORKER_TOKEN", worker.workerToken);
+        if (envMatches === false)
+            return false;
+    }
+    return true;
+}
+/**
+ * Persist current slice orchestrator state. Atomic write (tmp + rename) to
+ * prevent partial reads if the coordinator dies mid-write.
+ */
+function persistSliceState() {
+    if (!sliceState)
+        return;
+    try {
+        const dir = gsdRoot(sliceState.basePath);
+        if (!existsSync(dir))
+            mkdirSync(dir, { recursive: true });
+        const persisted = {
+            active: sliceState.active,
+            workers: [...sliceState.workers.values()].map((w) => ({
+                milestoneId: w.milestoneId,
+                sliceId: w.sliceId,
+                pid: w.pid,
+                workerToken: w.workerToken,
+                processStartFingerprint: w.processStartFingerprint,
+                worktreePath: w.worktreePath,
+                startedAt: w.startedAt,
+                state: w.state,
+                completedUnits: w.completedUnits,
+                cost: w.cost,
+            })),
+            totalCost: sliceState.totalCost,
+            budgetCeiling: sliceState.budgetCeiling,
+            maxWorkers: sliceState.maxWorkers,
+            startedAt: sliceState.startedAt,
+            basePath: sliceState.basePath,
+        };
+        const dest = sliceStateFilePath(sliceState.basePath);
+        const tmp = dest + TMP_SUFFIX;
+        writeFileSync(tmp, JSON.stringify(persisted, null, 2), "utf-8");
+        renameSync(tmp, dest);
+        lastPersistTs = Date.now();
+    }
+    catch {
+        /* non-fatal: persistence is best-effort */
+    }
+}
+/**
+ * Throttled wrapper around `persistSliceState`. Skips if the last successful
+ * persist was less than `PERSIST_THROTTLE_MS` ago; otherwise persists
+ * immediately. Use this on hot paths (e.g. `message_end` events) where we
+ * receive many events per second per worker. Terminal events (worker exit,
+ * crash, stop) should call `persistSliceState()` directly to guarantee the
+ * final state hits disk regardless of timing.
+ */
+const PERSIST_THROTTLE_MS = 1000;
+let lastPersistTs = 0;
+function persistSliceStateThrottled() {
+    if (Date.now() - lastPersistTs < PERSIST_THROTTLE_MS)
+        return;
+    persistSliceState();
+}
+function removeSliceStateFile(basePath) {
+    try {
+        const p = sliceStateFilePath(basePath);
+        if (existsSync(p))
+            unlinkSync(p);
+    }
+    catch {
+        /* non-fatal */
+    }
+}
+/**
+ * Restore slice orchestrator state from disk. Filters dead-PID workers and
+ * removes their orphaned worktrees so a clean restart is possible.
+ *
+ * Returns null if no state file exists or no workers survive.
+ */
+export function restoreSliceState(basePath) {
+    try {
+        const p = sliceStateFilePath(basePath);
+        if (!existsSync(p))
+            return null;
+        const persisted = JSON.parse(readFileSync(p, "utf-8"));
+        const survivors = [];
+        const dead = [];
+        for (const w of persisted.workers) {
+            if (w.state === "running" && isRecoveredSliceWorkerAlive(w)) {
+                survivors.push(w);
+            }
+            else if (w.state === "running") {
+                dead.push(w);
+            }
+            else {
+                survivors.push(w);
+            }
+        }
+        // Best-effort cleanup of orphaned worktrees from dead workers.
+        for (const w of dead) {
+            const wtName = `${w.milestoneId}-${w.sliceId}`;
+            try {
+                removeWorktree(persisted.basePath, wtName, { deleteBranch: true, force: true });
+            }
+            catch {
+                /* worktree may already be gone */
+            }
+        }
+        persisted.workers = survivors;
+        if (survivors.length === 0) {
+            removeSliceStateFile(basePath);
+            return null;
+        }
+        return persisted;
+    }
+    catch {
+        return null;
+    }
+}
 // ─── Public API ────────────────────────────────────────────────────────────
 /**
  * Check whether slice-level parallel is currently active.
+ *
+ * If in-memory state is unset but a persisted state file exists with at
+ * least one live-PID worker, treat as active and rehydrate so a coordinator
+ * crash followed by a fresh process is detectable. (Issue #4980 HIGH-8)
  */
-export function isSliceParallelActive() {
-    return sliceState?.active === true;
+export function isSliceParallelActive(basePath) {
+    if (sliceState?.active === true)
+        return true;
+    if (!basePath)
+        return false;
+    const restored = restoreSliceState(basePath);
+    if (!restored || restored.workers.length === 0)
+        return false;
+    // Rehydrate in-memory state from disk; processes are detached so we have
+    // no ChildProcess handles, only PIDs.
+    sliceState = {
+        active: restored.active,
+        workers: new Map(),
+        totalCost: restored.totalCost,
+        budgetCeiling: restored.budgetCeiling,
+        maxWorkers: restored.maxWorkers,
+        startedAt: restored.startedAt,
+        basePath: restored.basePath,
+    };
+    for (const w of restored.workers) {
+        sliceState.workers.set(w.sliceId, {
+            milestoneId: w.milestoneId,
+            sliceId: w.sliceId,
+            pid: w.pid,
+            process: null,
+            worktreePath: w.worktreePath,
+            workerToken: w.workerToken ?? "",
+            processStartFingerprint: w.processStartFingerprint ?? null,
+            startedAt: w.startedAt,
+            state: w.state,
+            completedUnits: w.completedUnits,
+            cost: w.cost,
+        });
+    }
+    return true;
 }
 /**
  * Get current slice orchestrator state (read-only snapshot).
@@ -80,6 +312,8 @@ export async function startSliceParallel(basePath, milestoneId, eligibleSlices,
                 milestoneId,
                 sliceId: slice.id,
                 pid: 0,
+                workerToken: createWorkerToken(milestoneId, slice.id),
+                processStartFingerprint: null,
                 process: null,
                 worktreePath: wtPath,
                 startedAt: Date.now(),
@@ -95,13 +329,18 @@ export async function startSliceParallel(basePath, milestoneId, eligibleSlices,
             }
             else {
                 errors.push({ sid: slice.id, error: "Failed to spawn worker process" });
-                worker.state = "error";
+                sliceState.workers.delete(slice.id);
+                try {
+                    removeWorktree(basePath, wtName, { deleteBranch: true, force: true });
+                }
+                catch { /* ignore cleanup failures */ }
             }
         }
         catch (err) {
             errors.push({ sid: slice.id, error: getErrorMessage(err) });
             // Best-effort cleanup of partially created worktree
             const wtName = `${milestoneId}-${slice.id}`;
+            sliceState.workers.delete(slice.id);
             try {
                 removeWorktree(basePath, wtName, { deleteBranch: true, force: true });
             }
@@ -111,6 +350,11 @@ export async function startSliceParallel(basePath, milestoneId, eligibleSlices,
     // If nothing started, deactivate
     if (started.length === 0) {
         sliceState.active = false;
+        removeSliceStateFile(basePath);
+    }
+    else {
+        // Persist state for crash recovery (Issue #4980 HIGH-8).
+        persistSliceState();
     }
     return { started, errors };
 }
@@ -120,13 +364,17 @@ export async function startSliceParallel(basePath, milestoneId, eligibleSlices,
 export function stopSliceParallel() {
     if (!sliceState)
         return;
+    const basePath = sliceState.basePath;
     for (const worker of sliceState.workers.values()) {
-        if (worker.process) {
-            try {
+        try {
+            if (worker.process) {
                 worker.process.kill("SIGTERM");
             }
-            catch { /* already dead */ }
+            else if (worker.state === "running" && isRecoveredSliceWorkerAlive(worker)) {
+                process.kill(worker.pid, "SIGTERM");
+            }
         }
+        catch { /* already dead */ }
         worker.cleanup?.();
         worker.cleanup = undefined;
         worker.process = null;
@@ -139,6 +387,9 @@ export function stopSliceParallel() {
         catch { /* best-effort cleanup */ }
     }
     sliceState.active = false;
+    // Clear persisted state — clean shutdown means no recovery on next start.
+    // (Issue #4980 HIGH-8)
+    removeSliceStateFile(basePath);
 }
 /**
  * Get aggregate cost across all slice workers.
@@ -170,6 +421,7 @@ export function resetSliceOrchestrator() {
         }
     }
     sliceState = null;
+    lastPersistTs = 0;
 }
 // ─── Internal: Conflict Filtering ──────────────────────────────────────────
 /**
@@ -254,6 +506,7 @@ function spawnSliceWorker(basePath, milestoneId, sliceId) {
                 GSD_MILESTONE_LOCK: milestoneId,
                 GSD_PROJECT_ROOT: basePath,
                 GSD_PARALLEL_WORKER: "1",
+                GSD_SLICE_WORKER_TOKEN: worker.workerToken,
             },
             stdio: ["ignore", "pipe", "pipe"],
             detached: false,
@@ -272,8 +525,17 @@ function spawnSliceWorker(basePath, milestoneId, sliceId) {
     });
     worker.process = child;
     worker.pid = child.pid ?? 0;
+    worker.processStartFingerprint = worker.pid > 0
+        ? readProcessStartFingerprint(worker.pid)
+        : null;
     if (!child.pid) {
         worker.process = null;
+        worker.pid = 0;
+        worker.processStartFingerprint = null;
+        try {
+            child.kill("SIGTERM");
+        }
+        catch { /* best-effort */ }
         return false;
     }
     // ── NDJSON stdout monitoring ────────────────────────────────────────
@@ -346,6 +608,9 @@ function spawnSliceWorker(basePath, milestoneId, sliceId) {
             startedAt: w.startedAt,
             worktreePath: w.worktreePath,
         });
+        // Persist worker terminal state for crash recovery.
+        // (Issue #4980 HIGH-8)
+        persistSliceState();
     });
     return true;
 }
@@ -374,6 +639,11 @@ function processSliceWorkerLine(_basePath, _milestoneId, sliceId, line) {
                 sliceState.totalCost += usage.cost;
             }
             worker.completedUnits++;
+            // Persist cost / progress updates so a crash mid-run preserves them.
+            // Throttled (~1/s per process) so high-frequency message_end traffic
+            // does not saturate disk I/O. Worker exit / start / stop paths persist
+            // unthrottled to guarantee the terminal state lands. (Issue #4980 HIGH-8)
+            persistSliceStateThrottled();
         }
     }
 }

package/dist/resources/extensions/gsd/state-transition-matrix.js

@@ -0,0 +1,118 @@
+export const STATE_TRANSITION_MATRIX = [
+    {
+        from: "needs-discussion",
+        event: "context-ready",
+        guard: "CONTEXT artifact exists or PRD/context express path produced context",
+        to: "researching",
+        onFail: "needs-discussion",
+        reasonCode: "state",
+    },
+    {
+        from: "researching",
+        event: "research-ready",
+        guard: "RESEARCH artifact exists or research is explicitly skipped",
+        to: "planning",
+        onFail: "researching",
+        reasonCode: "state",
+    },
+    {
+        from: "planning",
+        event: "plan-ready",
+        guard: "ROADMAP/PLAN artifacts exist and plan gate passes",
+        to: "executing",
+        onFail: "replanning-slice",
+        reasonCode: "state",
+    },
+    {
+        from: "executing",
+        event: "task-dispatched",
+        guard: "task inputs are ready and dependencies are closed",
+        to: "executing",
+        onFail: "blocked",
+        reasonCode: "dependency",
+    },
+    {
+        from: "executing",
+        event: "slice-complete",
+        guard: "all slice tasks are closed and verification gate passes",
+        to: "summarizing",
+        onFail: "validating-milestone",
+        reasonCode: "state",
+    },
+    {
+        from: "summarizing",
+        event: "summary-ready",
+        guard: "SUMMARY artifact exists for the completed work unit",
+        to: "validating-milestone",
+        onFail: "summarizing",
+        reasonCode: "state",
+    },
+    {
+        from: "validating-milestone",
+        event: "validation-pass",
+        guard: "validation verdict is terminal and not remediation-required",
+        to: "completing-milestone",
+        onFail: "blocked",
+        reasonCode: "state",
+    },
+    {
+        from: "blocked",
+        event: "recovery-plan-ready",
+        guard: "reassessment produced an executable next action",
+        to: "executing",
+        onFail: "blocked",
+        reasonCode: "recovery",
+    },
+    {
+        from: "replanning-slice",
+        event: "replan-ready",
+        guard: "replacement slice/task plan exists and plan gate passes",
+        to: "executing",
+        onFail: "blocked",
+        reasonCode: "recovery",
+    },
+    {
+        from: "completing-milestone",
+        event: "closeout-complete",
+        guard: "closeout gate passes and git transaction succeeds",
+        to: "complete",
+        onFail: "blocked",
+        reasonCode: "state",
+    },
+    {
+        from: "*",
+        event: "manual-block",
+        guard: "operator or hard gate requested manual attention",
+        to: "blocked",
+        onFail: "manual-attention",
+        reasonCode: "manual",
+    },
+    {
+        from: "*",
+        event: "retryable-failure",
+        guard: "retry budget remains for failure class",
+        to: "executing",
+        onFail: "blocked",
+        reasonCode: "retry",
+    },
+];
+export function findTransition(from, event) {
+    return STATE_TRANSITION_MATRIX.find((entry) => (entry.from === from || entry.from === "*") && entry.event === event);
+}
+export function validateTransitionMatrix(requiredEvents) {
+    const seen = new Set();
+    const duplicateKeys = [];
+    for (const entry of STATE_TRANSITION_MATRIX) {
+        const key = `${entry.from}:${entry.event}`;
+        if (seen.has(key))
+            duplicateKeys.push(key);
+        seen.add(key);
+    }
+    const availableEvents = new Set(STATE_TRANSITION_MATRIX.map((entry) => entry.event));
+    const missingEvents = requiredEvents.filter((event) => !availableEvents.has(event));
+    return {
+        ok: missingEvents.length === 0 && duplicateKeys.length === 0,
+        missingEvents,
+        duplicateKeys,
+    };
+}