gsd-pi 2.33.1 → 2.34.0-dev.bbb5216

package/dist/resources/extensions/gsd/parallel-orchestrator.js

@@ -0,0 +1,687 @@
+/**
+ * GSD Parallel Orchestrator — Core engine for parallel milestone orchestration.
+ *
+ * Manages worker lifecycle, budget tracking, and coordination. Workers are
+ * separate processes spawned via child_process, each running in its own git
+ * worktree with GSD_MILESTONE_LOCK env var set. The coordinator monitors
+ * workers via session status files (see session-status-io.ts).
+ */
+import { spawn } from "node:child_process";
+import { existsSync, writeFileSync, readFileSync, renameSync, unlinkSync, mkdirSync, } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { gsdRoot } from "./paths.js";
+import { createWorktree, worktreePath } from "./worktree-manager.js";
+import { autoWorktreeBranch, runWorktreePostCreateHook } from "./auto-worktree.js";
+import { nativeBranchExists } from "./native-git-bridge.js";
+import { readIntegrationBranch } from "./git-service.js";
+import { resolveParallelConfig } from "./preferences.js";
+import { writeSessionStatus, readAllSessionStatuses, removeSessionStatus, sendSignal, cleanupStaleSessions, } from "./session-status-io.js";
+import { analyzeParallelEligibility, } from "./parallel-eligibility.js";
+import { getErrorMessage } from "./error-utils.js";
+// ─── Module State ──────────────────────────────────────────────────────────
+let state = null;
+// ─── Persistence ──────────────────────────────────────────────────────────
+const ORCHESTRATOR_STATE_FILE = "orchestrator.json";
+const TMP_SUFFIX = ".tmp";
+function stateFilePath(basePath) {
+    return join(gsdRoot(basePath), ORCHESTRATOR_STATE_FILE);
+}
+/**
+ * Persist the current orchestrator state to .gsd/orchestrator.json.
+ * Uses atomic write (tmp + rename) to prevent partial reads.
+ */
+export function persistState(basePath) {
+    if (!state)
+        return;
+    try {
+        const dir = gsdRoot(basePath);
+        if (!existsSync(dir))
+            mkdirSync(dir, { recursive: true });
+        const persisted = {
+            active: state.active,
+            workers: [...state.workers.values()].map((w) => ({
+                milestoneId: w.milestoneId,
+                title: w.title,
+                pid: w.pid,
+                worktreePath: w.worktreePath,
+                startedAt: w.startedAt,
+                state: w.state,
+                completedUnits: w.completedUnits,
+                cost: w.cost,
+            })),
+            totalCost: state.totalCost,
+            startedAt: state.startedAt,
+            configSnapshot: {
+                max_workers: state.config.max_workers,
+                budget_ceiling: state.config.budget_ceiling,
+            },
+        };
+        const dest = stateFilePath(basePath);
+        const tmp = dest + TMP_SUFFIX;
+        writeFileSync(tmp, JSON.stringify(persisted, null, 2), "utf-8");
+        renameSync(tmp, dest);
+    }
+    catch { /* non-fatal */ }
+}
+/**
+ * Remove the persisted state file.
+ */
+function removeStateFile(basePath) {
+    try {
+        const p = stateFilePath(basePath);
+        if (existsSync(p))
+            unlinkSync(p);
+    }
+    catch { /* non-fatal */ }
+}
+function isPidAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return false;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Restore orchestrator state from .gsd/orchestrator.json.
+ * Checks PID liveness for each worker:
+ * - Living PID → state "running", process stays null (no handle)
+ * - Dead PID → removed from restored state
+ * Returns null if no state file exists or no workers survive.
+ */
+export function restoreState(basePath) {
+    try {
+        const p = stateFilePath(basePath);
+        if (!existsSync(p))
+            return null;
+        const raw = readFileSync(p, "utf-8");
+        const persisted = JSON.parse(raw);
+        // Filter to only workers with living PIDs
+        persisted.workers = persisted.workers.filter((w) => {
+            if (w.state === "stopped" || w.state === "error")
+                return false;
+            return isPidAlive(w.pid);
+        });
+        if (persisted.workers.length === 0) {
+            // No surviving workers — clean up and return null
+            removeStateFile(basePath);
+            return null;
+        }
+        return persisted;
+    }
+    catch {
+        return null;
+    }
+}
+async function waitForWorkerExit(worker, timeoutMs) {
+    if (worker.process) {
+        await new Promise((resolve) => {
+            const done = () => resolve();
+            const timer = setTimeout(done, timeoutMs);
+            worker.process.once("exit", () => {
+                clearTimeout(timer);
+                resolve();
+            });
+        });
+        return worker.process === null || !isPidAlive(worker.pid);
+    }
+    const startedAt = Date.now();
+    while (Date.now() - startedAt < timeoutMs) {
+        if (!isPidAlive(worker.pid))
+            return true;
+        await new Promise((resolve) => setTimeout(resolve, 50));
+    }
+    return !isPidAlive(worker.pid);
+}
+// ─── Accessors ─────────────────────────────────────────────────────────────
+/** Returns true if the orchestrator is active and has been initialized. */
+export function isParallelActive() {
+    return state?.active ?? false;
+}
+/** Returns the current orchestrator state, or null if not initialized. */
+export function getOrchestratorState() {
+    return state;
+}
+/** Returns a snapshot of all tracked workers as an array. */
+export function getWorkerStatuses() {
+    if (!state)
+        return [];
+    return [...state.workers.values()];
+}
+// ─── Preparation ───────────────────────────────────────────────────────────
+/**
+ * Analyze eligibility and prepare for parallel start.
+ * Returns the candidates report without actually starting workers.
+ * Also detects orphaned sessions from prior crashes.
+ */
+export async function prepareParallelStart(basePath, _prefs) {
+    // Detect orphaned sessions before eligibility analysis
+    const sessions = readAllSessionStatuses(basePath);
+    const orphans = [];
+    for (const session of sessions) {
+        const alive = isPidAlive(session.pid);
+        orphans.push({ milestoneId: session.milestoneId, pid: session.pid, alive });
+        if (!alive) {
+            // Clean up dead session
+            removeSessionStatus(basePath, session.milestoneId);
+        }
+    }
+    const candidates = await analyzeParallelEligibility(basePath);
+    return orphans.length > 0 ? { ...candidates, orphans } : candidates;
+}
+// ─── Start ─────────────────────────────────────────────────────────────────
+/**
+ * Start parallel execution with the given eligible milestones.
+ * Creates worktrees, spawns worker processes, and begins monitoring.
+ */
+export async function startParallel(basePath, milestoneIds, prefs) {
+    // Prevent workers from spawning nested parallel sessions
+    if (process.env.GSD_PARALLEL_WORKER) {
+        return { started: [], errors: [{ mid: "all", error: "Cannot start parallel from within a parallel worker" }] };
+    }
+    const config = resolveParallelConfig(prefs);
+    // Try to restore from a previous crash
+    const restored = restoreState(basePath);
+    if (restored && restored.workers.length > 0) {
+        // Adopt surviving workers instead of starting new ones
+        state = {
+            active: true,
+            workers: new Map(),
+            config,
+            totalCost: restored.totalCost,
+            startedAt: restored.startedAt,
+        };
+        const adopted = [];
+        for (const w of restored.workers) {
+            state.workers.set(w.milestoneId, {
+                milestoneId: w.milestoneId,
+                title: w.title,
+                pid: w.pid,
+                process: null, // no handle for adopted workers
+                worktreePath: w.worktreePath,
+                startedAt: w.startedAt,
+                state: "running",
+                completedUnits: w.completedUnits,
+                cost: w.cost,
+            });
+            adopted.push(w.milestoneId);
+        }
+        return { started: adopted, errors: [] };
+    }
+    const now = Date.now();
+    // Initialize orchestrator state
+    state = {
+        active: true,
+        workers: new Map(),
+        config,
+        totalCost: 0,
+        startedAt: now,
+    };
+    const started = [];
+    const errors = [];
+    // Cap to max_workers
+    const toStart = milestoneIds.slice(0, config.max_workers);
+    for (const mid of toStart) {
+        // Check budget ceiling before each spawn
+        if (isBudgetExceeded()) {
+            errors.push({ mid, error: `Budget ceiling ($${config.budget_ceiling}) reached — skipping` });
+            continue;
+        }
+        try {
+            // Create the worktree (without chdir — coordinator stays in project root)
+            let wtPath;
+            try {
+                wtPath = createMilestoneWorktree(basePath, mid);
+            }
+            catch {
+                // Worktree creation may fail in test environments or when git
+                // is not available. Fall back to a placeholder path.
+                wtPath = worktreePath(basePath, mid);
+            }
+            const worker = {
+                milestoneId: mid,
+                title: mid,
+                pid: 0, // placeholder — real PID set by spawnWorker()
+                process: null,
+                worktreePath: wtPath,
+                startedAt: now,
+                state: "running",
+                completedUnits: 0,
+                cost: 0,
+            };
+            state.workers.set(mid, worker);
+            // Spawn BEFORE writing session status so the file gets the real worker PID.
+            const spawned = spawnWorker(basePath, mid);
+            if (!spawned) {
+                worker.state = "error";
+            }
+            // Write session status with real PID (or 0 if spawn failed)
+            writeSessionStatus(basePath, {
+                milestoneId: mid,
+                pid: worker.pid,
+                state: worker.state,
+                currentUnit: null,
+                completedUnits: 0,
+                cost: 0,
+                lastHeartbeat: now,
+                startedAt: now,
+                worktreePath: wtPath,
+            });
+            started.push(mid);
+        }
+        catch (err) {
+            const message = getErrorMessage(err);
+            errors.push({ mid, error: message });
+        }
+    }
+    // If nothing started successfully, deactivate
+    if (started.length === 0) {
+        state.active = false;
+    }
+    // Persist state for crash recovery
+    persistState(basePath);
+    return { started, errors };
+}
+// ─── Worktree Creation ────────────────────────────────────────────────────
+/**
+ * Create a git worktree for a milestone without changing the coordinator's cwd.
+ * Uses milestone/<MID> branch naming (same as auto-worktree.ts).
+ */
+function createMilestoneWorktree(basePath, milestoneId) {
+    const branch = autoWorktreeBranch(milestoneId);
+    const branchExists = nativeBranchExists(basePath, branch);
+    let info;
+    if (branchExists) {
+        info = createWorktree(basePath, milestoneId, { branch, reuseExistingBranch: true });
+    }
+    else {
+        const integrationBranch = readIntegrationBranch(basePath, milestoneId) ?? undefined;
+        info = createWorktree(basePath, milestoneId, { branch, startPoint: integrationBranch });
+    }
+    // Run post-create hook if configured
+    runWorktreePostCreateHook(basePath, info.path);
+    return info.path;
+}
+// ─── Worker Spawning ───────────────────────────────────────────────────
+/**
+ * Spawn a worker process for a milestone.
+ * The worker runs `gsd --print "/gsd auto"` in the milestone's worktree
+ * with GSD_MILESTONE_LOCK set to isolate state derivation.
+ */
+export function spawnWorker(basePath, milestoneId) {
+    if (!state)
+        return false;
+    const worker = state.workers.get(milestoneId);
+    if (!worker)
+        return false;
+    if (worker.process)
+        return true; // already spawned
+    // Resolve the GSD CLI binary path
+    const binPath = resolveGsdBin();
+    if (!binPath)
+        return false;
+    let child;
+    try {
+        child = spawn(process.execPath, [binPath, "--mode", "json", "--print", "/gsd auto"], {
+            cwd: worker.worktreePath,
+            env: {
+                ...process.env,
+                GSD_MILESTONE_LOCK: milestoneId,
+                // Prevent workers from spawning their own parallel sessions
+                GSD_PARALLEL_WORKER: "1",
+            },
+            stdio: ["ignore", "pipe", "pipe"],
+            detached: false,
+        });
+    }
+    catch {
+        return false;
+    }
+    // Handle spawn errors (e.g., ENOENT when binary doesn't exist)
+    child.on("error", () => {
+        if (!state)
+            return;
+        const w = state.workers.get(milestoneId);
+        if (w) {
+            w.process = null;
+            // Don't change state — spawn failure is non-fatal, coordinator can retry
+        }
+    });
+    worker.process = child;
+    worker.pid = child.pid ?? 0;
+    if (!child.pid) {
+        // Spawn returned but no PID — process failed to start
+        worker.process = null;
+        return false;
+    }
+    // ── NDJSON stdout monitoring ────────────────────────────────────────
+    // Workers run with --mode json, emitting one JSON event per line.
+    // We parse message_end events to extract cost/token usage, keeping
+    // the coordinator's cost tracking in sync with actual API spend.
+    if (child.stdout) {
+        let stdoutBuffer = "";
+        child.stdout.on("data", (data) => {
+            stdoutBuffer += data.toString();
+            const lines = stdoutBuffer.split("\n");
+            stdoutBuffer = lines.pop() || "";
+            for (const line of lines) {
+                processWorkerLine(basePath, milestoneId, line);
+            }
+        });
+        // Flush remaining buffer on close
+        child.stdout.on("close", () => {
+            if (stdoutBuffer.trim()) {
+                processWorkerLine(basePath, milestoneId, stdoutBuffer);
+            }
+        });
+    }
+    // Update session status with real PID
+    writeSessionStatus(basePath, {
+        milestoneId,
+        pid: worker.pid,
+        state: "running",
+        currentUnit: null,
+        completedUnits: worker.completedUnits,
+        cost: worker.cost,
+        lastHeartbeat: Date.now(),
+        startedAt: worker.startedAt,
+        worktreePath: worker.worktreePath,
+    });
+    // Handle worker exit
+    child.on("exit", (code) => {
+        if (!state)
+            return;
+        const w = state.workers.get(milestoneId);
+        if (!w)
+            return;
+        w.process = null;
+        if (w.state === "stopped")
+            return; // graceful stop, already handled
+        if (code === 0) {
+            w.state = "stopped";
+        }
+        else {
+            w.state = "error";
+        }
+        // Update session status and persist orchestrator state for crash recovery
+        writeSessionStatus(basePath, {
+            milestoneId,
+            pid: w.pid,
+            state: w.state,
+            currentUnit: null,
+            completedUnits: w.completedUnits,
+            cost: w.cost,
+            lastHeartbeat: Date.now(),
+            startedAt: w.startedAt,
+            worktreePath: w.worktreePath,
+        });
+        persistState(basePath);
+    });
+    return true;
+}
+/**
+ * Resolve the GSD CLI binary path.
+ * Uses GSD_BIN_PATH env var (set by loader.ts) or falls back to
+ * finding the binary relative to the current module.
+ */
+function resolveGsdBin() {
+    // GSD_BIN_PATH is set by loader.ts to the absolute path of dist/loader.js
+    if (process.env.GSD_BIN_PATH && existsSync(process.env.GSD_BIN_PATH)) {
+        return process.env.GSD_BIN_PATH;
+    }
+    // Fallback: try to find loader.js relative to this file
+    // This file is at dist/resources/extensions/gsd/parallel-orchestrator.js
+    // loader.js is at dist/loader.js
+    let thisDir;
+    try {
+        thisDir = dirname(fileURLToPath(import.meta.url));
+    }
+    catch {
+        thisDir = process.cwd();
+    }
+    const candidates = [
+        join(thisDir, "..", "..", "..", "loader.js"),
+        join(thisDir, "..", "..", "..", "..", "dist", "loader.js"),
+    ];
+    for (const candidate of candidates) {
+        if (existsSync(candidate))
+            return candidate;
+    }
+    return null;
+}
+// ─── NDJSON Processing ──────────────────────────────────────────────────────
+/**
+ * Process a single NDJSON line from a worker's stdout.
+ * Extracts cost and token usage from message_end events and updates
+ * the worker's tracking state + session status file.
+ */
+function processWorkerLine(basePath, milestoneId, line) {
+    if (!line.trim() || !state)
+        return;
+    let event;
+    try {
+        event = JSON.parse(line);
+    }
+    catch {
+        return; // Not valid JSON — skip (stderr leakage, debug output, etc.)
+    }
+    const type = String(event.type ?? "");
+    // message_end carries usage data with cost
+    if (type === "message_end" && event.message) {
+        const msg = event.message;
+        const usage = msg.usage;
+        if (usage) {
+            const cost = usage.cost?.total;
+            if (typeof cost === "number") {
+                const worker = state.workers.get(milestoneId);
+                if (worker) {
+                    worker.cost += cost;
+                    // Update aggregate
+                    state.totalCost = 0;
+                    for (const w of state.workers.values()) {
+                        state.totalCost += w.cost;
+                    }
+                }
+            }
+        }
+        // Track completed units (each message_end from assistant = progress)
+        if (msg.role === "assistant") {
+            const worker = state.workers.get(milestoneId);
+            if (worker) {
+                worker.completedUnits++;
+            }
+        }
+        // Update session status file so dashboard sees live cost
+        const worker = state.workers.get(milestoneId);
+        if (worker) {
+            writeSessionStatus(basePath, {
+                milestoneId,
+                pid: worker.pid,
+                state: worker.state,
+                currentUnit: null,
+                completedUnits: worker.completedUnits,
+                cost: worker.cost,
+                lastHeartbeat: Date.now(),
+                startedAt: worker.startedAt,
+                worktreePath: worker.worktreePath,
+            });
+        }
+    }
+    // tool_execution_start can track current unit
+    if (type === "extension_ui_request" && event.method === "notify") {
+        // GSD auto-mode sends notifications about current unit
+        const worker = state.workers.get(milestoneId);
+        if (worker) {
+            writeSessionStatus(basePath, {
+                milestoneId,
+                pid: worker.pid,
+                state: worker.state,
+                currentUnit: null,
+                completedUnits: worker.completedUnits,
+                cost: worker.cost,
+                lastHeartbeat: Date.now(),
+                startedAt: worker.startedAt,
+                worktreePath: worker.worktreePath,
+            });
+        }
+    }
+}
+// ─── Stop ──────────────────────────────────────────────────────────────────
+/**
+ * Stop all workers or a specific milestone's worker.
+ * Sends stop signals and updates tracking state.
+ */
+export async function stopParallel(basePath, milestoneId) {
+    if (!state)
+        return;
+    const targets = milestoneId
+        ? [milestoneId]
+        : [...state.workers.keys()];
+    for (const mid of targets) {
+        const worker = state.workers.get(mid);
+        if (!worker)
+            continue;
+        // Send stop signal via file-based IPC (worker checks on next dispatch)
+        sendSignal(basePath, mid, "stop");
+        // Send SIGTERM to the process for immediate response.
+        // Use process handle when available, fall back to PID-based kill
+        // (handles are null after coordinator restart / deserialization).
+        if (worker.pid > 0) {
+            try {
+                if (worker.process) {
+                    worker.process.kill("SIGTERM");
+                }
+                else if (worker.pid !== process.pid) {
+                    process.kill(worker.pid, "SIGTERM");
+                }
+            }
+            catch { /* process may already be dead */ }
+        }
+        const exitedAfterTerm = await waitForWorkerExit(worker, 750);
+        if (!exitedAfterTerm && worker.pid > 0) {
+            try {
+                if (worker.process) {
+                    worker.process.kill("SIGKILL");
+                }
+                else if (worker.pid !== process.pid) {
+                    process.kill(worker.pid, "SIGKILL");
+                }
+            }
+            catch { /* process may already be dead */ }
+            await waitForWorkerExit(worker, 250);
+        }
+        // Update in-memory state
+        worker.state = "stopped";
+        worker.process = null;
+        // Clean up session status file
+        removeSessionStatus(basePath, mid);
+    }
+    // If stopping all workers, deactivate the orchestrator
+    if (!milestoneId) {
+        state.active = false;
+    }
+    // Persist final state and clean up state file
+    removeStateFile(basePath);
+}
+export async function shutdownParallel(basePath) {
+    if (!state)
+        return;
+    await stopParallel(basePath);
+    resetOrchestrator();
+}
+// ─── Pause / Resume ────────────────────────────────────────────────────────
+/** Pause a specific worker or all workers. */
+export function pauseWorker(basePath, milestoneId) {
+    if (!state)
+        return;
+    const targets = milestoneId
+        ? [milestoneId]
+        : [...state.workers.keys()];
+    for (const mid of targets) {
+        const worker = state.workers.get(mid);
+        if (!worker || worker.state !== "running")
+            continue;
+        sendSignal(basePath, mid, "pause");
+        worker.state = "paused";
+    }
+}
+/** Resume a specific worker or all workers. */
+export function resumeWorker(basePath, milestoneId) {
+    if (!state)
+        return;
+    const targets = milestoneId
+        ? [milestoneId]
+        : [...state.workers.keys()];
+    for (const mid of targets) {
+        const worker = state.workers.get(mid);
+        if (!worker || worker.state !== "paused")
+            continue;
+        sendSignal(basePath, mid, "resume");
+        worker.state = "running";
+    }
+}
+// ─── Status Refresh ────────────────────────────────────────────────────────
+/**
+ * Poll worker statuses from disk and update orchestrator state.
+ * Call this periodically from the dashboard refresh cycle.
+ */
+export function refreshWorkerStatuses(basePath) {
+    if (!state)
+        return;
+    // Clean up stale sessions first
+    const staleIds = cleanupStaleSessions(basePath);
+    for (const mid of staleIds) {
+        const worker = state.workers.get(mid);
+        if (worker) {
+            worker.state = "error";
+            worker.process = null;
+        }
+    }
+    // Read all live session statuses from disk
+    const statuses = readAllSessionStatuses(basePath);
+    const statusMap = new Map();
+    for (const s of statuses) {
+        statusMap.set(s.milestoneId, s);
+    }
+    // Update in-memory worker state from disk data
+    for (const [mid, worker] of state.workers) {
+        const diskStatus = statusMap.get(mid);
+        if (!diskStatus)
+            continue;
+        worker.state = diskStatus.state;
+        worker.completedUnits = diskStatus.completedUnits;
+        worker.cost = diskStatus.cost;
+        worker.pid = diskStatus.pid;
+    }
+    // Recalculate aggregate cost
+    state.totalCost = 0;
+    for (const worker of state.workers.values()) {
+        state.totalCost += worker.cost;
+    }
+    // Persist updated state for crash recovery
+    persistState(basePath);
+}
+// ─── Budget ────────────────────────────────────────────────────────────────
+/** Get aggregate cost across all workers. */
+export function getAggregateCost() {
+    if (!state)
+        return 0;
+    return state.totalCost;
+}
+/** Check if budget ceiling has been reached. */
+export function isBudgetExceeded() {
+    if (!state)
+        return false;
+    if (state.config.budget_ceiling == null)
+        return false;
+    return state.totalCost >= state.config.budget_ceiling;
+}
+// ─── Reset ─────────────────────────────────────────────────────────────────
+/** Reset orchestrator state. Called on clean shutdown. */
+export function resetOrchestrator() {
+    state = null;
+}