@fitchmultz/pi-subagents 0.0.0-stage → 0.44.4

package/dist/runs/background/stale-run-reconciler.js

@@ -0,0 +1,335 @@
+import * as fs from "node:fs";
+import { readOutputPage } from "../../shared/journal-reader.js";
+import * as path from "node:path";
+import { writeAtomicJson } from "../../shared/atomic-json.js";
+import { RESULTS_DIR, RUNNER_ERROR_LOG_FILE } from "../../shared/types.js";
+import { normalizeParallelGroups } from "./parallel-groups.js";
+import { nestedSummaryFromAsyncStatus, projectNestedEvents, resolveNestedAsyncDir, writeNestedEvent } from "../shared/nested-events.js";
+import { isDurableRun, readAsyncResultFileIfExists } from "./async-result-file.js";
+import { readRunJson } from "../shared/supervisor-questions.js";
+import { readStatus } from "../../shared/utils.js";
+function isNotFoundError(error) {
+    return typeof error === "object"
+        && error !== null
+        && "code" in error
+        && error.code === "ENOENT";
+}
+function appendJsonl(filePath, payload) {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.appendFileSync(filePath, `${JSON.stringify(payload)}\n`, "utf-8");
+}
+function readStatusFile(asyncDir) {
+    return readStatus(asyncDir);
+}
+function readResultRepairData(resultPath) {
+    const data = readAsyncResultFileIfExists(resultPath);
+    if (!data)
+        return undefined;
+    return { state: data.terminalState, timestamp: data.timestamp, ...(Array.isArray(data.results) ? { results: data.results } : {}) };
+}
+function childState(overallState, child) {
+    if (child?.success === true)
+        return "complete";
+    if (child?.interrupted === true)
+        return "paused";
+    if (child?.acceptance?.status === "blocked" && (child.exitCode === 0 || child.exitCode === undefined))
+        return "blocked";
+    if (child?.success === false)
+        return "failed";
+    return overallState;
+}
+function withoutLiveActivity(value) {
+    return {
+        ...value,
+        activityState: undefined,
+        currentTool: undefined,
+        currentToolArgs: undefined,
+        currentToolStartedAt: undefined,
+        currentPath: undefined,
+    };
+}
+function terminalStatusFromResult(status, resultPath, now) {
+    const repair = readResultRepairData(resultPath);
+    if (!repair)
+        return undefined;
+    now = repair.timestamp ?? now;
+    const steps = (status.steps ?? []).map((step, index) => {
+        if (step.status !== "running" && step.status !== "pending")
+            return withoutLiveActivity(step);
+        const child = repair.results?.[index];
+        if (isDurableRun(status) && step.status === "pending" && !child)
+            return withoutLiveActivity(step);
+        const state = childState(repair.state, child);
+        return withoutLiveActivity({
+            ...step,
+            status: state === "complete" ? "complete" : state,
+            endedAt: step.endedAt ?? now,
+            durationMs: step.startedAt !== undefined && step.durationMs === undefined ? Math.max(0, now - step.startedAt) : step.durationMs,
+            exitCode: step.exitCode ?? (state === "complete" || state === "blocked" || state === "paused" ? 0 : 1),
+            error: state === "failed" ? step.error ?? child?.error : step.error,
+            sessionFile: step.sessionFile ?? child?.sessionFile,
+            model: step.model ?? child?.model,
+            attemptedModels: step.attemptedModels ?? child?.attemptedModels,
+            modelAttempts: step.modelAttempts ?? child?.modelAttempts,
+            acceptance: step.acceptance ?? child?.acceptance,
+            agentProcessExit: step.agentProcessExit ?? child?.agentProcessExit,
+        });
+    });
+    return withoutLiveActivity({
+        ...status,
+        state: repair.state,
+        lastUpdate: now,
+        endedAt: status.endedAt ?? now,
+        steps,
+    });
+}
+function buildStartedStatus(asyncDir, startedRun, now) {
+    const startedAt = startedRun.startedAt ?? now;
+    const agents = startedRun.agents?.length ? startedRun.agents : ["subagent"];
+    const chainStepCount = startedRun.chainStepCount;
+    const parallelGroups = chainStepCount !== undefined
+        ? normalizeParallelGroups(startedRun.parallelGroups, agents.length, chainStepCount)
+        : [];
+    return {
+        runId: startedRun.runId || path.basename(asyncDir),
+        ...(startedRun.sessionId ? { sessionId: startedRun.sessionId } : {}),
+        mode: startedRun.mode ?? "single",
+        state: "running",
+        pid: startedRun.pid,
+        startedAt,
+        lastUpdate: now,
+        currentStep: 0,
+        ...(chainStepCount !== undefined ? { chainStepCount } : {}),
+        ...(parallelGroups.length ? { parallelGroups } : {}),
+        steps: agents.map((agent) => ({
+            agent,
+            status: "running",
+            startedAt,
+        })),
+        ...(startedRun.sessionFile ? { sessionFile: startedRun.sessionFile } : {}),
+    };
+}
+function readCompletedStepOutput(asyncDir, index) {
+    try {
+        return readOutputPage(path.join(asyncDir, `output-${index}.log`)).text.trim();
+    }
+    catch (error) {
+        if (isNotFoundError(error))
+            return "";
+        throw error;
+    }
+}
+const RUNNER_STDERR_EXCERPT_BYTES = 16 * 1024;
+const UNSAFE_CONTROL_CHARACTERS = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F]/g;
+function readRunnerStderr(asyncDir) {
+    let fd;
+    try {
+        fd = fs.openSync(path.join(asyncDir, RUNNER_ERROR_LOG_FILE), "r");
+        const size = fs.fstatSync(fd).size;
+        const length = Math.min(size, RUNNER_STDERR_EXCERPT_BYTES);
+        const buffer = Buffer.alloc(length);
+        const bytesRead = fs.readSync(fd, buffer, 0, length, Math.max(0, size - length));
+        const excerpt = buffer.subarray(0, bytesRead).toString("utf-8").replace(UNSAFE_CONTROL_CHARACTERS, "�").trim();
+        if (!excerpt)
+            return undefined;
+        return size > length ? `[runner stderr truncated to last ${length} bytes]\n${excerpt}` : excerpt;
+    }
+    catch {
+        return undefined;
+    }
+    finally {
+        if (fd !== undefined) {
+            try {
+                fs.closeSync(fd);
+            }
+            catch { }
+        }
+    }
+}
+function buildFailedRepair(status, asyncDir, now, reason) {
+    const runId = status.runId || path.basename(asyncDir);
+    const pid = typeof status.pid === "number" ? status.pid : "unknown";
+    const defaultMessage = `Async runner process ${pid} exited or disappeared before writing a result. Completion is unconfirmed.`;
+    const runnerStderr = reason ? undefined : readRunnerStderr(asyncDir);
+    const message = reason ?? (runnerStderr ? `${defaultMessage}\n\nRunner stderr:\n${runnerStderr}` : defaultMessage);
+    const steps = status.steps?.length ? status.steps : [{ agent: "subagent", status: "running" }];
+    const repairedSteps = steps.map((step) => step.status === "running" || String(step.status) === "queued" || step.status === "pending"
+        ? withoutLiveActivity({
+            ...step,
+            status: "failed",
+            endedAt: step.endedAt ?? now,
+            durationMs: step.startedAt !== undefined && step.durationMs === undefined ? Math.max(0, now - step.startedAt) : step.durationMs,
+            exitCode: step.exitCode ?? 1,
+            error: step.error ?? message,
+        })
+        : withoutLiveActivity(step));
+    const repairedStatus = withoutLiveActivity({
+        ...status,
+        state: "failed",
+        lastUpdate: now,
+        endedAt: now,
+        steps: repairedSteps,
+    });
+    const resultAgent = repairedSteps[status.currentStep ?? 0]?.agent ?? repairedSteps[0]?.agent ?? "subagent";
+    return {
+        status: repairedStatus,
+        message,
+        result: {
+            id: runId,
+            agent: resultAgent,
+            mode: status.mode,
+            success: false,
+            state: "failed",
+            summary: message,
+            results: repairedSteps.map((step, index) => {
+                const success = step.status === "complete" || step.status === "completed";
+                return {
+                    agent: step.agent,
+                    output: success ? readCompletedStepOutput(asyncDir, index) : message,
+                    error: success ? undefined : step.error ?? message,
+                    success,
+                    model: step.model,
+                    attemptedModels: step.attemptedModels,
+                    modelAttempts: step.modelAttempts,
+                    sessionFile: step.sessionFile,
+                };
+            }),
+            exitCode: 1,
+            timestamp: now,
+            durationMs: Math.max(0, now - status.startedAt),
+            asyncDir,
+            sessionId: status.sessionId,
+            cwd: status.cwd,
+            sessionFile: status.sessionFile,
+        },
+    };
+}
+function writeFailedRepair(asyncDir, status, resultPath, now, reason) {
+    const repair = buildFailedRepair(status, asyncDir, now, reason);
+    writeAtomicJson(resultPath, repair.result);
+    writeAtomicJson(path.join(asyncDir, "status.json"), repair.status);
+    appendJsonl(path.join(asyncDir, "events.jsonl"), {
+        type: "subagent.run.repaired_stale",
+        ts: now,
+        runId: repair.status.runId,
+        pid: status.pid,
+        resultPath,
+        message: repair.message,
+    });
+    return { status: repair.status, repaired: true, resultPath, message: repair.message };
+}
+function terminal(state) {
+    return state === "complete" || state === "failed" || state === "blocked" || state === "paused";
+}
+function* nestedRuns(children) {
+    for (const child of children ?? []) {
+        yield child;
+        yield* nestedRuns(child.children);
+        yield* nestedRuns(child.steps?.flatMap((step) => step.children ?? []));
+    }
+}
+export function reconcileNestedAsyncDescendants(route, options = {}) {
+    const registry = projectNestedEvents(route);
+    for (const run of nestedRuns(registry.children)) {
+        if (run.state !== "running" && run.state !== "queued")
+            continue;
+        const asyncDir = resolveNestedAsyncDir(route.rootRunId, run);
+        if (!asyncDir)
+            continue;
+        const result = reconcileAsyncRun(asyncDir, {
+            ...options,
+            resultsDir: path.join(options.resultsDir ?? RESULTS_DIR, "nested", route.rootRunId),
+        });
+        const status = result.status;
+        if (!status)
+            continue;
+        const ts = options.now?.() ?? Date.now();
+        const child = nestedSummaryFromAsyncStatus(status, asyncDir, {
+            id: run.id, parentRunId: run.parentRunId, parentStepIndex: run.parentStepIndex,
+            depth: run.depth, path: run.path, mode: run.mode, ts,
+        });
+        child.steps?.forEach((step, index) => { step.children = run.steps?.[index]?.children; });
+        Object.assign(run, child, result.message ? { error: result.message } : {});
+        if (isDurableRun(status))
+            continue;
+        if (!result.repaired && !terminal(status.state))
+            continue;
+        writeNestedEvent(route, {
+            type: terminal(status.state) ? "subagent.nested.completed" : "subagent.nested.updated",
+            ts,
+            parentRunId: run.parentRunId,
+            parentStepIndex: run.parentStepIndex,
+            child,
+        });
+    }
+    return registry.children;
+}
+export function checkPidLiveness(pid, kill = process.kill) {
+    try {
+        kill(pid, 0);
+        return "alive";
+    }
+    catch (error) {
+        const code = typeof error === "object" && error !== null && "code" in error
+            ? error.code
+            : undefined;
+        if (code === "ESRCH")
+            return "dead";
+        if (code === "EPERM")
+            return "unknown";
+        return "unknown";
+    }
+}
+export function reconcileAsyncRun(asyncDir, options = {}, record) {
+    const now = options.now?.() ?? Date.now();
+    const status = record ? record.status : readStatusFile(asyncDir);
+    const startedStatus = !status && options.startedRun ? buildStartedStatus(asyncDir, options.startedRun, now) : undefined;
+    const effectiveStatus = status ?? startedStatus;
+    if (!effectiveStatus)
+        return { status: null, repaired: false };
+    const runId = effectiveStatus.runId || path.basename(asyncDir);
+    const durable = isDurableRun(effectiveStatus) || (record ? record.durable : isDurableRun(readRunJson(path.join(asyncDir, "launch.json"))));
+    const resultPath = durable ? path.join(asyncDir, "result.json") : path.join(options.resultsDir ?? RESULTS_DIR, `${runId}.json`);
+    if (durable) {
+        // The detached owner is the only execution writer. Inspection projects evidence.
+        const final = terminalStatusFromResult(effectiveStatus, resultPath, effectiveStatus.endedAt ?? now);
+        if (final)
+            return { status: final, repaired: false, resultPath };
+        if (typeof effectiveStatus.pid === "number" && checkPidLiveness(effectiveStatus.pid, options.kill) === "dead") {
+            const failure = buildFailedRepair(effectiveStatus, asyncDir, effectiveStatus.lastUpdate ?? effectiveStatus.startedAt);
+            return { status: failure.status, repaired: false, message: failure.message };
+        }
+        if (terminal(effectiveStatus.state))
+            return { status: { ...effectiveStatus, state: "running" }, repaired: false, message: "Waiting for the owner's final result. Completion is unconfirmed." };
+        return { status: effectiveStatus, repaired: false };
+    }
+    if (fs.existsSync(resultPath)) {
+        const terminalStatus = effectiveStatus.state === "running" || effectiveStatus.state === "queued"
+            ? terminalStatusFromResult(effectiveStatus, resultPath, now)
+            : undefined;
+        if (terminalStatus) {
+            writeAtomicJson(path.join(asyncDir, "status.json"), terminalStatus);
+            return { status: terminalStatus, repaired: true, resultPath, message: "Existing async result file was used to repair stale running status." };
+        }
+        return { status: effectiveStatus, repaired: false, resultPath };
+    }
+    if ((effectiveStatus.state !== "running" && effectiveStatus.state !== "queued") || typeof effectiveStatus.pid !== "number") {
+        return { status: status ?? null, repaired: false, resultPath };
+    }
+    if (!status) {
+        const startedAt = options.startedRun?.startedAt ?? effectiveStatus.startedAt;
+        if (now - startedAt < (options.missingStatusGraceMs ?? 1000)) {
+            return { status: null, repaired: false, resultPath };
+        }
+    }
+    const liveness = checkPidLiveness(effectiveStatus.pid, options.kill);
+    if (liveness !== "dead") {
+        const staleAfterMs = options.staleAlivePidMs ?? 24 * 60 * 60 * 1000;
+        const lastUpdate = effectiveStatus.lastUpdate ?? effectiveStatus.startedAt;
+        if (now - lastUpdate <= staleAfterMs)
+            return { status: status ?? null, repaired: false, resultPath };
+        const message = `Async runner process ${effectiveStatus.pid} still has a live PID, but status has not updated for ${now - lastUpdate}ms. Marked run failed by stale-run reconciliation because PID ownership cannot be verified.`;
+        return writeFailedRepair(asyncDir, effectiveStatus, resultPath, now, message);
+    }
+    return writeFailedRepair(asyncDir, effectiveStatus, resultPath, now);
+}

package/dist/runs/background/subagent-runner-launcher.js

@@ -0,0 +1,55 @@
+import { spawn } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+const RETRY_WINDOW_MS = 30_000;
+const RETRY_DELAY_MS = 500;
+const MAX_STDERR_LENGTH = 64 * 1024;
+function run(runnerPath, configPath, piPackageRoot) {
+    return new Promise((resolve) => {
+        const child = spawn(process.execPath, [runnerPath, configPath], {
+            stdio: ["ignore", "inherit", "pipe"],
+            env: piPackageRoot ? { ...process.env, PI_PACKAGE_DIR: process.env.PI_PACKAGE_DIR ?? piPackageRoot } : process.env,
+        });
+        let stderr = "";
+        child.stderr.on("data", (chunk) => {
+            process.stderr.write(chunk);
+            stderr = `${stderr}${chunk.toString()}`.slice(-MAX_STDERR_LENGTH);
+        });
+        child.once("error", (error) => {
+            const stderr = error.stack ?? error.message;
+            process.stderr.write(`${stderr}\n`);
+            resolve({ code: 1, stderr });
+        });
+        child.once("close", (code) => resolve({ code: code ?? 1, stderr }));
+    });
+}
+const [runnerPath, configPath] = process.argv.slice(2);
+if (!runnerPath || !configPath)
+    throw new Error("Usage: subagent-runner-launcher <runner> <config>");
+let statusPath;
+let piPackageRoot;
+try {
+    const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+    if (typeof config.piPackageRoot === "string")
+        piPackageRoot = config.piPackageRoot;
+    if (typeof config.asyncDir === "string")
+        statusPath = path.join(config.asyncDir, "status.json");
+}
+catch { }
+// ponytail: bridge short in-place extension updates; use shared update locking if longer gaps appear.
+const deadline = Date.now() + RETRY_WINDOW_MS;
+let announcedRetry = false;
+while (true) {
+    const result = await run(runnerPath, configPath, piPackageRoot);
+    const failedBeforeStartup = !statusPath || !fs.existsSync(statusPath);
+    if (result.code === 0 || !failedBeforeStartup || !result.stderr.includes("ERR_MODULE_NOT_FOUND") || Date.now() >= deadline) {
+        process.exitCode = result.code;
+        break;
+    }
+    if (!announcedRetry) {
+        process.stderr.write("[pi-subagents] Runner dependencies are being updated; retrying startup.\n");
+        announcedRetry = true;
+    }
+    await delay(RETRY_DELAY_MS);
+}