@fitchmultz/pi-subagents 0.0.0-stage → 0.44.4

package/dist/runs/background/result-watcher.js

@@ -0,0 +1,426 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { setImmediate as yieldToInput } from "node:timers/promises";
+import { getRunMetadataDir } from "../shared/supervisor-questions.js";
+import { buildCompletionKey, markSeenWithTtl } from "./completion-dedupe.js";
+import { createFileCoalescer } from "../../shared/file-coalescer.js";
+import { journalStamp } from "../../shared/journal-reader.js";
+import { resolveOrchestratorIntercomTarget } from "../../intercom/intercom-bridge.js";
+import { SUBAGENT_ASYNC_COMPLETE_EVENT, } from "../../shared/types.js";
+import { attachNestedChildrenToResultChildren, buildSubagentResultIntercomPayload, compactNestedResultChildren, deliverSubagentResultIntercomEvent, resolveSubagentResultStatus, } from "../../intercom/result-intercom.js";
+import { projectNestedRegistryForRoot, sanitizeSummary } from "../shared/nested-events.js";
+import { isDurableRun, readAsyncResultFile } from "./async-result-file.js";
+const WATCHER_RESTART_DELAY_MS = 3000;
+const POLL_INTERVAL_MS = 3000;
+function sanitizeNestedResultChildren(value, resultPath, label) {
+    if (value === undefined)
+        return undefined;
+    if (!Array.isArray(value)) {
+        console.error(`Ignoring invalid nested children in subagent result file '${resultPath}' at ${label}: expected an array.`);
+        return undefined;
+    }
+    const children = value.map((child) => sanitizeSummary(child)).filter((child) => Boolean(child));
+    if (children.length !== value.length) {
+        console.error(`Ignoring ${value.length - children.length} invalid nested child record(s) in subagent result file '${resultPath}' at ${label}.`);
+    }
+    return children.length ? children : undefined;
+}
+function getErrorCode(error) {
+    return typeof error === "object" && error !== null && "code" in error
+        ? error.code
+        : undefined;
+}
+function isNotFoundError(error) {
+    if (getErrorCode(error) === "ENOENT")
+        return true;
+    const cause = error instanceof Error ? error.cause : undefined;
+    return getErrorCode(cause) === "ENOENT";
+}
+function shouldFallBackToPolling(error) {
+    const code = getErrorCode(error);
+    return code === "EMFILE" || code === "ENOSPC";
+}
+export function createResultWatcher(pi, state, resultsDir, deps = {}) {
+    let periodicScanTimer = null;
+    const processingCompletionKeys = new Set();
+    const inFlight = new Set();
+    let startTail = Promise.resolve(), generation = 0, preserveBeforeGeneration = 0;
+    let joiningGeneration;
+    const foreignResults = new Map();
+    const pendingResultFiles = () => {
+        const files = fs.existsSync(resultsDir) ? fs.readdirSync(resultsDir).filter((name) => name.endsWith(".json")) : [];
+        const notified = new Map(files.map((file, index) => [file, index]));
+        for (const run of state.ownedRuns?.values() ?? []) {
+            if (run.source !== "async" || run.accounting?.state !== "incomplete" && run.delivery?.entryId)
+                continue;
+            const file = path.join(getRunMetadataDir(run.runId), "result.json");
+            if (!fs.existsSync(file))
+                continue;
+            // Disposable hints cannot veto canonical recovery, including corrupt hints.
+            const notification = notified.get(`${run.runId}.json`);
+            if (notification === undefined)
+                files.push(file);
+            else
+                files[notification] = file;
+        }
+        const present = new Set(files.map((file) => path.isAbsolute(file) ? file : path.join(resultsDir, file)));
+        for (const file of foreignResults.keys())
+            if (!present.has(file))
+                foreignResults.delete(file);
+        return files.filter((file) => {
+            const resultPath = path.isAbsolute(file) ? file : path.join(resultsDir, file);
+            try {
+                return !isForeignUnchanged(resultPath);
+            }
+            catch (error) {
+                if (isNotFoundError(error)) {
+                    foreignResults.delete(resultPath);
+                    return false;
+                }
+                return true;
+            }
+        });
+    };
+    const isForeignUnchanged = (resultPath) => {
+        const cached = foreignResults.get(resultPath);
+        if (!cached || cached.sessionId !== state.currentSessionId || cached.ownerSessionId !== state.ownedRuns?.get(cached.runId)?.ownerSessionId)
+            return false;
+        return cached.stamp === journalStamp(fs.statSync(resultPath, { bigint: true }))
+            && (!cached.canonicalPath || cached.canonicalStamp === journalStamp(fs.statSync(cached.canonicalPath, { bigint: true })));
+    };
+    const handleResult = async (file) => {
+        const startedGeneration = generation, ownerSessionId = state.currentSessionId;
+        let claimedCompletionKey;
+        let completionEmitted = false;
+        const durableFile = path.isAbsolute(file);
+        const resultPath = durableFile ? file : path.join(resultsDir, file);
+        if (!fs.existsSync(resultPath))
+            return;
+        try {
+            if (isForeignUnchanged(resultPath))
+                return;
+            const stamp = journalStamp(fs.statSync(resultPath, { bigint: true }));
+            const notification = readAsyncResultFile(resultPath);
+            const runId = notification.runId ?? notification.id ?? path.basename(file, ".json");
+            const run = state.ownedRuns?.get(runId);
+            const canonicalPath = isDurableRun(notification) && !durableFile ? path.join(getRunMetadataDir(runId), "result.json") : undefined;
+            const canonicalStamp = canonicalPath ? journalStamp(fs.statSync(canonicalPath, { bigint: true })) : undefined;
+            const data = canonicalPath ? readAsyncResultFile(canonicalPath) : notification;
+            if ((data.runId ?? data.id ?? runId) !== runId)
+                throw new Error(`Result identity does not match notification '${runId}'.`);
+            if (durableFile && resultPath !== path.join(getRunMetadataDir(runId), "result.json"))
+                throw new Error(`Canonical result identity does not match path '${resultPath}'.`);
+            if (data.sessionId ? data.sessionId !== state.currentSessionId && run?.ownerSessionId !== state.currentSessionId : !run) {
+                if (stamp)
+                    foreignResults.set(resultPath, { stamp, sessionId: state.currentSessionId, runId, ownerSessionId: run?.ownerSessionId, canonicalPath, canonicalStamp });
+                return;
+            }
+            foreignResults.delete(resultPath);
+            const consumeNotification = () => {
+                // A legacy hint can be the only saved result. Native queue admission
+                // is not publication; retain it until the verified parent receipt exists.
+                if (!durableFile && !canonicalPath && deps.isCompletionPublished && !deps.isCompletionPublished(runId, completionKey))
+                    return;
+                const hint = durableFile ? path.join(resultsDir, `${runId}.json`) : resultPath;
+                if (fs.existsSync(hint))
+                    fs.unlinkSync(hint);
+            };
+            data.completionId ??= `legacy:${runId}:${data.timestamp ?? "unknown"}`;
+            const completionKey = buildCompletionKey({ ...data, id: runId }, "result");
+            if (deps.reconcileDelivery ? deps.reconcileDelivery(runId, completionKey) : state.isRunResultConsumed?.(runId)) {
+                consumeNotification();
+                return;
+            }
+            if (state.waitingRuns?.has(runId)) {
+                pi.events.emit(SUBAGENT_ASYNC_COMPLETE_EVENT, { ...data, runId, suppressNotification: true, intercomResultDelivered: false });
+                return;
+            }
+            const hasExplicitNestedChildren = data.nestedChildren !== undefined;
+            let nestedChildren = compactNestedResultChildren(sanitizeNestedResultChildren(data.nestedChildren, resultPath, "nestedChildren"));
+            if (!nestedChildren?.length && !hasExplicitNestedChildren) {
+                try {
+                    nestedChildren = compactNestedResultChildren(projectNestedRegistryForRoot(runId)?.children);
+                }
+                catch (error) {
+                    console.error(`Failed to enrich subagent result file '${resultPath}' with nested registry children; will retry later:`, error);
+                    return;
+                }
+            }
+            const hasResultChildren = Array.isArray(data.results) && data.results.length > 0;
+            const resultChildren = hasResultChildren
+                ? data.results
+                : [{
+                        agent: data.agent,
+                        output: data.summary,
+                        success: data.success,
+                    }];
+            const normalizedChildren = attachNestedChildrenToResultChildren(runId, resultChildren.map((result = {}, index) => {
+                const baseOutput = result.output ?? result.finalOutput ?? data.summary;
+                const hasRealOutput = typeof baseOutput === "string" && baseOutput.trim().length > 0;
+                const output = hasRealOutput ? baseOutput : "(no output)";
+                const summary = result.success === false && result.error
+                    ? `${result.error}${hasRealOutput ? `\n\nOutput:\n${baseOutput}` : ""}`
+                    : output;
+                const sessionPath = result.sessionFile ?? (resultChildren.length === 1 ? data.sessionFile : undefined);
+                const childNestedChildren = sanitizeNestedResultChildren(result.children, resultPath, `results[${index}].children`);
+                return {
+                    agent: result.agent ?? data.agent ?? `step-${index + 1}`,
+                    status: resolveSubagentResultStatus({
+                        success: result.success,
+                        exitCode: result.exitCode ?? undefined,
+                        acceptance: result.acceptance,
+                        interrupted: result.interrupted,
+                        state: result.interrupted || typeof result.success !== "boolean" ? data.state : undefined,
+                    }),
+                    summary,
+                    index,
+                    artifactPath: result.artifactPaths?.outputPath,
+                    metadataPath: result.artifactPaths?.metadataPath,
+                    ...(typeof sessionPath === "string" && fs.existsSync(sessionPath) ? { sessionPath } : {}),
+                    ...(result.intercomTarget ? { intercomTarget: result.intercomTarget } : {}),
+                    ...(childNestedChildren ? { children: childNestedChildren } : {}),
+                };
+            }), nestedChildren);
+            if (processingCompletionKeys.has(completionKey))
+                return;
+            // Owned queue/receipt reconciliation above remains authoritative after TTL expiry.
+            if (markSeenWithTtl(state.completionSeen, completionKey, Date.now(), 10 * 60 * 1000)) {
+                consumeNotification();
+                return;
+            }
+            processingCompletionKeys.add(completionKey);
+            claimedCompletionKey = completionKey;
+            // Saved ownership is stable; the owner's live intercom identity can change on restart.
+            const intercomTarget = resolveOrchestratorIntercomTarget(pi.events, data.intercomTarget?.trim() ?? "");
+            let intercomResultDelivered = false;
+            if (intercomTarget) {
+                const mode = data.mode === "single" || data.mode === "parallel" || data.mode === "chain"
+                    ? data.mode
+                    : resultChildren.length > 1 ? "chain" : "single";
+                const savedResultPath = path.join(getRunMetadataDir(runId), "result.json");
+                const payload = buildSubagentResultIntercomPayload({
+                    to: intercomTarget,
+                    runId,
+                    completionId: data.completionId,
+                    mode,
+                    source: "async",
+                    ...(fs.existsSync(savedResultPath) ? { resultPath: savedResultPath } : {}),
+                    status: resolveSubagentResultStatus({ state: data.terminalState }),
+                    error: data.workflowGraph?.nodes.find((node) => node.error)?.error,
+                    children: normalizedChildren,
+                    asyncId: data.id,
+                    asyncDir: data.asyncDir,
+                });
+                intercomResultDelivered = await deliverSubagentResultIntercomEvent(pi.events, payload);
+            }
+            if (startedGeneration !== generation && startedGeneration !== joiningGeneration || ownerSessionId !== state.currentSessionId) {
+                state.completionSeen.delete(completionKey);
+                return;
+            }
+            // Shutdown closes native turn ingress. Retain failed deliveries for the
+            // next startup, while still recording acknowledgements already accepted.
+            if (!intercomResultDelivered && startedGeneration < preserveBeforeGeneration)
+                return;
+            const { terminalState: _terminalState, ...eventData } = data;
+            pi.events.emit(SUBAGENT_ASYNC_COMPLETE_EVENT, {
+                ...eventData,
+                agent: data.agent ?? normalizedChildren.map((child) => child.agent).join(", "),
+                summary: data.summary?.trim() ? data.summary : normalizedChildren.map((child) => child.summary).join("\n\n"),
+                runId,
+                completionKey,
+                intercomResultDelivered,
+                ...(nestedChildren?.length ? { nestedChildren } : {}),
+                ...(Array.isArray(data.results) ? {
+                    results: hasResultChildren
+                        ? normalizedChildren.map((child, index) => ({
+                            ...data.results[index],
+                            agent: child.agent,
+                            status: child.status,
+                            summary: child.summary,
+                            index: child.index,
+                            artifactPath: child.artifactPath,
+                            sessionPath: child.sessionPath,
+                            children: child.children,
+                        }))
+                        : [],
+                } : {}),
+            });
+            completionEmitted = true;
+            consumeNotification();
+        }
+        catch (error) {
+            if (isNotFoundError(error))
+                return;
+            console.error(`Failed to process subagent result file '${resultPath}':`, error);
+        }
+        finally {
+            if (claimedCompletionKey) {
+                processingCompletionKeys.delete(claimedCompletionKey);
+                if (!completionEmitted)
+                    state.completionSeen.delete(claimedCompletionKey);
+            }
+        }
+    };
+    const scheduled = new Set();
+    let drainGeneration;
+    const scheduleResult = (file) => {
+        scheduled.add(file);
+        if (drainGeneration === generation)
+            return;
+        drainGeneration = generation;
+        const scheduledGeneration = generation;
+        startTail = startTail.then(async () => {
+            try {
+                while (scheduled.size) {
+                    await yieldToInput();
+                    if (scheduledGeneration !== generation)
+                        return;
+                    const files = [...scheduled].slice(0, 4);
+                    for (const file of files)
+                        scheduled.delete(file);
+                    // Only synchronous starts share publication proof. Async delivery
+                    // resumes outside this scope; yield between batches for native input.
+                    const start = () => {
+                        for (const file of files) {
+                            if (scheduledGeneration !== generation)
+                                return;
+                            const tail = handleResult(file);
+                            inFlight.add(tail);
+                            void tail.finally(() => inFlight.delete(tail));
+                        }
+                    };
+                    if (deps.withReceiptBatch)
+                        deps.withReceiptBatch(start);
+                    else
+                        start();
+                }
+            }
+            finally {
+                if (drainGeneration === scheduledGeneration)
+                    drainGeneration = undefined;
+            }
+        });
+    };
+    state.resultFileCoalescer = createFileCoalescer(scheduleResult, 50);
+    const primeExistingResults = () => {
+        try {
+            // Admit one recovery scan synchronously; separate zero-delay file timers
+            // can split the same scan into additional verification batches.
+            pendingResultFiles().forEach(scheduleResult);
+        }
+        catch (error) {
+            if (isNotFoundError(error))
+                return;
+            console.error(`Failed to scan subagent result directory '${resultsDir}':`, error);
+        }
+    };
+    const ensurePeriodicScan = () => {
+        if (periodicScanTimer)
+            return;
+        periodicScanTimer = setInterval(primeExistingResults, POLL_INTERVAL_MS);
+        periodicScanTimer.unref?.();
+    };
+    const clearPeriodicScan = () => {
+        if (!periodicScanTimer)
+            return;
+        clearInterval(periodicScanTimer);
+        periodicScanTimer = null;
+    };
+    const startPollingFallback = (reason) => {
+        state.watcher?.close();
+        state.watcher = null;
+        clearPeriodicScan();
+        if (state.watcherRestartTimer)
+            return;
+        console.error(`Subagent result watcher for '${resultsDir}' fell back to polling because native fs.watch is unavailable (${getErrorCode(reason) ?? "unknown error"}).`);
+        primeExistingResults();
+        state.watcherRestartTimer = setInterval(primeExistingResults, POLL_INTERVAL_MS);
+        state.watcherRestartTimer.unref?.();
+    };
+    const scheduleRestart = () => {
+        if (state.watcherRestartTimer)
+            return;
+        state.watcherRestartTimer = setTimeout(() => {
+            state.watcherRestartTimer = null;
+            try {
+                fs.mkdirSync(resultsDir, { recursive: true });
+                startResultWatcher();
+            }
+            catch (error) {
+                if (shouldFallBackToPolling(error)) {
+                    startPollingFallback(error);
+                    return;
+                }
+                console.error(`Failed to restart subagent result watcher for '${resultsDir}':`, error);
+                scheduleRestart();
+            }
+        }, WATCHER_RESTART_DELAY_MS);
+        state.watcherRestartTimer.unref?.();
+    };
+    const startResultWatcher = () => {
+        if (state.watcher) {
+            ensurePeriodicScan();
+            return;
+        }
+        if (state.watcherRestartTimer) {
+            clearTimeout(state.watcherRestartTimer);
+            clearInterval(state.watcherRestartTimer);
+            state.watcherRestartTimer = null;
+        }
+        try {
+            state.watcher = fs.watch(resultsDir, (ev, file) => {
+                if (ev !== "rename" || !file)
+                    return;
+                const fileName = file.toString();
+                if (!fileName.endsWith(".json"))
+                    return;
+                if (!fs.existsSync(path.join(resultsDir, fileName)))
+                    return;
+                state.resultFileCoalescer.schedule(fileName);
+            });
+            state.watcher.on("error", (error) => {
+                if (shouldFallBackToPolling(error)) {
+                    startPollingFallback(error);
+                    return;
+                }
+                console.error(`Subagent result watcher failed for '${resultsDir}':`, error);
+                state.watcher?.close();
+                state.watcher = null;
+                scheduleRestart();
+            });
+            state.watcher.unref?.();
+            ensurePeriodicScan();
+        }
+        catch (error) {
+            if (shouldFallBackToPolling(error)) {
+                startPollingFallback(error);
+                return;
+            }
+            console.error(`Failed to start subagent result watcher for '${resultsDir}':`, error);
+            state.watcher = null;
+            scheduleRestart();
+        }
+    };
+    const stopResultWatcher = (options = {}) => {
+        joiningGeneration = options.joinInFlight ? generation : undefined;
+        generation++;
+        if (options.preservePending)
+            preserveBeforeGeneration = generation;
+        state.watcher?.close();
+        state.watcher = null;
+        clearPeriodicScan();
+        if (state.watcherRestartTimer) {
+            clearTimeout(state.watcherRestartTimer);
+            clearInterval(state.watcherRestartTimer);
+        }
+        state.watcherRestartTimer = null;
+        state.resultFileCoalescer.clear();
+        scheduled.clear();
+        foreignResults.clear();
+    };
+    const joinInFlight = async () => {
+        await Promise.all([...inFlight]);
+        joiningGeneration = undefined;
+    };
+    return { startResultWatcher, primeExistingResults, stopResultWatcher, joinInFlight };
+}

package/dist/runs/background/run-id-resolver.js

@@ -0,0 +1,64 @@
+import { ASYNC_DIR, RESULTS_DIR } from "../../shared/types.js";
+import { formatRunIdAmbiguity } from "../shared/run-id-ambiguity.js";
+import { exactAsyncRunLocation, findAsyncRunPrefixMatches } from "./async-resume.js";
+import { assertSafeNestedId, findNestedRunMatchesById, findNestedRouteForRootId } from "../shared/nested-events.js";
+function nestedScopeFromState(state) {
+    if (!state)
+        return undefined;
+    const routes = [];
+    const seen = new Set();
+    const add = (route) => {
+        if (!route)
+            return;
+        const key = `${route.rootRunId}:${route.eventSink}:${route.controlInbox}`;
+        if (seen.has(key))
+            return;
+        seen.add(key);
+        routes.push(route);
+    };
+    for (const run of state.ownedRuns?.values() ?? [])
+        add(findNestedRouteForRootId(run.runId));
+    for (const job of state.asyncJobs.values())
+        add(job.nestedRoute);
+    return { routes };
+}
+function asyncPrefixMatches(prefix, asyncDirRoot, resultsDir) {
+    return findAsyncRunPrefixMatches(prefix, asyncDirRoot, resultsDir);
+}
+export function resolveSubagentRunId(id, deps = {}) {
+    assertSafeNestedId("id", id);
+    const asyncDirRoot = deps.asyncDirRoot ?? ASYNC_DIR;
+    const resultsDir = deps.resultsDir ?? RESULTS_DIR;
+    const nestedScope = deps.nested ?? nestedScopeFromState(deps.state);
+    // In a session scope, storage discovery must not hide an authorized descendant route.
+    const preferDirectAsync = (runId) => !nestedScope || deps.state?.ownedRuns?.has(runId) || deps.state?.asyncJobs.has(runId);
+    const exactAsync = exactAsyncRunLocation(id, asyncDirRoot, resultsDir);
+    if ((exactAsync.asyncDir || exactAsync.resultPath) && preferDirectAsync(id))
+        return { kind: "async", id, location: exactAsync };
+    const exactNested = findNestedRunMatchesById(id, nestedScope ? { scope: nestedScope } : {});
+    if (exactNested.length > 1)
+        throw new Error(`Nested run id '${id}' is ambiguous across authorized registries. Provide the full id after stale registries are cleaned up.`);
+    if (exactNested[0])
+        return { kind: "nested", id, match: exactNested[0] };
+    if (exactAsync.asyncDir || exactAsync.resultPath)
+        return { kind: "async", id, location: exactAsync };
+    const asyncMatches = asyncPrefixMatches(id, asyncDirRoot, resultsDir);
+    const nestedMatches = findNestedRunMatchesById(id, nestedScope ? { prefix: true, scope: nestedScope } : { prefix: true });
+    const matches = [];
+    for (const match of asyncMatches) {
+        if (!preferDirectAsync(match.id) && nestedMatches.some((nested) => nested.run.id === match.id))
+            continue;
+        matches.push({ kind: "async", id: match.id, location: match.location });
+    }
+    for (const match of nestedMatches) {
+        if (preferDirectAsync(match.run.id) && asyncMatches.some((candidate) => candidate.id === match.run.id))
+            continue;
+        matches.push({ kind: "nested", id: match.run.id, match });
+    }
+    const unique = new Map(matches.map((match) => [match.kind === "nested" ? `nested:${match.match.rootRunId}:${match.id}` : `async:${match.id}`, match]));
+    const values = [...unique.values()];
+    if (values.length > 1) {
+        throw new Error(formatRunIdAmbiguity("subagent", id, values.map((match) => `${match.kind}:${match.id}`)));
+    }
+    return values[0];
+}