@mono-agent/agent-app 0.11.5 → 0.13.0

package/dist/background.js

@@ -7,12 +7,15 @@ import process from "node:process";
 import { listRecordedRuns, listTraceSources } from "@mono-agent/observability";
 import { describeSensitiveDataExportWarning, phoenixAppBaseUrl, resolveAppTraceRegistryDir, resolveAppTraceStaleAfterMs, } from "./app-config.js";
 import { formatChannelFactValue } from "./channel-fact-format.js";
+import { formatHumanChannelSections } from "./channel-status-display.js";
+import { hasCompletedManagedStartup } from "./managed-startup.js";
 import { bootout, bootstrap, buildLaunchdMaintenancePlistXml, buildPlistXml, defaultPathEnv, deriveLaunchdLabel, deriveLaunchdMaintenanceLabel, isLoaded, launchdServiceInfo, launchdPathsFor, MANAGED_LAUNCHD_LOG_MAINTENANCE_ENV, makeLaunchctlRunner, } from "./launchd.js";
 import { selectBackgroundOperationalEnvironment } from "./background-environment.js";
 import { beginLaunchdLogMaintenanceIntent, clearLaunchdLogMaintenanceIntent, inspectLaunchdLogs, LAUNCHD_LOG_MAINTENANCE_INTERVAL_SECONDS, markLaunchdLogMaintenanceRestoring, markLaunchdLogMaintenanceStopped, markLaunchdLogMaintenanceStopping, readLaunchdLogMaintenanceIntent, rotateStoppedLaunchdLogs, } from "./launchd-logs.js";
 import { ensureManagedBackgroundRuntime, MANAGED_BACKGROUND_WORKER_ENV, } from "./background-runtime.js";
 import { backgroundSnapshotFromMetadata, captureBackgroundSnapshot, encodeBackgroundSnapshot, sameBackgroundSnapshot, } from "./background-snapshot.js";
 import { resolveConfiguredManagedRuntimePackages } from "./managed-runtime-packages.js";
+import { acquireManagedRuntimePublicationBarrier } from "./managed-runtime-publication.js";
 import { processIncarnationFromJson, } from "./process-incarnation.js";
 import { acquireOwnerPrivateLock, validateOwnerPrivateLockInputs } from "./owner-private-lock.js";
 import { buildRunsHealthDisplay, RUNS_HEALTH_MAX_RUNS } from "./runs-health.js";
@@ -146,6 +149,10 @@ export function defaultBackgroundDeps() {
             env: target.configurationEnvironment,
         }),
         acquireLifecycleLock: acquireFilesystemLifecycleLock,
+        acquireRuntimePublicationBarrier: (target) => acquireManagedRuntimePublicationBarrier({
+            label: target.label,
+            managedRoot: dirname(target.paths.logDir),
+        }),
         probeTui: probeTuiEndpoint,
         stdout: (text) => void process.stdout.write(text),
         stderr: (text) => void process.stderr.write(text),
@@ -156,12 +163,13 @@ export function defaultBackgroundDeps() {
         }),
     };
 }
-const DEFAULT_POLL = { timeoutMs: 18_000, intervalMs: 400 };
-export async function startBackground(target, deps, poll = DEFAULT_POLL) {
+const DEFAULT_CONTROL_POLL = { timeoutMs: 18_000, intervalMs: 400 };
+const DEFAULT_READINESS_POLL = { timeoutMs: 60_000, intervalMs: 400 };
+export async function startBackground(target, deps, poll) {
     return (await ensureBackgroundReady(target, deps, poll)).ok ? 0 : 1;
 }
 /** Restart is behaviourally identical: ensure a single fresh running instance. */
-export async function restartBackground(target, deps, poll = DEFAULT_POLL) {
+export async function restartBackground(target, deps, poll) {
     return (await ensureBackgroundReady(target, deps, poll)).ok ? 0 : 1;
 }
 /**
@@ -169,7 +177,7 @@ export async function restartBackground(target, deps, poll = DEFAULT_POLL) {
  * retaining one lifecycle lock. This closes the force-restart gap in which a
  * concurrent start could previously enter while the session store was purged.
  */
-export async function forceRestartBackground(target, deps, whileStopped, poll = DEFAULT_POLL) {
+export async function forceRestartBackground(target, deps, whileStopped, poll) {
     const release = await deps.acquireLifecycleLock(target);
     if (release === undefined) {
         deps.stderr(ui.errorLine(`Another lifecycle command is already active for ${target.label}.`));
@@ -177,11 +185,12 @@ export async function forceRestartBackground(target, deps, whileStopped, poll =
         return 1;
     }
     try {
-        const stopCode = await stopBackgroundUnlocked(target, deps, poll);
+        const controlPoll = poll ?? DEFAULT_CONTROL_POLL;
+        const stopCode = await stopBackgroundUnlocked(target, deps, controlPoll);
         if (stopCode !== 0)
             return stopCode;
         await whileStopped();
-        return (await ensureBackgroundReadyUnlocked(target, deps, poll)).ok ? 0 : 1;
+        return (await ensureBackgroundReadyUnlocked(target, deps, controlPoll, poll ?? DEFAULT_READINESS_POLL)).ok ? 0 : 1;
     }
     finally {
         await release().catch((error) => {
@@ -195,7 +204,7 @@ export async function forceRestartBackground(target, deps, whileStopped, poll =
  * intentionally stopped service; the sole exception is recovery authorized by
  * its own durable lifecycle intent after a prior maintainer died post-bootout.
  */
-export async function maintainLaunchdLogs(target, deps, poll = DEFAULT_POLL) {
+export async function maintainLaunchdLogs(target, deps, poll = DEFAULT_CONTROL_POLL) {
     const release = await deps.acquireLifecycleLock(target);
     if (release === undefined) {
         // Controller start/restart/stop owns the same lock. Skipping this periodic
@@ -398,11 +407,11 @@ function maintenanceErrorMessage(error) {
 }
 /**
  * Ensure the canonical per-config LaunchAgent is running and return the fresh
- * trace source that proved it reached `startup-complete`. This is the shared
+ * trace source that proved it completed the full startup lifecycle. This is the shared
  * lifecycle boundary for CLI start/restart and remote configuration handoffs;
  * callers must not open a console when the result is not ok.
  */
-export async function ensureBackgroundReady(target, deps, poll = DEFAULT_POLL) {
+export async function ensureBackgroundReady(target, deps, poll) {
     const release = await deps.acquireLifecycleLock(target);
     if (release === undefined) {
         deps.stderr(ui.errorLine(`Another lifecycle command is already active for ${target.label}.`));
@@ -410,7 +419,7 @@ export async function ensureBackgroundReady(target, deps, poll = DEFAULT_POLL) {
         return { ok: false, action: "start", reason: "ownership" };
     }
     try {
-        return await ensureBackgroundReadyUnlocked(target, deps, poll);
+        return await ensureBackgroundReadyUnlocked(target, deps, poll ?? DEFAULT_CONTROL_POLL, poll ?? DEFAULT_READINESS_POLL);
     }
     finally {
         await release().catch((error) => {
@@ -418,18 +427,58 @@ export async function ensureBackgroundReady(target, deps, poll = DEFAULT_POLL) {
         });
     }
 }
-async function ensureBackgroundReadyUnlocked(target, deps, poll) {
+async function ensureBackgroundReadyUnlocked(target, deps, controlPoll, readinessPoll) {
     if (target.expectedSnapshot === undefined) {
         deps.stderr(ui.errorLine("Refusing to launch a managed worker without an approved background snapshot."));
         deps.stderr(ui.hint("No LaunchAgent changes were made. Retry from the wizard or `mono-agent start`."));
         return { ok: false, action: "start", reason: "snapshot" };
     }
-    const uid = deps.getuid();
     if (!(await snapshotStillMatches(target, deps))) {
         reportSnapshotDrift(target, deps);
         return { ok: false, action: "start", reason: "snapshot" };
     }
+    let publicationBarrier;
+    try {
+        publicationBarrier = await (deps.acquireRuntimePublicationBarrier
+            ?? ((input) => acquireManagedRuntimePublicationBarrier({
+                label: input.label,
+                managedRoot: dirname(input.paths.logDir),
+            })))(target);
+    }
+    catch (error) {
+        reportLifecycleException(target, deps, "establish the managed runtime publication barrier", error);
+        return { ok: false, action: "start", reason: "ownership" };
+    }
+    if (publicationBarrier === undefined) {
+        deps.stderr(ui.errorLine(`Another runtime publication is already active for ${target.label}.`));
+        return { ok: false, action: "start", reason: "ownership" };
+    }
+    return await ensureBackgroundReadyWithPublicationBarrier(target, deps, controlPoll, readinessPoll, publicationBarrier);
+}
+async function ensureBackgroundReadyWithPublicationBarrier(target, deps, controlPoll, readinessPoll, publicationBarrier) {
+    let barrierReleased = false;
+    const releaseBarrier = async () => {
+        if (barrierReleased)
+            return;
+        await publicationBarrier.release();
+        barrierReleased = true;
+    };
+    try {
+        return await ensureBackgroundReadyAfterPublicationBarrier(target, deps, controlPoll, readinessPoll, releaseBarrier);
+    }
+    finally {
+        if (!barrierReleased) {
+            await releaseBarrier().catch((error) => {
+                deps.stderr(ui.errorLine(`Could not release runtime publication barrier for ${target.label}: ${error instanceof Error ? error.message : String(error)}`));
+            });
+        }
+    }
+}
+async function ensureBackgroundReadyAfterPublicationBarrier(target, deps, controlPoll, readinessPoll, releaseBarrier) {
+    const uid = deps.getuid();
     let launchTarget;
+    const runtimeStartedAt = deps.now();
+    deps.stdout(ui.hint("Verifying the durable managed runtime…"));
     try {
         const additionalPackages = await deps.resolveManagedRuntimePackages?.(target) ?? [];
         const runtime = await deps.ensureManagedRuntime({
@@ -441,7 +490,10 @@ async function ensureBackgroundReadyUnlocked(target, deps, poll) {
             ...target,
             cliPath: runtime.cliPath,
             nodePath: runtime.nodePath,
+            managedRuntimeLaunchProof: runtime.launchProof,
         };
+        deps.stdout(ui.hint(`Managed runtime ready (${runtimeVerificationLabel(runtime.verificationMode)}, `
+            + `${formatLifecycleDuration(deps.now() - runtimeStartedAt)}).`));
     }
     catch (error) {
         reportLifecycleException(target, deps, "install and verify the durable managed runtime", error);
@@ -475,6 +527,7 @@ async function ensureBackgroundReadyUnlocked(target, deps, poll) {
     }
     let outcome;
     let prepared = false;
+    deps.stdout(ui.hint("Replacing the managed worker…"));
     try {
         let interruptedMaintenance = await deps.readLaunchdLogMaintenanceIntent(launchTarget.paths);
         if (interruptedMaintenance?.phase === "stopping" || interruptedMaintenance?.phase === "restoring") {
@@ -483,7 +536,7 @@ async function ensureBackgroundReadyUnlocked(target, deps, poll) {
                 throw new Error("Interrupted launchd-log maintenance lost its writer PID before durable stopped-writer proof; refusing restart.");
             }
         }
-        outcome = await bootstrapOrRestart(launchTarget, deps, uid, poll, async () => {
+        outcome = await bootstrapOrRestart(launchTarget, deps, uid, controlPoll, async () => {
             const stoppedIntent = interruptedMaintenance;
             if (stoppedIntent?.phase === "stopped") {
                 interruptedMaintenance = await deps.markLaunchdLogMaintenanceStopping(launchTarget.paths, stoppedIntent);
@@ -504,6 +557,9 @@ async function ensureBackgroundReadyUnlocked(target, deps, poll) {
             await writePlists(launchTarget, deps);
             sinceMs = deps.now();
             prepared = true;
+            // The replacement plist now contains the finalized-runtime proof. Let
+            // launchd respawn only after the stopped-window commit is complete.
+            await releaseBarrier();
         });
     }
     catch (error) {
@@ -519,18 +575,20 @@ async function ensureBackgroundReadyUnlocked(target, deps, poll) {
     let ready;
     try {
         ready = await pollInstanceReady(launchTarget, deps, {
-            ...poll,
+            ...readinessPoll,
             sinceMs,
             ...(launchTarget.requireTui === true ? { requireTui: true } : {}),
         });
     }
     catch (error) {
-        reportLifecycleException(launchTarget, deps, "read the worker readiness trace", error);
+        reportReadinessException(deps, error);
+        const stopped = await cleanUpUnreadyBackground(launchTarget, deps, controlPoll);
+        reportReadinessCleanup(launchTarget, deps, stopped);
         return { ok: false, action, reason: "readiness" };
     }
     if (ready === undefined) {
         if (!(await snapshotStillMatches(launchTarget, deps))) {
-            const stopped = await stopBackgroundUnlocked(launchTarget, deps, poll);
+            const stopped = await stopBackgroundUnlocked(launchTarget, deps, controlPoll);
             reportSnapshotDrift(launchTarget, deps);
             if (stopped === 0) {
                 deps.stderr(ui.style.dim("The drifted LaunchAgent was stopped before returning control.") + "\n");
@@ -540,13 +598,27 @@ async function ensureBackgroundReadyUnlocked(target, deps, poll) {
             }
             return { ok: false, action, reason: "snapshot" };
         }
-        reportTimeout(launchTarget, deps);
+        reportTimeout(deps);
+        const stopped = await cleanUpUnreadyBackground(launchTarget, deps, controlPoll);
+        reportReadinessCleanup(launchTarget, deps, stopped);
         return { ok: false, action, reason: "timeout" };
     }
     const completedAction = outcome.restarted ? "restarted" : "started";
     printInstanceInfo(ready, launchTarget, deps, completedAction);
     return { ok: true, action: completedAction, source: ready };
 }
+function runtimeVerificationLabel(mode) {
+    switch (mode) {
+        case "fast-reuse": return "warm reuse";
+        case "full-reuse": return "full verification";
+        case "installed": return "installed";
+        case "repaired": return "repaired";
+    }
+}
+function formatLifecycleDuration(milliseconds) {
+    const safe = Math.max(0, milliseconds);
+    return safe < 1_000 ? `${Math.round(safe)} ms` : `${(safe / 1_000).toFixed(1)} s`;
+}
 async function prepareLaunchdDirectories(target, deps) {
     // launchd will not create the log file's parent directory, so make both dirs
     // before writing the plist that references them.
@@ -558,6 +630,9 @@ async function writePlists(target, deps) {
     if (target.expectedSnapshot === undefined) {
         throw new Error("A managed LaunchAgent requires an approved background snapshot.");
     }
+    if (target.managedRuntimeLaunchProof === undefined) {
+        throw new Error("A managed LaunchAgent requires a finalized runtime launch proof.");
+    }
     const maintenance = maintenancePathsForTarget(target);
     const mainXml = buildPlistXml({
         label: target.label,
@@ -567,6 +642,7 @@ async function writePlists(target, deps) {
         cwd: target.cwd,
         ...(target.envFile === undefined ? {} : { envFile: target.envFile }),
         expectedBackgroundSnapshot: encodeBackgroundSnapshot(target.expectedSnapshot),
+        expectedManagedRuntimeLaunch: target.managedRuntimeLaunchProof,
         stdoutPath: target.paths.stdoutPath,
         stderrPath: target.paths.stderrPath,
         environment: target.environment,
@@ -586,7 +662,7 @@ async function writePlists(target, deps) {
     await deps.writeFile(target.paths.plistPath, mainXml);
     await deps.writeFile(maintenance.plistPath, maintenanceXml);
 }
-async function ensureOwnerPrivateLaunchdDirectory(path) {
+export async function ensureOwnerPrivateLaunchdDirectory(path) {
     const parent = dirname(path);
     const parentDetails = await lstat(parent);
     assertOwnerDirectory(parentDetails, parent, "LaunchAgent parent");
@@ -624,7 +700,7 @@ async function ensureOwnerPrivateLaunchdDirectory(path) {
         await handle.close();
     }
 }
-async function writeOwnerPrivateLaunchdFile(path, data) {
+export async function writeOwnerPrivateLaunchdFile(path, data) {
     let existing;
     try {
         existing = await lstat(path);
@@ -913,7 +989,7 @@ function lifecycleFailure(result, detail) {
 function uniquePids(values) {
     return [...new Set(values.filter((value) => value !== undefined && value > 0))];
 }
-export async function stopBackground(target, deps, poll = DEFAULT_POLL) {
+export async function stopBackground(target, deps, poll = DEFAULT_CONTROL_POLL) {
     const release = await deps.acquireLifecycleLock(target);
     if (release === undefined) {
         deps.stderr(ui.errorLine(`Another lifecycle command is already active for ${target.label}.`));
@@ -1015,13 +1091,27 @@ async function stopBackgroundUnlocked(target, deps, poll) {
     deps.stdout(ui.keyValue([["config", target.configPath]]));
     return 0;
 }
-export async function statusBackground(target, deps) {
+export async function statusBackground(target, deps, options = {}) {
     const result = await deps.listTraceSources({ registryDir: target.registryDir, staleAfterMs: target.staleAfterMs });
     const classified = await Promise.all(result.sources.map(async (source) => ({
         source,
         matches: await matchesConfig(source, target.configPath),
     })));
     const current = classified.find((entry) => entry.matches)?.source;
+    // Only surface other instances that are live or crashed — cleanly stopped
+    // manifests linger in the registry and would just be noise.
+    const others = classified
+        .filter((entry) => !entry.matches && entry.source.health !== "stopped")
+        .map((entry) => entry.source);
+    if (options.json === true) {
+        const instance = current === undefined ? null : await assembleInstanceStatus(current, target, deps);
+        deps.stdout(`${JSON.stringify({
+            ok: current?.health === "running",
+            instance,
+            others: others.map(assembleOtherInstanceStatus),
+        })}\n`);
+        return current?.health === "running" ? 0 : 1;
+    }
     if (current === undefined) {
         deps.stdout(ui.style.dim(`No running mono-agent instance for ${target.configPath}.`) + "\n");
         deps.stdout(ui.hint(`Start it with: mono-agent start${commandFlags(target)}`));
@@ -1030,11 +1120,6 @@ export async function statusBackground(target, deps) {
         writeInstanceDetail(current, target, deps);
         await writeRunsHealthDetail(current, deps);
     }
-    // Only surface other instances that are live or crashed — cleanly stopped
-    // manifests linger in the registry and would just be noise.
-    const others = classified
-        .filter((entry) => !entry.matches && entry.source.health !== "stopped")
-        .map((entry) => entry.source);
     if (others.length > 0) {
         deps.stdout("\n" + ui.rule("Other mono-agent instances"));
         for (const source of others) {
@@ -1043,6 +1128,62 @@ export async function statusBackground(target, deps) {
     }
     return current?.health === "running" ? 0 : 1;
 }
+/**
+ * The machine-readable instance record for `status --json`. It mirrors the
+ * fields the human `writeInstanceDetail`/`writeRunsHealthDetail` render, reading
+ * only the already-computed trace-source manifest (content-free, safe to publish)
+ * and the same runs-health probe the human path performs — no extra probes.
+ */
+async function assembleInstanceStatus(source, target, deps) {
+    const metadata = source.metadata ?? {};
+    const observability = metadata.observability;
+    const sandbox = metadata.sandbox;
+    const session = metadata.session;
+    const channels = metadata.channels;
+    return {
+        sourceId: source.sourceId,
+        label: source.label,
+        launchdLabel: target.label,
+        pid: source.pid ?? null,
+        health: source.health,
+        status: source.status,
+        configPath: target.configPath,
+        startedAt: source.startedAt,
+        updatedAt: source.updatedAt,
+        ...(source.artifactDir === undefined ? {} : { artifactDir: source.artifactDir }),
+        ...(source.transports === undefined ? {} : { transports: source.transports }),
+        logs: { stdout: target.paths.stdoutPath, stderr: target.paths.stderrPath },
+        ...(isPlainRecord(observability) ? { observability } : {}),
+        ...(isPlainRecord(sandbox) ? { sandbox } : {}),
+        ...(isPlainRecord(session) ? { session } : {}),
+        ...(isPlainRecord(channels) ? { channels } : {}),
+        ...(source.memoryHealth === undefined ? {} : { memoryHealth: source.memoryHealth }),
+        runsHealth: await assembleRunsHealthStatus(source, deps),
+    };
+}
+async function assembleRunsHealthStatus(source, deps) {
+    if (source.artifactDir === undefined || source.artifactDir.trim().length === 0) {
+        return null;
+    }
+    const { totalRuns, warnings } = await deps.listRecordedRuns({
+        artifactDir: source.artifactDir,
+        maxRuns: RUNS_HEALTH_MAX_RUNS,
+        scope: "agent",
+    });
+    return { totalRuns, warnings };
+}
+function assembleOtherInstanceStatus(source) {
+    return {
+        sourceId: source.sourceId,
+        label: source.label,
+        health: source.health,
+        pid: source.pid ?? null,
+        ...(source.configPath === undefined ? {} : { configPath: source.configPath }),
+    };
+}
+function isPlainRecord(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
 export async function tailLogs(target, deps, options) {
     const args = [
         "-n",
@@ -1146,9 +1287,27 @@ function reportLifecycleException(target, deps, operation, error) {
     deps.stderr(ui.style.dim("The committed agent files were preserved.") + "\n");
     reportLaunchRecovery(target, deps);
 }
-function reportTimeout(target, deps) {
+function reportReadinessException(deps, error) {
+    deps.stderr(ui.errorLine(`Failed to read the worker readiness trace: ${error instanceof Error ? error.message : String(error)}`));
+    deps.stderr(ui.style.dim("The committed agent files were preserved.") + "\n");
+}
+function reportTimeout(deps) {
     deps.stderr(ui.errorLine("mono-agent did not report ready within the timeout."));
-    deps.stderr(ui.style.dim("The committed agent files were preserved. It may still be starting, or it may have failed.") + "\n");
+    deps.stderr(ui.style.dim("The committed agent files were preserved.") + "\n");
+}
+async function cleanUpUnreadyBackground(target, deps, poll) {
+    try {
+        return await stopBackgroundUnlocked(target, { ...deps, stdout: () => undefined }, poll) === 0;
+    }
+    catch (error) {
+        deps.stderr(ui.errorLine(`Failed while trying to stop the unready LaunchAgent: ${error instanceof Error ? error.message : String(error)}`));
+        return false;
+    }
+}
+function reportReadinessCleanup(target, deps, stopped) {
+    deps.stderr(stopped
+        ? ui.style.dim("The unready LaunchAgent and scheduled maintenance were stopped and their definitions removed before returning control.") + "\n"
+        : ui.style.yellow("The unready LaunchAgent could not be proven stopped; a worker or maintenance helper may still be running.") + "\n");
     reportLaunchRecovery(target, deps);
 }
 function reportLaunchRecovery(target, deps) {
@@ -1190,10 +1349,10 @@ function writeInstanceDetail(source, target, deps) {
             deps.stdout(`  ${line}\n`);
         }
     }
-    const channelLines = formatChannels(source);
-    if (channelLines.length > 0) {
-        deps.stdout(ui.rule("channels"));
-        for (const line of channelLines) {
+    const channelSections = formatChannels(source);
+    for (const section of channelSections) {
+        deps.stdout(ui.rule(section.title));
+        for (const line of section.lines) {
             deps.stdout(`${line}\n`);
         }
     }
@@ -1247,7 +1406,7 @@ async function matchesConfig(source, configPath) {
     return await canonicalBackgroundConfigPath(process.cwd(), source.configPath) === configPath;
 }
 function isReady(source, requireTui) {
-    if (source.health !== "running" || metadataReason(source) !== "startup-complete")
+    if (source.health !== "running" || !hasCompletedManagedStartup(source))
         return false;
     if (source.memoryHealth?.status === "unhealthy")
         return false;
@@ -1268,10 +1427,6 @@ function channelRecords(source) {
     return Object.values(channels)
         .filter((value) => value !== null && typeof value === "object");
 }
-function metadataReason(source) {
-    const reason = source.metadata?.reason;
-    return typeof reason === "string" ? reason : undefined;
-}
 function startedAtMs(source) {
     const parsed = Date.parse(source.startedAt);
     return Number.isFinite(parsed) ? parsed : 0;
@@ -1539,10 +1694,10 @@ function formatChannels(source) {
     if (channels === null || typeof channels !== "object") {
         return [];
     }
-    return Object.entries(channels).map(([id, value]) => {
+    return formatHumanChannelSections(Object.entries(channels).map(([id, value]) => {
         const { kind, text } = describeChannel(value);
-        return `  ${ui.channelBadge(kind)}${ui.style.bold(id.padEnd(11))} ${text}`;
-    });
+        return { id, kind, text };
+    }));
 }
 export function describeChannel(value) {
     if (value === null || typeof value !== "object") {