@mono-agent/agent-app 0.13.0 → 0.14.0

package/dist/background.js

@@ -9,10 +9,10 @@ import { describeSensitiveDataExportWarning, phoenixAppBaseUrl, resolveAppTraceR
 import { formatChannelFactValue } from "./channel-fact-format.js";
 import { formatHumanChannelSections } from "./channel-status-display.js";
 import { hasCompletedManagedStartup } from "./managed-startup.js";
-import { bootout, bootstrap, buildLaunchdMaintenancePlistXml, buildPlistXml, defaultPathEnv, deriveLaunchdLabel, deriveLaunchdMaintenanceLabel, isLoaded, launchdServiceInfo, launchdPathsFor, MANAGED_LAUNCHD_LOG_MAINTENANCE_ENV, makeLaunchctlRunner, } from "./launchd.js";
+import { bootout, bootstrap, buildLaunchdMaintenancePlistXml, buildPlistXml, defaultPathEnv, deriveLaunchdLabel, deriveLaunchdMaintenanceLabel, isLoaded, launchdManagedWorkerInfo, launchdServiceInfo, launchdPathsFor, MANAGED_LAUNCHD_LOG_MAINTENANCE_ENV, makeLaunchctlRunner, } from "./launchd.js";
 import { selectBackgroundOperationalEnvironment } from "./background-environment.js";
 import { beginLaunchdLogMaintenanceIntent, clearLaunchdLogMaintenanceIntent, inspectLaunchdLogs, LAUNCHD_LOG_MAINTENANCE_INTERVAL_SECONDS, markLaunchdLogMaintenanceRestoring, markLaunchdLogMaintenanceStopped, markLaunchdLogMaintenanceStopping, readLaunchdLogMaintenanceIntent, rotateStoppedLaunchdLogs, } from "./launchd-logs.js";
-import { ensureManagedBackgroundRuntime, MANAGED_BACKGROUND_WORKER_ENV, } from "./background-runtime.js";
+import { ensureManagedBackgroundRuntime, inspectManagedRuntimeSourceIdentity, MANAGED_BACKGROUND_WORKER_ENV, verifyManagedRuntimeLaunch, } from "./background-runtime.js";
 import { backgroundSnapshotFromMetadata, captureBackgroundSnapshot, encodeBackgroundSnapshot, sameBackgroundSnapshot, } from "./background-snapshot.js";
 import { resolveConfiguredManagedRuntimePackages } from "./managed-runtime-packages.js";
 import { acquireManagedRuntimePublicationBarrier } from "./managed-runtime-publication.js";
@@ -68,6 +68,7 @@ export async function resolveInstanceTarget(input) {
         paths: launchdPathsFor(label),
         nodePath: process.execPath,
         cliPath: input.cliPath,
+        controllerCliPath: input.cliPath,
         // Bake an explicit --env-file (resolved absolute) into the plist so the
         // launchd worker loads the same env file the launcher did.
         ...(input.args.envFile === undefined ? {} : { envFile: resolve(cwd, input.args.envFile) }),
@@ -110,6 +111,7 @@ export function defaultBackgroundDeps() {
     return {
         runner: makeLaunchctlRunner(),
         getuid: () => process.getuid?.() ?? 0,
+        currentPid: () => process.pid,
         now: () => Date.now(),
         sleep: (ms) => new Promise((resolvePromise) => setTimeout(resolvePromise, ms)),
         listRecordedRuns,
@@ -143,6 +145,8 @@ export function defaultBackgroundDeps() {
             }
         },
         ensureManagedRuntime: (input) => ensureManagedBackgroundRuntime(input),
+        inspectManagedRuntimeSourceIdentity,
+        verifyManagedRuntimeLaunch,
         resolveManagedRuntimePackages: (target) => resolveConfiguredManagedRuntimePackages({
             cwd: target.cwd,
             configPath: target.configPath,
@@ -172,6 +176,118 @@ export async function startBackground(target, deps, poll) {
 export async function restartBackground(target, deps, poll) {
     return (await ensureBackgroundReady(target, deps, poll)).ok ? 0 : 1;
 }
+/**
+ * Private one-shot recovery controller invoked by the maintenance LaunchAgent.
+ * It authenticates the exact helper PID launchd owns, compares the inert source
+ * CLI identity with the strictly parsed loaded worker definition, and repairs
+ * only when the worker, snapshot, definition, or available source closure has
+ * drifted. A healthy worker remains serving while a replacement runtime is
+ * materialized.
+ */
+export async function maintainLaunchdController(target, deps, options) {
+    const helper = maintenancePathsForTarget(target);
+    const uid = deps.getuid();
+    const helperService = await launchdServiceInfo(deps.runner, helper.label, uid);
+    const helperPid = deps.currentPid();
+    if (!helperService.loaded || helperService.pid !== helperPid || !deps.isAlive(helperPid)) {
+        reportMaintenanceFailure(target, deps, "authenticate the launchd-owned recovery controller", new Error(`launchd does not own this helper pid ${helperPid}`));
+        return 1;
+    }
+    const release = await deps.acquireLifecycleLock(target);
+    if (release === undefined) {
+        // A manual lifecycle command owns the label. The helper exits cleanly so
+        // that command can boot it out; StartInterval supplies the next retry.
+        return 0;
+    }
+    let maintainLogsOnly = false;
+    let resultCode = 1;
+    try {
+        const [worker, sources, desiredIdentity] = await Promise.all([
+            launchdManagedWorkerInfo(deps.runner, target.label, uid),
+            findInstances(target, deps),
+            deps.inspectManagedRuntimeSourceIdentity(target.cliPath),
+        ]);
+        let loadedIdentity;
+        if (worker.definition !== undefined) {
+            try {
+                loadedIdentity = await deps.verifyManagedRuntimeLaunch({
+                    currentCliPath: worker.definition.cliPath,
+                    launchProof: worker.definition.expectedManagedRuntimeLaunch,
+                });
+            }
+            catch {
+                // A malformed/unverified loaded closure is a recovery condition. Do not
+                // trust any identity or execute its advertised CLI.
+            }
+        }
+        const source = worker.pid === undefined
+            ? undefined
+            : sources.find((candidate) => candidate.pid === worker.pid);
+        const durableSnapshotStillMatches = await snapshotStillMatches(target, deps);
+        const workerHealthy = worker.loaded
+            && worker.pid !== undefined
+            && deps.isAlive(worker.pid)
+            && source !== undefined
+            && isReady(source, false)
+            && snapshotMetadataMatches(source, target.expectedSnapshot)
+            && durableSnapshotStillMatches;
+        const definitionMatches = worker.definition !== undefined
+            && managedWorkerDefinitionMatchesTarget(worker.definition, target);
+        const runtimeMatches = loadedIdentity !== undefined
+            && sameManagedRuntimeIdentity(loadedIdentity, desiredIdentity);
+        const needsRecovery = !workerHealthy
+            || !definitionMatches
+            || loadedIdentity === undefined
+            || (options.sourceAvailable && !runtimeMatches);
+        if (!needsRecovery) {
+            maintainLogsOnly = true;
+            resultCode = 0;
+        }
+        else {
+            if (!options.sourceAvailable) {
+                deps.stderr(ui.style.dim(`The original controller CLI is unavailable; recovering ${target.label} from the helper's private closure without claiming an upgrade.`) + "\n");
+            }
+            const recovered = await ensureBackgroundReadyUnlocked(target, deps, options.controlPoll ?? DEFAULT_CONTROL_POLL, options.readinessPoll ?? DEFAULT_READINESS_POLL, { preserveMaintenanceService: true, preserveDefinitionsOnFailure: true });
+            resultCode = recovered.ok ? 0 : 1;
+        }
+    }
+    catch (error) {
+        reportMaintenanceFailure(target, deps, "reconcile the managed worker", error);
+        resultCode = 1;
+    }
+    finally {
+        await release().catch((error) => {
+            deps.stderr(ui.errorLine(`Could not release lifecycle lock for ${target.label}: ${error instanceof Error ? error.message : String(error)}`));
+        });
+    }
+    return maintainLogsOnly
+        ? await maintainLaunchdLogs(target, deps, options.controlPoll ?? DEFAULT_CONTROL_POLL)
+        : resultCode;
+}
+function managedWorkerDefinitionMatchesTarget(definition, target) {
+    return target.expectedSnapshot !== undefined
+        && definition.plistPath === target.paths.plistPath
+        && definition.nodePath === target.nodePath
+        && definition.configPath === target.configPath
+        && definition.cwd === target.cwd
+        && definition.envFile === target.envFile
+        && definition.stdoutPath === target.paths.stdoutPath
+        && definition.stderrPath === target.paths.stderrPath
+        && definition.expectedBackgroundSnapshot === encodeBackgroundSnapshot(target.expectedSnapshot)
+        && sameStringRecord(definition.environment, target.environment);
+}
+function sameManagedRuntimeIdentity(loaded, desired) {
+    return loaded.packageVersion === desired.packageVersion && loaded.cliSha256 === desired.cliSha256;
+}
+function sameStringRecord(left, right) {
+    const leftEntries = Object.entries(left).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+    const rightEntries = Object.entries(right).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
+    return leftEntries.length === rightEntries.length
+        && leftEntries.every(([name, value], index) => {
+            const other = rightEntries[index];
+            return other?.[0] === name && other[1] === value;
+        });
+}
 /**
  * Stop, perform the caller's stopped-worker mutation, and start again while
  * retaining one lifecycle lock. This closes the force-restart gap in which a
@@ -427,7 +543,7 @@ export async function ensureBackgroundReady(target, deps, poll) {
         });
     }
 }
-async function ensureBackgroundReadyUnlocked(target, deps, controlPoll, readinessPoll) {
+async function ensureBackgroundReadyUnlocked(target, deps, controlPoll, readinessPoll, options = {}) {
     if (target.expectedSnapshot === undefined) {
         deps.stderr(ui.errorLine("Refusing to launch a managed worker without an approved background snapshot."));
         deps.stderr(ui.hint("No LaunchAgent changes were made. Retry from the wizard or `mono-agent start`."));
@@ -453,9 +569,9 @@ async function ensureBackgroundReadyUnlocked(target, deps, controlPoll, readines
         deps.stderr(ui.errorLine(`Another runtime publication is already active for ${target.label}.`));
         return { ok: false, action: "start", reason: "ownership" };
     }
-    return await ensureBackgroundReadyWithPublicationBarrier(target, deps, controlPoll, readinessPoll, publicationBarrier);
+    return await ensureBackgroundReadyWithPublicationBarrier(target, deps, controlPoll, readinessPoll, publicationBarrier, options);
 }
-async function ensureBackgroundReadyWithPublicationBarrier(target, deps, controlPoll, readinessPoll, publicationBarrier) {
+async function ensureBackgroundReadyWithPublicationBarrier(target, deps, controlPoll, readinessPoll, publicationBarrier, options) {
     let barrierReleased = false;
     const releaseBarrier = async () => {
         if (barrierReleased)
@@ -464,7 +580,7 @@ async function ensureBackgroundReadyWithPublicationBarrier(target, deps, control
         barrierReleased = true;
     };
     try {
-        return await ensureBackgroundReadyAfterPublicationBarrier(target, deps, controlPoll, readinessPoll, releaseBarrier);
+        return await ensureBackgroundReadyAfterPublicationBarrier(target, deps, controlPoll, readinessPoll, releaseBarrier, options);
     }
     finally {
         if (!barrierReleased) {
@@ -474,7 +590,7 @@ async function ensureBackgroundReadyWithPublicationBarrier(target, deps, control
         }
     }
 }
-async function ensureBackgroundReadyAfterPublicationBarrier(target, deps, controlPoll, readinessPoll, releaseBarrier) {
+async function ensureBackgroundReadyAfterPublicationBarrier(target, deps, controlPoll, readinessPoll, releaseBarrier, options) {
     const uid = deps.getuid();
     let launchTarget;
     const runtimeStartedAt = deps.now();
@@ -560,7 +676,7 @@ async function ensureBackgroundReadyAfterPublicationBarrier(target, deps, contro
             // The replacement plist now contains the finalized-runtime proof. Let
             // launchd respawn only after the stopped-window commit is complete.
             await releaseBarrier();
-        });
+        }, { preserveMaintenanceService: options.preserveMaintenanceService === true });
     }
     catch (error) {
         reportLifecycleException(launchTarget, deps, prepared ? "start the LaunchAgent" : "prepare the stopped LaunchAgent", error);
@@ -582,13 +698,13 @@ async function ensureBackgroundReadyAfterPublicationBarrier(target, deps, contro
     }
     catch (error) {
         reportReadinessException(deps, error);
-        const stopped = await cleanUpUnreadyBackground(launchTarget, deps, controlPoll);
-        reportReadinessCleanup(launchTarget, deps, stopped);
+        const stopped = await cleanUpUnreadyBackground(launchTarget, deps, controlPoll, options);
+        reportReadinessCleanup(launchTarget, deps, stopped, options);
         return { ok: false, action, reason: "readiness" };
     }
     if (ready === undefined) {
         if (!(await snapshotStillMatches(launchTarget, deps))) {
-            const stopped = await stopBackgroundUnlocked(launchTarget, deps, controlPoll);
+            const stopped = await cleanUpUnreadyBackground(launchTarget, deps, controlPoll, options) ? 0 : 1;
             reportSnapshotDrift(launchTarget, deps);
             if (stopped === 0) {
                 deps.stderr(ui.style.dim("The drifted LaunchAgent was stopped before returning control.") + "\n");
@@ -599,8 +715,8 @@ async function ensureBackgroundReadyAfterPublicationBarrier(target, deps, contro
             return { ok: false, action, reason: "snapshot" };
         }
         reportTimeout(deps);
-        const stopped = await cleanUpUnreadyBackground(launchTarget, deps, controlPoll);
-        reportReadinessCleanup(launchTarget, deps, stopped);
+        const stopped = await cleanUpUnreadyBackground(launchTarget, deps, controlPoll, options);
+        reportReadinessCleanup(launchTarget, deps, stopped, options);
         return { ok: false, action, reason: "timeout" };
     }
     const completedAction = outcome.restarted ? "restarted" : "started";
@@ -633,6 +749,10 @@ async function writePlists(target, deps) {
     if (target.managedRuntimeLaunchProof === undefined) {
         throw new Error("A managed LaunchAgent requires a finalized runtime launch proof.");
     }
+    const agentPath = target.environment.PATH;
+    if (agentPath === undefined || agentPath.length === 0) {
+        throw new Error("A managed LaunchAgent recovery controller requires the durable worker PATH.");
+    }
     const maintenance = maintenancePathsForTarget(target);
     const mainXml = buildPlistXml({
         label: target.label,
@@ -652,6 +772,10 @@ async function writePlists(target, deps) {
         nodePath: target.nodePath,
         cliPath: target.cliPath,
         configPath: target.configPath,
+        controllerCliPath: target.controllerCliPath ?? target.cliPath,
+        agentCwd: target.cwd,
+        agentPath,
+        ...(target.envFile === undefined ? {} : { envFile: target.envFile }),
         // The agent folder may be renamed or removed while its already-running
         // worker still owns log descriptors. Keep the scheduler's cwd in the
         // durable account state tree; its config argument is already absolute.
@@ -832,7 +956,7 @@ function sameFileSnapshot(left, right) {
  * before bootstrap so launchd cannot retain stale ProgramArguments or env from
  * the previous plist.
  */
-async function bootstrapOrRestart(target, deps, uid, poll, beforeMainBootout, whileStopped) {
+async function bootstrapOrRestart(target, deps, uid, poll, beforeMainBootout, whileStopped, options = {}) {
     const maintenance = maintenancePathsForTarget(target);
     const [service, maintenanceService] = await Promise.all([
         launchdServiceInfo(deps.runner, target.label, uid),
@@ -842,9 +966,11 @@ async function bootstrapOrRestart(target, deps, uid, poll, beforeMainBootout, wh
     // Stop the scheduler first. A helper that is already running either holds
     // this lifecycle lock (so the controller could not enter) or is waiting to
     // acquire it; bootout plus PID death prevents a post-stop resurrection.
-    const maintenanceStopped = await unloadLaunchdService(maintenance.label, maintenanceService, uniquePids([maintenanceService.pid]), deps, uid, poll);
-    if (!maintenanceStopped.ok) {
-        return { ok: false, restarted, failure: maintenanceStopped.failure };
+    if (options.preserveMaintenanceService !== true) {
+        const maintenanceStopped = await unloadLaunchdService(maintenance.label, maintenanceService, uniquePids([maintenanceService.pid]), deps, uid, poll);
+        if (!maintenanceStopped.ok) {
+            return { ok: false, restarted, failure: maintenanceStopped.failure };
+        }
     }
     let mainStopProven = false;
     if (service.loaded) {
@@ -860,7 +986,9 @@ async function bootstrapOrRestart(target, deps, uid, poll, beforeMainBootout, wh
         ]);
         const mainStopped = await unloadLaunchdService(target.label, service, oldPids, deps, uid, poll, beforeMainBootout);
         if (!mainStopped.ok) {
-            const restoreFailure = await restoreMaintenanceDefinition(maintenanceService.loaded, maintenance, deps, uid, poll);
+            const restoreFailure = options.preserveMaintenanceService === true
+                ? undefined
+                : await restoreMaintenanceDefinition(maintenanceService.loaded, maintenance, deps, uid, poll);
             return {
                 ok: false,
                 restarted: true,
@@ -872,19 +1000,28 @@ async function bootstrapOrRestart(target, deps, uid, poll, beforeMainBootout, wh
         mainStopProven = true;
     }
     await whileStopped(mainStopProven);
-    const maintenanceBooted = await bootstrap(deps.runner, maintenance.plistPath, uid);
-    const maintenanceLoaded = await pollUntil(deps, poll, async () => await isLoaded(deps.runner, maintenance.label, uid));
-    if (!maintenanceLoaded) {
-        return {
-            ok: false,
-            restarted,
-            failure: lifecycleFailure(maintenanceBooted, "scheduled log maintenance did not report loaded"),
-        };
+    if (options.preserveMaintenanceService !== true) {
+        const maintenanceBooted = await bootstrap(deps.runner, maintenance.plistPath, uid);
+        const maintenanceLoaded = await pollUntil(deps, poll, async () => await isLoaded(deps.runner, maintenance.label, uid));
+        if (!maintenanceLoaded) {
+            return {
+                ok: false,
+                restarted,
+                failure: lifecycleFailure(maintenanceBooted, "scheduled recovery controller did not report loaded"),
+            };
+        }
     }
     const booted = await bootstrap(deps.runner, target.paths.plistPath, uid);
     if (booted.code === 0 || (await isLoaded(deps.runner, target.label, uid))) {
         return { ok: true, restarted };
     }
+    if (options.preserveMaintenanceService === true) {
+        return {
+            ok: false,
+            restarted,
+            failure: lifecycleFailure(booted, "main service did not load; scheduled recovery remains loaded for retry"),
+        };
+    }
     const maintenanceServiceAfterFailure = await launchdServiceInfo(deps.runner, maintenance.label, uid);
     const maintenanceRemoval = await unloadLaunchdService(maintenance.label, maintenanceServiceAfterFailure, uniquePids([maintenanceServiceAfterFailure.pid]), deps, uid, poll);
     return {
@@ -1092,12 +1229,24 @@ async function stopBackgroundUnlocked(target, deps, poll) {
     return 0;
 }
 export async function statusBackground(target, deps, options = {}) {
-    const result = await deps.listTraceSources({ registryDir: target.registryDir, staleAfterMs: target.staleAfterMs });
+    const [result, service] = await Promise.all([
+        deps.listTraceSources({ registryDir: target.registryDir, staleAfterMs: target.staleAfterMs }),
+        launchdServiceInfo(deps.runner, target.label, deps.getuid()),
+    ]);
     const classified = await Promise.all(result.sources.map(async (source) => ({
         source,
         matches: await matchesConfig(source, target.configPath),
     })));
-    const current = classified.find((entry) => entry.matches)?.source;
+    const matchingSources = classified.filter((entry) => entry.matches).map((entry) => entry.source);
+    const recorded = service.pid === undefined
+        ? matchingSources[0]
+        : matchingSources.find((source) => source.pid === service.pid) ?? matchingSources[0];
+    const active = service.loaded
+        && service.pid !== undefined
+        && deps.isAlive(service.pid)
+        && recorded?.pid === service.pid
+        && recorded.health === "running";
+    const current = recorded === undefined || active ? recorded : inactiveStatusSource(recorded);
     // Only surface other instances that are live or crashed — cleanly stopped
     // manifests linger in the registry and would just be noise.
     const others = classified
@@ -1106,11 +1255,11 @@ export async function statusBackground(target, deps, options = {}) {
     if (options.json === true) {
         const instance = current === undefined ? null : await assembleInstanceStatus(current, target, deps);
         deps.stdout(`${JSON.stringify({
-            ok: current?.health === "running",
+            ok: active,
             instance,
             others: others.map(assembleOtherInstanceStatus),
         })}\n`);
-        return current?.health === "running" ? 0 : 1;
+        return active ? 0 : 1;
     }
     if (current === undefined) {
         deps.stdout(ui.style.dim(`No running mono-agent instance for ${target.configPath}.`) + "\n");
@@ -1126,7 +1275,32 @@ export async function statusBackground(target, deps, options = {}) {
             deps.stdout(`${formatOtherInstance(source)}\n`);
         }
     }
-    return current?.health === "running" ? 0 : 1;
+    return active ? 0 : 1;
+}
+function inactiveStatusSource(source) {
+    const { transports: _staleTransports, pid: _stalePid, ...withoutLiveProcessFacts } = source;
+    const metadata = source.metadata;
+    const channels = metadata?.channels;
+    const stoppedChannels = isPlainRecord(channels)
+        ? Object.fromEntries(Object.entries(channels).map(([id, value]) => {
+            if (!isPlainRecord(value) || value.kind !== "running")
+                return [id, value];
+            return [id, { kind: "stopped", reason: "instance is not running" }];
+        }))
+        : undefined;
+    return {
+        ...withoutLiveProcessFacts,
+        health: "stopped",
+        status: "stopped",
+        ...(metadata === undefined
+            ? {}
+            : {
+                metadata: {
+                    ...metadata,
+                    ...(stoppedChannels === undefined ? {} : { channels: stoppedChannels }),
+                },
+            }),
+    };
 }
 /**
  * The machine-readable instance record for `status --json`. It mirrors the
@@ -1295,8 +1469,24 @@ function reportTimeout(deps) {
     deps.stderr(ui.errorLine("mono-agent did not report ready within the timeout."));
     deps.stderr(ui.style.dim("The committed agent files were preserved.") + "\n");
 }
-async function cleanUpUnreadyBackground(target, deps, poll) {
+async function cleanUpUnreadyBackground(target, deps, poll, options = {}) {
     try {
+        if (options.preserveDefinitionsOnFailure === true) {
+            const uid = deps.getuid();
+            const [service, sources] = await Promise.all([
+                launchdServiceInfo(deps.runner, target.label, uid),
+                findInstances(target, deps),
+            ]);
+            const liveTracePids = uniquePids(sources
+                .filter((source) => source.health !== "stopped"
+                && source.pid !== undefined
+                && deps.isAlive(source.pid))
+                .map((source) => source.pid));
+            if (liveTracePids.some((pid) => pid !== service.pid))
+                return false;
+            const stopped = await unloadLaunchdService(target.label, service, uniquePids([service.pid, ...liveTracePids]), deps, uid, poll);
+            return stopped.ok;
+        }
         return await stopBackgroundUnlocked(target, { ...deps, stdout: () => undefined }, poll) === 0;
     }
     catch (error) {
@@ -1304,9 +1494,11 @@ async function cleanUpUnreadyBackground(target, deps, poll) {
         return false;
     }
 }
-function reportReadinessCleanup(target, deps, stopped) {
+function reportReadinessCleanup(target, deps, stopped, options = {}) {
     deps.stderr(stopped
-        ? ui.style.dim("The unready LaunchAgent and scheduled maintenance were stopped and their definitions removed before returning control.") + "\n"
+        ? options.preserveDefinitionsOnFailure === true
+            ? ui.style.dim("The unready main LaunchAgent was stopped; both definitions and the scheduled recovery controller remain for retry.") + "\n"
+            : ui.style.dim("The unready LaunchAgent and scheduled maintenance were stopped and their definitions removed before returning control.") + "\n"
         : ui.style.yellow("The unready LaunchAgent could not be proven stopped; a worker or maintenance helper may still be running.") + "\n");
     reportLaunchRecovery(target, deps);
 }