@mono-agent/agent-app 0.19.0 → 0.20.0

package/dist/doctor.js

@@ -1,9 +1,9 @@
 import { execFile as execFileCallback } from "node:child_process";
 import { constants } from "node:fs";
-import { access, lstat, mkdir, readdir, readFile, stat } from "node:fs/promises";
+import { access, lstat, mkdir, readdir, readFile, realpath, stat } from "node:fs/promises";
 import { homedir } from "node:os";
 import { isIP } from "node:net";
-import { join } from "node:path";
+import { basename, join, relative, resolve, sep } from "node:path";
 import { promisify } from "node:util";
 // `BuiltinProvider`, not the root `KnownProvider`: since pi-ai 0.83.0 the
 // latter also covers purely dynamic providers (e.g. "radius") that have no
@@ -28,6 +28,9 @@ import { CONTINUATION_RECORD_STORE_SCHEMA_VERSION, } from "./continuation-store.
 import { applyRetention, isDurableGeneration, isOriginContextGroupCommit, isRecord, isRecordTransaction, isStoreFile, normalizeLegacyContinuationRecords, requiredDate, resolveRetention, } from "./continuation-store-policy.js";
 import { MAX_LEGACY_STORE_BYTES, MAX_MANIFEST_BYTES, MAX_RECORD_BYTES, MAX_TRANSACTION_BYTES, } from "./continuation-store-types.js";
 import { CONTINUATION_STATES, continuationDigest } from "./continuations.js";
+import { isProcessJobState, PROCESS_JOB_STATES } from "@mono-agent/agent-contracts";
+import { loadProcessJobsSettings } from "./process-jobs-config.js";
+import { MAX_PROCESS_JOB_HEALTH_BYTES, PROCESS_JOB_HEALTH_FILE, PROCESS_JOB_QUARANTINE_DIRECTORY, PROCESS_JOB_RECORDS_DIRECTORY, PROCESS_JOB_SECRET_FILE, } from "./process-jobs-store.js";
 import { formatInteractionBridgeUrl, loadInteractionSettings } from "./interaction-bridge.js";
 import { FIRST_RUN_MEMORY_INITIALIZING_MARKER } from "./first-run-managed-memory.js";
 import { DEFAULT_MEMORY_EMBEDDING_ENDPOINTS, probeMemoryEmbeddingSelection, } from "./memory-embedding-service.js";
@@ -37,8 +40,11 @@ import { checkManagedProjectSkills, managedProjectSkillsExist } from "./project-
 import { configuredRuntimeFallbackModels, configuredRuntimeModels } from "./runtime-routes.js";
 import { runtimeProvenanceDetail } from "./runtime-provenance.js";
 import { loadSupermemoryPlugin } from "./supermemory-plugin.js";
-import { DEFAULT_LAUNCHD_LOG_POLICY, inspectLaunchdLogs, LAUNCHD_LOG_MAINTENANCE_INTERVAL_SECONDS, launchdLogPathsForConfig, } from "./launchd-logs.js";
+import { DEFAULT_LAUNCHD_LOG_POLICY, inspectLaunchdLogs, LAUNCHD_LOG_MONITOR_INTERVAL_SECONDS, launchdLogPathsForConfig, } from "./launchd-logs.js";
+import { readLaunchdLogMonitorStatus } from "./launchd-log-monitor-status.js";
 import { exporterSection, runsSection } from "./doctor-observability.js";
+import { sessionToolHistorySection } from "./doctor-session-history.js";
+import { runtimeRouteContainsUnsupportedHistoryTool } from "./app-controller-utils.js";
 const execFile = promisify(execFileCallback);
 const CONTINUATION_V2_ROLLBACK_GUARD = "UPGRADED-TO-RECORDS-V3";
 const CONTINUATION_V2_ROLLBACK_GUARD_CONTENT = "This state directory uses continuation records v3. Older runtimes must not open records-v2.\n";
@@ -88,8 +94,13 @@ export async function validateMonoAgentFolder(options) {
         sections.push(await contextSection(coreConfig, options.cwd));
         sections.push(await memorySection(coreConfig, options.cwd, options.env, liveness, allowFilesystemWrites, options.preferAppPluginInstall === true));
         sections.push(await toolsSection(coreConfig, options));
+        sections.push(await sessionToolHistorySection({
+            historyRoot: join(coreConfig.artifacts.dir, "..", "history"),
+            requestScopedToolSupported: !runtimeRouteContainsUnsupportedHistoryTool(coreConfig),
+        }));
         sections.push(await webToolsSection(coreConfig, options, liveness));
         sections.push(await continuationSection(coreConfig, options));
+        sections.push(await processJobsSection(coreConfig, options));
         sections.push(await sandboxSection(coreConfig, options.sandboxEngine));
     }
     sections.push(await exporterSection(options, liveness));
@@ -120,9 +131,20 @@ export async function validateMonoAgentFolder(options) {
 /** Read-only launchd log inventory used by both `validate` and its `doctor` alias. */
 export async function launchdLogsSection(configPath) {
     let inspection;
+    let monitorStatus;
     try {
         const paths = await launchdLogPathsForConfig(configPath);
         inspection = await inspectLaunchdLogs(paths);
+        const stdoutName = basename(paths.stdoutPath);
+        const mainLabel = stdoutName.endsWith(".out.log")
+            ? stdoutName.slice(0, -".out.log".length)
+            : "";
+        try {
+            monitorStatus = await readLaunchdLogMonitorStatus(mainLabel, paths);
+        }
+        catch {
+            monitorStatus = "unavailable";
+        }
     }
     catch {
         return {
@@ -132,15 +154,16 @@ export async function launchdLogsSection(configPath) {
             details: ["[WARN] Managed launchd log metadata could not be inspected safely."],
         };
     }
-    return launchdLogsSectionFromInspection(inspection);
+    return launchdLogsSectionFromInspection(inspection, monitorStatus);
 }
 /** Pure renderer kept separate so exact byte accounting is deterministic in tests. */
-export function launchdLogsSectionFromInspection(inspection) {
+export function launchdLogsSectionFromInspection(inspection, monitorStatus) {
     const policy = DEFAULT_LAUNCHD_LOG_POLICY;
     const details = [
-        `Policy: ${policy.maxBytes} bytes per file, ${policy.rotationCount} retained generations, checked every ${LAUNCHD_LOG_MAINTENANCE_INTERVAL_SECONDS} seconds.`,
+        `Policy: ${policy.maxBytes} bytes per file, ${policy.rotationCount} retained generations, checked every ${LAUNCHD_LOG_MONITOR_INTERVAL_SECONDS} seconds.`,
         streamSizeDetail("stdout", inspection.stdout),
         streamSizeDetail("stderr", inspection.stderr),
+        ...monitorStatusDetails(monitorStatus),
         ...inspection.issues.map((issue) => `[WARN] ${issue}.`),
         ...oversizedLogDetails("stdout", inspection.stdout, policy.maxBytes),
         ...oversizedLogDetails("stderr", inspection.stderr, policy.maxBytes),
@@ -160,6 +183,19 @@ export function launchdLogsSectionFromInspection(inspection) {
         details,
     };
 }
+function monitorStatusDetails(status) {
+    if (status === "unavailable") {
+        return ["Monitor: owner-private status is unavailable or unsafe."];
+    }
+    if (status === undefined)
+        return ["Monitor: no observational snapshot has been recorded yet."];
+    return [
+        `Monitor last inspection: ${status.lastInspectionAt}.`,
+        `Monitor wake count: ${status.wakeCount}.`,
+        `Monitor last outcome: ${status.lastOutcome}.`,
+        `Monitor cooldown deadline: ${status.cooldownDeadline}.`,
+    ];
+}
 function streamSizeDetail(label, stream) {
     if (!stream.byteAccountingComplete) {
         return `${label}: byte inventory unavailable because one or more paths could not be inspected safely.`;
@@ -1910,9 +1946,21 @@ async function probeSearxngEndpoint(endpoint) {
         if (!response.ok)
             return { ok: false, reason: `HTTP ${response.status}` };
         const body = await response.json();
-        return Array.isArray(body.results)
-            ? { ok: true, reason: "" }
-            : { ok: false, reason: "response did not contain a results array" };
+        if (!Array.isArray(body.results)) {
+            return { ok: false, reason: "response did not contain a results array" };
+        }
+        // An instance whose engines are all captcha'd or suspended answers 200 with
+        // an empty result list, which the array check alone reported as healthy. The
+        // empty array is not the signal on its own — this probe's query can
+        // legitimately match very little — so only flag it alongside failed engines.
+        const unresponsive = Array.isArray(body.unresponsive_engines) ? body.unresponsive_engines : [];
+        if (body.results.length === 0 && unresponsive.length > 0) {
+            const detail = unresponsive
+                .map((entry) => (Array.isArray(entry) ? `${String(entry[0])}: ${String(entry[1])}` : String(entry)))
+                .join("; ");
+            return { ok: false, reason: `no results, and ${unresponsive.length === 1 ? "1 engine" : `${unresponsive.length} engines`} unresponsive — ${detail}` };
+        }
+        return { ok: true, reason: "" };
     }
     catch (error) {
         return { ok: false, reason: error instanceof Error ? error.message : String(error) };
@@ -2002,6 +2050,269 @@ async function continuationSection(config, input) {
         details,
     };
 }
+async function processJobsSection(config, input) {
+    let settings;
+    try {
+        settings = await loadProcessJobsSettings({ cwd: input.cwd, configPath: input.configPath, env: input.env });
+    }
+    catch (error) {
+        return {
+            id: "process-jobs",
+            label: "Background process jobs",
+            status: "error",
+            details: [`Process-job configuration is invalid: ${continuationReason(error)}`],
+        };
+    }
+    if (!settings.enabled) {
+        return {
+            id: "process-jobs",
+            label: "Background process jobs",
+            status: "disabled",
+            details: [
+                "Background Exec/Bash jobs are opt-in (processJobs.enabled=false).",
+                ...(process.platform === "win32" ? ["Windows is unsupported; detached POSIX process-group ownership is required."] : []),
+            ],
+        };
+    }
+    if (process.platform === "win32") {
+        return {
+            id: "process-jobs",
+            label: "Background process jobs",
+            status: "error",
+            details: ["Process jobs are enabled but Windows is unsupported; detached POSIX process-group ownership is required."],
+        };
+    }
+    const details = [
+        `Owner-only local state: ${settings.stateDir}.`,
+        `Concurrency: ${String(settings.maxConcurrent)} global, ${String(settings.maxActivePerConversation)} per conversation, ${String(settings.maxQueued)} queued.`,
+        `Caps: runtime=${String(settings.maxRuntimeMs)}ms, queue-age=${String(settings.maxQueueAgeMs)}ms, output=${String(settings.maxOutputBytes)} bytes, chain-depth=${String(settings.maxChainDepth)}.`,
+        `Runtime availability: Pi-native Exec/Bash only; configured primary is ${config.runtime.model.sdk}.`,
+    ];
+    const inspection = await inspectProcessJobState(input.cwd, settings.stateDir);
+    return {
+        id: "process-jobs",
+        label: "Background process jobs",
+        status: inspection.status,
+        details: [...details, ...inspection.details],
+    };
+}
+async function inspectProcessJobState(cwd, stateDir) {
+    const confinementError = await inspectProcessJobConfinement(cwd, stateDir);
+    if (confinementError !== undefined) {
+        return { status: "error", details: [confinementError] };
+    }
+    let root;
+    try {
+        root = await lstat(stateDir);
+    }
+    catch (error) {
+        return continuationFsCode(error) === "ENOENT"
+            ? { status: "ok", details: ["State has not been initialized; the app will create it owner-only on first start."] }
+            : { status: "error", details: [`Process-job state cannot be inspected: ${continuationReason(error)}`] };
+    }
+    if (!root.isDirectory() || root.isSymbolicLink()) {
+        return { status: "error", details: ["Process-job state root must be a real directory, not a symlink."] };
+    }
+    const rootSecurity = processJobOwnershipError(root, "state directory", 0o700);
+    if (rootSecurity !== undefined)
+        return { status: "error", details: [rootSecurity] };
+    const healthInspection = await inspectProcessJobHealth(stateDir);
+    if (healthInspection !== undefined)
+        return healthInspection;
+    const quarantineInspection = await inspectProcessJobQuarantine(stateDir);
+    if (quarantineInspection !== undefined)
+        return quarantineInspection;
+    const secretPath = join(stateDir, PROCESS_JOB_SECRET_FILE);
+    try {
+        const secret = await lstat(secretPath);
+        if (!secret.isFile() || secret.isSymbolicLink() || Number(secret.nlink) !== 1 || Number(secret.size) > 256) {
+            return { status: "error", details: ["Process-job operator secret must be one regular file."] };
+        }
+        const secretSecurity = processJobOwnershipError(secret, "operator secret", 0o600);
+        if (secretSecurity !== undefined)
+            return { status: "error", details: [secretSecurity] };
+    }
+    catch (error) {
+        if (continuationFsCode(error) !== "ENOENT") {
+            return { status: "error", details: [`Process-job operator secret cannot be inspected: ${continuationReason(error)}`] };
+        }
+    }
+    const recordsPath = join(stateDir, PROCESS_JOB_RECORDS_DIRECTORY);
+    let names;
+    try {
+        const records = await lstat(recordsPath);
+        if (!records.isDirectory() || records.isSymbolicLink()) {
+            return { status: "error", details: ["Process-job records path must be a real directory."] };
+        }
+        const recordsSecurity = processJobOwnershipError(records, "records directory", 0o700);
+        if (recordsSecurity !== undefined)
+            return { status: "error", details: [recordsSecurity] };
+        names = (await readdir(recordsPath)).filter((name) => name.endsWith(".json"));
+    }
+    catch (error) {
+        return continuationFsCode(error) === "ENOENT"
+            ? { status: "ok", details: ["No local process-job records have been written yet."] }
+            : { status: "error", details: [`Process-job records cannot be inspected: ${continuationReason(error)}`] };
+    }
+    if (names.length > 10_000)
+        return { status: "error", details: ["Process-job record count exceeds the compiled inspection bound of 10000."] };
+    const counts = Object.fromEntries(PROCESS_JOB_STATES.map((state) => [state, 0]));
+    for (const name of names) {
+        const path = join(recordsPath, name);
+        try {
+            const info = await lstat(path);
+            if (!info.isFile() || info.isSymbolicLink() || Number(info.nlink) !== 1)
+                throw new Error("record is not one regular file");
+            const security = processJobOwnershipError(info, `record ${name}`, 0o600);
+            if (security !== undefined)
+                throw new Error(security);
+            if (Number(info.size) > 128 * 1024)
+                throw new Error("record exceeds 131072 bytes");
+            const raw = JSON.parse(await readFile(path, "utf8"));
+            if (!isDoctorObject(raw) || !isProcessJobState(raw.state) || raw.jobId !== name.slice(0, -5)) {
+                throw new Error("record identity or state is invalid");
+            }
+            counts[raw.state] = (counts[raw.state] ?? 0) + 1;
+        }
+        catch (error) {
+            return { status: "error", details: [`Process-job record ${name} is unsafe or malformed: ${continuationReason(error)}`] };
+        }
+    }
+    return {
+        status: "ok",
+        details: [
+            `Local records: ${String(names.length)}.`,
+            `States: ${PROCESS_JOB_STATES.map((state) => `${state}=${String(counts[state] ?? 0)}`).join(", ")}.`,
+        ],
+    };
+}
+async function inspectProcessJobHealth(stateDir) {
+    const path = join(stateDir, PROCESS_JOB_HEALTH_FILE);
+    try {
+        const info = await lstat(path);
+        if (!info.isFile() || info.isSymbolicLink() || Number(info.nlink) !== 1) {
+            throw new Error("runtime health marker is not one regular file");
+        }
+        const security = processJobOwnershipError(info, "runtime health marker", 0o600);
+        if (security !== undefined)
+            throw new Error(security);
+        if (Number(info.size) > MAX_PROCESS_JOB_HEALTH_BYTES) {
+            throw new Error(`runtime health marker exceeds ${String(MAX_PROCESS_JOB_HEALTH_BYTES)} bytes`);
+        }
+        const value = JSON.parse(await readFile(path, "utf8"));
+        if (!isDoctorObject(value)
+            || Object.keys(value).sort().join(",") !== "detectedAt,operation,schemaVersion,state"
+            || value.schemaVersion !== 1
+            || value.state !== "degraded"
+            || typeof value.operation !== "string"
+            || !/^[a-z][a-z0-9_.-]{0,63}$/u.test(value.operation)
+            || typeof value.detectedAt !== "string"
+            || !Number.isFinite(Date.parse(value.detectedAt))) {
+            throw new Error("runtime health marker is malformed");
+        }
+        return {
+            status: "error",
+            details: [
+                `Process-job storage degraded during ${value.operation} at ${value.detectedAt}; new admission is closed until a clean restart recovery.`,
+            ],
+        };
+    }
+    catch (error) {
+        return continuationFsCode(error) === "ENOENT"
+            ? undefined
+            : {
+                status: "error",
+                details: [`Process-job runtime health cannot be inspected: ${continuationReason(error)}`],
+            };
+    }
+}
+async function inspectProcessJobQuarantine(stateDir) {
+    const quarantinePath = join(stateDir, PROCESS_JOB_QUARANTINE_DIRECTORY);
+    let names;
+    try {
+        const directory = await lstat(quarantinePath);
+        if (!directory.isDirectory() || directory.isSymbolicLink()) {
+            return { status: "error", details: ["Process-job quarantine path must be a real directory."] };
+        }
+        const security = processJobOwnershipError(directory, "quarantine directory", 0o700);
+        if (security !== undefined)
+            return { status: "error", details: [security] };
+        names = await readdir(quarantinePath);
+    }
+    catch (error) {
+        return continuationFsCode(error) === "ENOENT"
+            ? undefined
+            : { status: "error", details: [`Process-job quarantine cannot be inspected: ${continuationReason(error)}`] };
+    }
+    if (names.length > 10_000) {
+        return { status: "error", details: ["Process-job quarantined transaction count exceeds 10000."] };
+    }
+    for (const name of names) {
+        const path = join(quarantinePath, name);
+        try {
+            if (!/^transaction-\d{4}-\d{2}-\d{2}T[0-9.-]+Z-[0-9a-f-]{36}\.json$/iu.test(name)) {
+                throw new Error("quarantine filename is invalid");
+            }
+            const info = await lstat(path);
+            if (!info.isFile() || info.isSymbolicLink() || Number(info.nlink) !== 1) {
+                throw new Error("quarantined transaction is not one regular file");
+            }
+            const security = processJobOwnershipError(info, `quarantined transaction ${name}`, 0o600);
+            if (security !== undefined)
+                throw new Error(security);
+            if (Number(info.size) > 256 * 1024)
+                throw new Error("quarantined transaction exceeds 262144 bytes");
+        }
+        catch (error) {
+            return {
+                status: "error",
+                details: [`Process-job quarantined transaction ${name} is unsafe: ${continuationReason(error)}`],
+            };
+        }
+    }
+    return names.length === 0
+        ? undefined
+        : {
+            status: "error",
+            details: [
+                `Quarantined unreplayable process-job transactions: ${String(names.length)}.`,
+                "The agent can continue without the affected transaction, but operator review is required.",
+            ],
+        };
+}
+function processJobOwnershipError(info, label, expectedMode) {
+    if (typeof process.getuid === "function" && Number(info.uid) !== process.getuid()) {
+        return `Process-job ${label} is not owned by the current user.`;
+    }
+    if (process.platform !== "win32" && (Number(info.mode) & 0o777) !== expectedMode) {
+        return `Process-job ${label} permissions must be ${expectedMode.toString(8)}.`;
+    }
+    return undefined;
+}
+async function inspectProcessJobConfinement(cwd, stateDir) {
+    const lexicalRoot = resolve(cwd);
+    const root = await realpath(lexicalRoot).catch(() => lexicalRoot);
+    const rel = relative(root, resolve(stateDir));
+    if (rel.length === 0 || rel === ".." || rel.startsWith(`..${sep}`)) {
+        return "Process-job state must be a child directory inside the canonical agent root.";
+    }
+    let current = root;
+    for (const component of rel.split(sep).filter(Boolean)) {
+        current = join(current, component);
+        try {
+            const info = await lstat(current);
+            if (!info.isDirectory() || info.isSymbolicLink()) {
+                return `Process-job state component must be a real directory, not a symlink: ${current}.`;
+            }
+        }
+        catch (error) {
+            if (continuationFsCode(error) === "ENOENT")
+                return undefined;
+            return `Process-job state confinement cannot be inspected: ${continuationReason(error)}`;
+        }
+    }
+    return undefined;
+}
 async function inspectContinuationState(stateDir, retention) {
     const details = [];
     const retentionPolicy = resolveRetention(retention);