@ouro.bot/cli 0.1.0-alpha.51 → 0.1.0-alpha.510

package/dist/heart/daemon/process-manager.js

@@ -51,15 +51,70 @@ class DaemonProcessManager {
     now;
     setTimeoutFn;
     clearTimeoutFn;
+    cooldownRecoveryMs;
+    maxCooldownRetries;
+    existsSyncFn;
+    configCheckFn;
+    statusWriterFn;
+    onSnapshotChangeFn;
+    /**
+     * Notify the snapshot-change observer (if registered). Swallows any
+     * errors from the observer so process lifecycle code never fails
+     * because the observer threw.
+     */
+    notifySnapshotChange(snapshot) {
+        if (!this.onSnapshotChangeFn)
+            return;
+        try {
+            this.onSnapshotChangeFn(snapshot);
+        }
+        catch (error) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "warn",
+                component: "daemon",
+                event: "daemon.snapshot_change_observer_error",
+                message: "snapshot-change observer threw",
+                meta: {
+                    agent: snapshot.name,
+                    error: error instanceof Error ? error.message : /* v8 ignore next -- defensive: non-Error catch branch @preserve */ String(error),
+                },
+            });
+        }
+    }
+    writeStatus(agent, text) {
+        try {
+            this.statusWriterFn(text);
+        }
+        catch (error) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "warn",
+                component: "daemon",
+                event: "daemon.status_writer_error",
+                message: "daemon status writer threw",
+                meta: {
+                    agent,
+                    error: error instanceof Error ? error.message : String(error),
+                },
+            });
+        }
+    }
     constructor(options) {
         this.maxRestartsPerHour = options.maxRestartsPerHour ?? 10;
         this.stabilityThresholdMs = options.stabilityThresholdMs ?? 60_000;
         this.initialBackoffMs = options.initialBackoffMs ?? 1_000;
         this.maxBackoffMs = options.maxBackoffMs ?? 60_000;
+        this.cooldownRecoveryMs = options.cooldownRecoveryMs ?? 5 * 60 * 1_000;
+        this.maxCooldownRetries = options.maxCooldownRetries ?? 3;
         this.spawnFn = options.spawn ?? ((command, args, spawnOptions) => (0, child_process_1.spawn)(command, args, spawnOptions));
         this.now = options.now ?? (() => Date.now());
         this.setTimeoutFn = options.setTimeoutFn ?? ((cb, delay) => setTimeout(cb, delay));
         this.clearTimeoutFn = options.clearTimeoutFn ?? ((timer) => clearTimeout(timer));
+        this.existsSyncFn = options.existsSync ?? null;
+        this.configCheckFn = options.configCheck ?? null;
+        this.statusWriterFn = options.statusWriter ?? ((text) => {
+            process.stderr.write(text);
+        });
+        this.onSnapshotChangeFn = options.onSnapshotChange ?? null;
         for (const agent of options.agents) {
             this.agents.set(agent.name, {
                 config: agent,
@@ -67,6 +122,9 @@ class DaemonProcessManager {
                 restartTimer: null,
                 crashTimestamps: [],
                 stopRequested: false,
+                cooldownTimer: null,
+                cooldownRetryCount: 0,
+                fastCrashCount: 0,
                 snapshot: {
                     name: agent.name,
                     channel: agent.channel,
@@ -76,6 +134,10 @@ class DaemonProcessManager {
                     startedAt: null,
                     lastCrashAt: null,
                     backoffMs: this.initialBackoffMs,
+                    lastExitCode: null,
+                    lastSignal: null,
+                    errorReason: null,
+                    fixHint: null,
                 },
             });
         }
@@ -94,14 +156,75 @@ class DaemonProcessManager {
         this.clearRestartTimer(state);
         state.stopRequested = false;
         state.snapshot.status = "starting";
+        if (this.configCheckFn) {
+            const result = await this.configCheckFn(agent);
+            if (result.skip) {
+                state.snapshot.status = "stopped";
+                state.snapshot.errorReason = null;
+                state.snapshot.fixHint = null;
+                (0, runtime_1.emitNervesEvent)({
+                    component: "daemon",
+                    event: "daemon.agent_config_skipped",
+                    message: result.error ?? "agent start skipped by config check",
+                    meta: { agent, fix: result.fix ?? null },
+                });
+                this.notifySnapshotChange(state.snapshot);
+                return;
+            }
+            if (!result.ok) {
+                state.snapshot.status = "crashed";
+                // Surface the error and fix to the snapshot so sibling agents can
+                // read it via the pulse. Without this, the diagnosis stayed
+                // trapped in the nerves event and stderr — visible to humans
+                // running `ouro status` or grepping logs, but invisible to
+                // peer agents trying to coordinate around the broken state.
+                state.snapshot.errorReason = result.error ?? "agent config validation failed";
+                state.snapshot.fixHint = result.fix ?? null;
+                (0, runtime_1.emitNervesEvent)({
+                    level: "error",
+                    component: "daemon",
+                    event: "daemon.agent_config_invalid",
+                    message: result.error ?? "agent config validation failed",
+                    meta: { agent, fix: result.fix ?? null },
+                });
+                this.writeStatus(agent, `[daemon] ${agent}: ${result.error}\n` +
+                    (result.fix ? `  Fix: ${result.fix}\n` : ""));
+                this.notifySnapshotChange(state.snapshot);
+                return;
+            }
+            // Config check passed — clear any prior error so the pulse stops
+            // reporting the broken state. This is the recovery path: the user
+            // fixed their secrets/config, the next startAgent attempt sees a
+            // valid config, and the pulse goes quiet.
+            state.snapshot.errorReason = null;
+            state.snapshot.fixHint = null;
+        }
         const runCwd = (0, identity_1.getRepoRoot)();
         const entryScript = path.join((0, identity_1.getRepoRoot)(), "dist", state.config.entry);
+        if (this.existsSyncFn && !this.existsSyncFn(entryScript)) {
+            state.snapshot.status = "crashed";
+            (0, runtime_1.emitNervesEvent)({
+                level: "error",
+                component: "daemon",
+                event: "daemon.agent_entry_missing",
+                message: "agent entry script does not exist — cannot spawn. Run 'ouro daemon install' from the correct location.",
+                meta: { agent, entryScript },
+            });
+            this.notifySnapshotChange(state.snapshot);
+            return;
+        }
         const args = [entryScript, "--agent", state.config.agentArg ?? agent, ...(state.config.args ?? [])];
         const child = this.spawnFn("node", args, {
             cwd: runCwd,
             env: state.config.env ? { ...process.env, ...state.config.env } : process.env,
             stdio: ["ignore", "ignore", "ignore", "ipc"],
         });
+        /* v8 ignore next 7 -- defensive: spawn should always return a ChildProcess @preserve */
+        if (!child) {
+            state.snapshot.status = "crashed";
+            (0, runtime_1.emitNervesEvent)({ level: "error", component: "daemon", event: "daemon.agent_spawn_failed", message: "spawn returned null", meta: { agent } });
+            return;
+        }
         state.process = child;
         state.snapshot.status = "running";
         state.snapshot.pid = child.pid ?? null;
@@ -112,6 +235,18 @@ class DaemonProcessManager {
             message: "daemon started managed agent process",
             meta: { agent, pid: child.pid ?? null, cwd: runCwd },
         });
+        this.notifySnapshotChange(state.snapshot);
+        /* v8 ignore start — child process error handler; requires real spawn to trigger */
+        child.on("error", (err) => {
+            (0, runtime_1.emitNervesEvent)({
+                level: "warn",
+                component: "daemon",
+                event: "daemon.agent_process_error",
+                message: "managed agent process emitted error",
+                meta: { agent, error: err.message },
+            });
+        });
+        /* v8 ignore stop */
         child.once("exit", (code, signal) => {
             this.onExit(state, code, signal);
         });
@@ -119,10 +254,12 @@ class DaemonProcessManager {
     async stopAgent(agent) {
         const state = this.requireAgent(agent);
         this.clearRestartTimer(state);
+        this.clearCooldownTimer(state);
         state.stopRequested = true;
         if (!state.process) {
             state.snapshot.status = "stopped";
             state.snapshot.pid = null;
+            this.notifySnapshotChange(state.snapshot);
             return;
         }
         const child = state.process;
@@ -141,6 +278,7 @@ class DaemonProcessManager {
                 meta: { agent },
             });
         }
+        this.notifySnapshotChange(state.snapshot);
     }
     async restartAgent(agent) {
         await this.stopAgent(agent);
@@ -179,6 +317,8 @@ class DaemonProcessManager {
             return;
         state.process = null;
         state.snapshot.pid = null;
+        state.snapshot.lastExitCode = code;
+        state.snapshot.lastSignal = signal;
         const crashed = !state.stopRequested && code !== 0;
         const now = this.now();
         const startedAt = state.snapshot.startedAt ? Date.parse(state.snapshot.startedAt) : now;
@@ -195,13 +335,45 @@ class DaemonProcessManager {
             if (runDuration >= this.stabilityThresholdMs) {
                 state.snapshot.backoffMs = this.initialBackoffMs;
             }
+            this.notifySnapshotChange(state.snapshot);
             return;
         }
         state.snapshot.lastCrashAt = new Date(now).toISOString();
+        // Fast-crash detection: if the agent dies within 5 seconds of starting, it's likely
+        // a configuration issue (missing credentials, bad provider, etc.) not a transient failure.
+        // After 3 consecutive fast crashes, stop retrying and mark as config-failed.
+        const FAST_CRASH_THRESHOLD_MS = 5000;
+        const FAST_CRASH_MAX = 3;
+        if (runDuration < FAST_CRASH_THRESHOLD_MS) {
+            state.fastCrashCount = state.fastCrashCount + 1;
+            if (state.fastCrashCount >= FAST_CRASH_MAX) {
+                state.snapshot.status = "crashed";
+                // Capture the fast-crash diagnosis on the snapshot so it surfaces
+                // via the pulse. The error message is prescriptive: it tells the
+                // user (and their sibling agents) exactly what to do.
+                state.snapshot.errorReason = `agent crashed ${FAST_CRASH_MAX} times within ${FAST_CRASH_THRESHOLD_MS}ms of startup — likely a configuration issue (missing credentials, bad provider).`;
+                state.snapshot.fixHint = `Fix the config and run \`ouro up\` to restart, or check daemon logs for the underlying error.`;
+                (0, runtime_1.emitNervesEvent)({
+                    level: "error",
+                    component: "daemon",
+                    event: "daemon.agent_config_failure",
+                    message: `agent crashed ${FAST_CRASH_MAX} times within ${FAST_CRASH_THRESHOLD_MS}ms of startup — likely a configuration issue (missing credentials, bad provider). Not retrying. Fix the config and run \`ouro up\` to restart.`,
+                    meta: { agent: state.config.name, fastCrashCount: state.fastCrashCount, avgRunDurationMs: runDuration },
+                });
+                this.notifySnapshotChange(state.snapshot);
+                return; // Don't schedule cooldown recovery — this needs human/agent intervention
+            }
+        }
+        else {
+            // Reset fast-crash counter on a stable run
+            state.fastCrashCount = 0;
+        }
         state.crashTimestamps = state.crashTimestamps.filter((crashTs) => crashTs >= startOfHour(now));
         state.crashTimestamps.push(now);
         if (state.crashTimestamps.length > this.maxRestartsPerHour) {
             state.snapshot.status = "crashed";
+            state.snapshot.errorReason = `agent exceeded restart limit (${this.maxRestartsPerHour}/hr) — entering cooldown`;
+            state.snapshot.fixHint = "investigate why the agent keeps crashing; cooldown will retry shortly";
             (0, runtime_1.emitNervesEvent)({
                 level: "error",
                 component: "daemon",
@@ -209,6 +381,8 @@ class DaemonProcessManager {
                 message: "managed agent exceeded restart limit and is marked crashed",
                 meta: { agent: state.config.name, maxRestartsPerHour: this.maxRestartsPerHour },
             });
+            this.notifySnapshotChange(state.snapshot);
+            this.scheduleCooldownRecovery(state);
             return;
         }
         state.snapshot.status = "starting";
@@ -219,6 +393,7 @@ class DaemonProcessManager {
         state.restartTimer = this.setTimeoutFn(() => {
             void this.startAgent(state.config.name);
         }, waitMs);
+        this.notifySnapshotChange(state.snapshot);
     }
     clearRestartTimer(state) {
         if (state.restartTimer === null)
@@ -226,6 +401,45 @@ class DaemonProcessManager {
         this.clearTimeoutFn(state.restartTimer);
         state.restartTimer = null;
     }
+    scheduleCooldownRecovery(state) {
+        if (state.cooldownRetryCount >= this.maxCooldownRetries) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "error",
+                component: "daemon",
+                event: "daemon.agent_permanent_failure",
+                message: "managed agent exhausted all cooldown retries — permanently stopped",
+                meta: { agent: state.config.name, cooldownRetryCount: state.cooldownRetryCount, maxCooldownRetries: this.maxCooldownRetries },
+            });
+            return;
+        }
+        this.clearCooldownTimer(state);
+        state.cooldownTimer = this.setTimeoutFn(() => {
+            state.cooldownRetryCount += 1;
+            state.crashTimestamps = [];
+            state.snapshot.backoffMs = this.initialBackoffMs;
+            state.snapshot.status = "starting";
+            state.snapshot.restartCount += 1;
+            (0, runtime_1.emitNervesEvent)({
+                component: "daemon",
+                event: "daemon.agent_cooldown_recovery",
+                message: "attempting cooldown recovery for managed agent",
+                meta: { agent: state.config.name, cooldownRetryCount: state.cooldownRetryCount },
+            });
+            void this.startAgent(state.config.name);
+        }, this.cooldownRecoveryMs);
+        (0, runtime_1.emitNervesEvent)({
+            component: "daemon",
+            event: "daemon.agent_cooldown_scheduled",
+            message: `scheduled cooldown recovery in ${this.cooldownRecoveryMs}ms`,
+            meta: { agent: state.config.name, cooldownRecoveryMs: this.cooldownRecoveryMs, cooldownRetryCount: state.cooldownRetryCount },
+        });
+    }
+    clearCooldownTimer(state) {
+        if (state.cooldownTimer === null)
+            return;
+        this.clearTimeoutFn(state.cooldownTimer);
+        state.cooldownTimer = null;
+    }
     requireAgent(agent) {
         const state = this.agents.get(agent);
         if (!state) {

package/dist/heart/daemon/provider-discovery.js

@@ -0,0 +1,137 @@
+"use strict";
+/**
+ * Shared provider discovery for repair.
+ *
+ * Runtime repair only trusts the agent vault. First-run conveniences may still
+ * inspect env vars before credentials are stored, but once an agent exists the
+ * vault is the source of truth.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scanEnvVarCredentials = scanEnvVarCredentials;
+exports.discoverInstalledAgentCredentials = discoverInstalledAgentCredentials;
+exports.describeDiscoveredCredentialSource = describeDiscoveredCredentialSource;
+exports.discoverWorkingProvider = discoverWorkingProvider;
+const identity_1 = require("../identity");
+const provider_credentials_1 = require("../provider-credentials");
+const runtime_1 = require("../../nerves/runtime");
+/**
+ * Scan environment variables for API keys during first-run bootstrap.
+ * This does not participate in runtime provider repair.
+ */
+function scanEnvVarCredentials(env) {
+    const results = [];
+    for (const [provider, desc] of Object.entries(identity_1.PROVIDER_CREDENTIALS)) {
+        const cred = {};
+        for (const [envVar, credKey] of Object.entries(desc.envVars)) {
+            const value = env[envVar];
+            if (value) {
+                cred[credKey] = value;
+            }
+        }
+        const hasRequired = desc.required.some((key) => !!cred[key]);
+        if (hasRequired) {
+            results.push({
+                provider,
+                agentName: "env",
+                credentials: cred,
+                providerConfig: { ...cred },
+            });
+        }
+    }
+    return results;
+}
+function stringifyProviderFields(fields) {
+    const result = {};
+    for (const [key, value] of Object.entries(fields)) {
+        result[key] = String(value);
+    }
+    return result;
+}
+function discoveredFromVaultRecord(record, agentName = "vault") {
+    return {
+        provider: record.provider,
+        agentName,
+        credentials: stringifyProviderFields(record.credentials),
+        providerConfig: stringifyProviderFields(record.config),
+    };
+}
+async function discoverInstalledAgentCredentials(agentNames) {
+    const discovered = [];
+    for (const agentName of agentNames) {
+        if (agentName === "SerpentGuide")
+            continue;
+        const poolResult = await (0, provider_credentials_1.refreshProviderCredentialPool)(agentName, { preserveCachedOnFailure: true });
+        if (!poolResult.ok)
+            continue;
+        for (const record of Object.values(poolResult.pool.providers)) {
+            if (!record)
+                continue;
+            discovered.push(discoveredFromVaultRecord(record, agentName));
+        }
+    }
+    return discovered;
+}
+function describeDiscoveredCredentialSource(credential, envVar) {
+    if (credential.agentName === "env") {
+        return envVar ? `from env: $${envVar}` : "from env";
+    }
+    return `from ${credential.agentName}'s vault`;
+}
+async function discoverWorkingProvider(deps) {
+    const poolResult = await (0, provider_credentials_1.refreshProviderCredentialPool)(deps.agentName);
+    if (!poolResult.ok) {
+        (0, runtime_1.emitNervesEvent)({
+            level: "warn",
+            component: "daemon",
+            event: "daemon.provider_discovery_none",
+            message: "provider discovery could not read agent vault",
+            meta: { agentName: deps.agentName, reason: poolResult.reason },
+        });
+        return null;
+    }
+    const candidates = Object.entries(poolResult.pool.providers)
+        .map(([, record]) => discoveredFromVaultRecord(record));
+    if (candidates.length === 0) {
+        (0, runtime_1.emitNervesEvent)({
+            level: "info",
+            component: "daemon",
+            event: "daemon.provider_discovery_none",
+            message: "no provider credentials found in agent vault",
+            meta: { agentName: deps.agentName },
+        });
+        return null;
+    }
+    for (const candidate of candidates) {
+        const config = { ...candidate.providerConfig, ...candidate.credentials };
+        (0, runtime_1.emitNervesEvent)({
+            level: "info",
+            component: "daemon",
+            event: "daemon.provider_discovery_ping",
+            message: `pinging provider: ${candidate.provider}`,
+            meta: { agentName: deps.agentName, provider: candidate.provider, source: candidate.agentName },
+        });
+        const result = await deps.pingProvider(candidate.provider, config);
+        if (result.ok) {
+            (0, runtime_1.emitNervesEvent)({
+                level: "info",
+                component: "daemon",
+                event: "daemon.provider_discovery_ok",
+                message: `provider discovery succeeded: ${candidate.provider}`,
+                meta: { agentName: deps.agentName, provider: candidate.provider, source: candidate.agentName },
+            });
+            return {
+                provider: candidate.provider,
+                credentials: candidate.credentials,
+                providerConfig: candidate.providerConfig,
+            };
+        }
+    }
+    (0, runtime_1.emitNervesEvent)({
+        level: "warn",
+        component: "daemon",
+        event: "daemon.provider_discovery_all_failed",
+        message: "all vault provider candidates failed ping",
+        meta: { agentName: deps.agentName, candidateCount: candidates.length },
+    });
+    return null;
+}

package/dist/heart/daemon/provider-ping-progress.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatProviderAttemptProgress = formatProviderAttemptProgress;
+exports.formatProviderRetryProgress = formatProviderRetryProgress;
+exports.createProviderPingProgressReporter = createProviderPingProgressReporter;
+const runtime_1 = require("../../nerves/runtime");
+function formatProviderPingLabel(context) {
+    return context.model ? `${context.provider} / ${context.model}` : context.provider;
+}
+function providerRetryReason(record) {
+    switch (record.classification) {
+        case "auth-failure":
+            return "credentials were rejected";
+        case "usage-limit":
+            return "usage limit hit";
+        case "rate-limit":
+            return "provider asked us to slow down";
+        case "server-error":
+            return record.httpStatus === 529 ? "provider is busy right now" : "provider is having trouble right now";
+        case "network-error":
+            return "network connection dropped";
+        case "unknown":
+            return "last check failed";
+        default:
+            return "last check failed";
+    }
+}
+function providerRetryTiming(delayMs) {
+    if (!delayMs || delayMs <= 0)
+        return "now";
+    const seconds = delayMs / 1000;
+    const rounded = Number.isInteger(seconds) ? `${seconds.toFixed(0)}s` : `${seconds.toFixed(1).replace(/\.0$/, "")}s`;
+    return `in ${rounded}`;
+}
+function formatProviderAttemptProgress(context, attempt, maxAttempts) {
+    const prefix = context.subject ? `${context.subject}: ` : "";
+    if (maxAttempts <= 1)
+        return `${prefix}checking ${formatProviderPingLabel(context)}...`;
+    return `${prefix}checking ${formatProviderPingLabel(context)} (attempt ${attempt} of ${maxAttempts})...`;
+}
+function formatProviderRetryProgress(context, record, maxAttempts) {
+    const nextAttempt = Math.min(record.attempt + 1, maxAttempts);
+    const retryDetail = `${providerRetryReason(record)}; retrying ${providerRetryTiming(record.delayMs)} (attempt ${nextAttempt} of ${maxAttempts})`;
+    if (context.subject) {
+        return `${context.subject}: ${retryDetail} while checking ${formatProviderPingLabel(context)}`;
+    }
+    return `${formatProviderPingLabel(record)}: ${retryDetail}`;
+}
+function createProviderPingProgressReporter(context, onProgress) {
+    return {
+        onAttemptStart: async (attempt, maxAttempts) => {
+            (0, runtime_1.emitNervesEvent)({
+                component: "daemon",
+                event: "daemon.provider_ping_progress_reported",
+                message: "reported provider ping attempt progress",
+                meta: {
+                    provider: context.provider,
+                    model: context.model ?? null,
+                    kind: "attempt",
+                    attempt,
+                    maxAttempts,
+                },
+            });
+            onProgress(formatProviderAttemptProgress(context, attempt, maxAttempts));
+        },
+        onRetry: async (record, maxAttempts) => {
+            (0, runtime_1.emitNervesEvent)({
+                component: "daemon",
+                event: "daemon.provider_ping_progress_reported",
+                message: "reported provider ping retry progress",
+                meta: {
+                    provider: record.provider,
+                    model: record.model,
+                    kind: "retry",
+                    attempt: record.attempt,
+                    maxAttempts,
+                    classification: record.classification ?? "unknown",
+                },
+            });
+            onProgress(formatProviderRetryProgress(context, record, maxAttempts));
+        },
+    };
+}