@songsid/agend 2.1.1-beta.14 → 2.1.1-beta.16

package/dist/fleet-manager.js

@@ -26,7 +26,7 @@ import { processAttachments } from "./channel/attachment-handler.js";
 import { routeToolCall } from "./channel/tool-router.js";
 import { Scheduler } from "./scheduler/index.js";
 import { DEFAULT_SCHEDULER_CONFIG } from "./scheduler/index.js";
-import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG, resolveInstanceContext } from "./topic-commands.js";
+import { TopicCommands, saveCommandForBackend, parseSaveFilename, parsePauseWakeCommand, SAVE_FILENAME_RE, SAVE_UNSUPPORTED_MSG, resolveInstanceContext, forgetInstanceContext } from "./topic-commands.js";
 import { DailySummary } from "./daily-summary.js";
 import { WebhookEmitter } from "./webhook-emitter.js";
 import { TmuxControlClient } from "./tmux-control.js";
@@ -1134,8 +1134,16 @@ export class FleetManager {
                 }
             }
         }
-        // Signal systemd: generals ready
-        sdNotify("READY=1");
+        // The systemd watchdog answers exactly one question: is this process still
+        // turning its event loop? Pinging from a timer proves that, and after the
+        // blocking child-process calls were made async it is a meaningful signal —
+        // a deadlocked or frozen fleet stops pinging and systemd restarts it.
+        //
+        // It deliberately does NOT gate on fleet health. "No adapter connected" or
+        // "an instance crashed" must not kill the process: the fleet would be restarted
+        // into the same broken state, and a user who has legitimately stopped every
+        // instance would get a restart loop. Those conditions surface through /health
+        // (which now returns 503) and through the General-topic notifications instead.
         this.watchdogTimer = setInterval(() => sdNotify("WATCHDOG=1"), 30_000);
         // EventLog.prune() existed but was never called, so `events` and `activity`
         // grew without bound for the life of the install. Prune once at startup and
@@ -1278,6 +1286,15 @@ export class FleetManager {
         // rest of startup finishes. Replay one coalesced reload only after all
         // startup-owned lifecycle work and signal handlers are in place.
         this.finishStartup();
+        // Tell systemd we are ready only now. This used to fire right after the
+        // generals started — before adapters, classic instances, topic creation and the
+        // health server — so `systemctl start` returned success while the fleet was
+        // still deaf: no path existed for a user message to arrive.
+        sdNotify("READY=1");
+        const health = this.getFleetHealth();
+        if (health.status !== "ok") {
+            this.logger.warn({ health }, "Fleet started with problems — see /health");
+        }
     }
     /**
      * Delete inbox files older than retentionDays (by mtime). Cleans the shared
@@ -1394,6 +1411,66 @@ export class FleetManager {
     getAdapterStates() {
         return this.adapterState;
     }
+    /**
+     * Real, checkable fleet health for `/health` and the operator.
+     *
+     * `status` is:
+     *  - `ok`       — at least one adapter connected and every configured instance
+     *                 that should be running is running
+     *  - `degraded` — reachable, but something the operator should look at (an
+     *                 adapter retrying, an instance crashed or stopped)
+     *  - `down`     — the fleet cannot do its job: no adapter is connected, so no
+     *                 message can arrive or be answered
+     *
+     * Deliberately does NOT gate the systemd watchdog — see the comment at the
+     * WATCHDOG timer for why.
+     */
+    getFleetHealth() {
+        const names = Object.keys(this.fleetConfig?.instances ?? {});
+        const counts = { configured: names.length, running: 0, crashed: 0, paused: 0, stopped: 0 };
+        for (const name of names) {
+            const state = this.getInstanceStatus(name);
+            if (state === "running")
+                counts.running++;
+            else if (state === "crashed")
+                counts.crashed++;
+            else if (state === "paused")
+                counts.paused++;
+            else
+                counts.stopped++;
+        }
+        const states = {};
+        let connected = 0;
+        for (const [id, state] of this.adapterState) {
+            states[id] = state.status;
+            if (state.status === "connected")
+                connected++;
+        }
+        const problems = [];
+        if (this.adapterState.size > 0 && connected === 0)
+            problems.push("no channel adapter is connected");
+        if (counts.crashed > 0)
+            problems.push(`${counts.crashed} instance(s) crashed`);
+        for (const [id, state] of this.adapterState) {
+            if (state.status !== "connected")
+                problems.push(`adapter ${id} is ${state.status}`);
+        }
+        if (!this.startupComplete)
+            problems.push("startup has not completed");
+        // "down" is reserved for "cannot receive or answer a message at all". A fleet
+        // with adapters configured but none connected is exactly that.
+        const status = this.adapterState.size > 0 && connected === 0
+            ? "down"
+            : problems.length > 0 ? "degraded" : "ok";
+        return {
+            status,
+            uptime: Math.floor((Date.now() - this.startedAt) / 1000),
+            instances: counts,
+            adapters: { total: this.adapterState.size, connected, states },
+            startupComplete: this.startupComplete,
+            problems,
+        };
+    }
     /** Start the primary adapter (backward-compatible, sets this.adapter) */
     async startSingleAdapter(fleet, channelConfig) {
         const botToken = process.env[channelConfig.bot_token_env];
@@ -1594,17 +1671,7 @@ export class FleetManager {
                     await data.respond(t("not_authorized"));
                     return;
                 }
-                try {
-                    const { execSync } = await import("node:child_process");
-                    const backend = this.fleetConfig?.defaults?.backend || "claude-code";
-                    const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
-                    const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(clean || "No output");
-                }
-                catch (err) {
-                    const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(output);
-                }
+                await data.respond(await this.runBackendDoctor());
             }
             else if (data.command === "status") {
                 const text = await this.topicCommands.getStatusText();
@@ -1876,17 +1943,7 @@ export class FleetManager {
                     await data.respond(t("not_authorized"));
                     return;
                 }
-                try {
-                    const { execSync } = await import("node:child_process");
-                    const backend = this.fleetConfig?.defaults?.backend || "claude-code";
-                    const result = execSync(`agend backend doctor ${backend}`, { timeout: 30_000, encoding: "utf-8" });
-                    const clean = result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(clean || "No output");
-                }
-                catch (err) {
-                    const output = (err.stdout ?? err.message ?? "Doctor failed").replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                    await data.respond(output);
-                }
+                await data.respond(await this.runBackendDoctor());
             }
             else if (data.command === "status") {
                 const text = await this.topicCommands.getStatusText();
@@ -2114,9 +2171,22 @@ export class FleetManager {
                 if (existsSync(windowIdPath)) {
                     const windowId = readFileSync(windowIdPath, "utf-8").trim();
                     if (windowId) {
+                        // Async with an explicit timeout: this was execSync with NO timeout at
+                        // all, so a wedged tmux server blocked the whole fleet event loop
+                        // indefinitely — while we were here to diagnose a lost connection.
+                        // A timeout is also the correct signal: an unresponsive tmux server
+                        // means we cannot verify the pane, which is treated as dead (the same
+                        // conclusion the old code reached only by throwing).
                         try {
-                            const { execSync } = await import("node:child_process");
-                            execSync(`tmux list-panes -t "${windowId}"`, { stdio: "ignore" });
+                            const { execFile } = await import("node:child_process");
+                            const { promisify } = await import("node:util");
+                            const { getTmuxSocketName } = await import("./paths.js");
+                            // Honour socket isolation: without -L this queried the user's default
+                            // tmux server instead of the fleet's, so under a custom AGEND_HOME the
+                            // check was meaningless (it reported every pane dead).
+                            const socket = getTmuxSocketName();
+                            const args = socket ? ["-L", socket, "list-panes", "-t", windowId] : ["list-panes", "-t", windowId];
+                            await promisify(execFile)("tmux", args, { timeout: 5_000 });
                         }
                         catch {
                             // Pane dead — respawn
@@ -2137,6 +2207,12 @@ export class FleetManager {
         if (this.adapterRestarting.has(id))
             return;
         this.adapterRestarting.add(id);
+        // Reflect reality in adapterState throughout. This loop used to leave the state
+        // untouched, so getAdapterStates() — and therefore /health and the dashboard —
+        // kept reporting "connected" for an adapter that had been down for hours. An
+        // adapter's true status was simply not knowable from inside the process.
+        const previous = this.adapterState.get(id);
+        this.adapterState.set(id, { status: "retrying", retryCount: 0, lastError: previous?.lastError });
         try {
             for (let attempt = 1;; attempt++) {
                 if (this.ipcStoppingInstances.has("__fleet_stopping__"))
@@ -2149,9 +2225,16 @@ export class FleetManager {
                     await adapter.stop().catch(() => { });
                     await adapter.start();
                     this.logger.info({ id, attempt }, "Adapter restarted successfully");
+                    this.adapterState.set(id, { status: "connected", retryCount: 0 });
                     return;
                 }
-                catch { /* retry */ }
+                catch (err) {
+                    this.adapterState.set(id, {
+                        status: "retrying",
+                        retryCount: attempt,
+                        lastError: err?.message ?? String(err),
+                    });
+                }
                 if (attempt % 10 === 0) {
                     this.logger.warn({ id, attempt }, "Adapter restart still failing");
                 }
@@ -3358,6 +3441,9 @@ export class FleetManager {
         }
     }
     async removeInstance(name) {
+        // Drop cached pane context — the map is keyed by instance name and nothing
+        // else evicted deleted entries, so it grew for the life of the process.
+        forgetInstanceContext(name);
         // Clean up schedules (scheduler is fleet-level, not lifecycle-level)
         const config = this.fleetConfig?.instances[name];
         if (this.scheduler && config?.topic_id) {
@@ -3476,6 +3562,35 @@ export class FleetManager {
      * So: try, move a bad file aside and retry once with a fresh one, and if even
      * that fails carry on without an event log.
      */
+    /**
+     * Run `agend backend doctor` for the fleet's default backend and return its
+     * cleaned output.
+     *
+     * Async on purpose: this was `execSync` with a 30s timeout, reachable by any
+     * allowlisted user through `/doctor`. While it ran, the entire fleet event loop
+     * was frozen — no IPC, no adapter, no message delivery, no health responses,
+     * and critically no WATCHDOG ping, so a slow doctor could push past
+     * WatchdogSec and have systemd SIGABRT the fleet.
+     */
+    async runBackendDoctor() {
+        const stripAnsi = (s) => s.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
+        const backend = this.fleetConfig?.defaults?.backend || "claude-code";
+        try {
+            const { execFile } = await import("node:child_process");
+            const { promisify } = await import("node:util");
+            // execFile with an argv array — no shell, so the backend name cannot be
+            // interpreted as a command even if config is malformed.
+            const { stdout } = await promisify(execFile)("agend", ["backend", "doctor", backend], {
+                timeout: 30_000,
+                encoding: "utf-8",
+            });
+            return stripAnsi(stdout) || "No output";
+        }
+        catch (err) {
+            const e = err;
+            return stripAnsi(e.stdout ?? e.message ?? "Doctor failed");
+        }
+    }
     /** Drop event/activity rows older than the retention window. Best-effort. */
     pruneEventLog() {
         try {
@@ -5441,10 +5556,20 @@ When users create specialized instances, suggest these configurations:
     // ── Update check ────────────────────────────────────────────────────
     async checkForUpdates() {
         try {
-            const { execSync } = await import("node:child_process");
+            // Both npm lookups are async: as execSync they froze the fleet event loop for
+            // up to 15s each, and on a beta build BOTH ran — 30s with no WATCHDOG ping,
+            // past WatchdogSec's half-interval and enough for systemd to SIGABRT the fleet
+            // for a background version check.
+            const { execFile } = await import("node:child_process");
+            const { promisify } = await import("node:util");
+            const execFileP = promisify(execFile);
+            const npmVersion = async (spec) => {
+                const { stdout } = await execFileP("npm", ["view", spec, "version"], { timeout: 15_000 });
+                return stdout.toString().trim();
+            };
             const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
             const currentVersion = JSON.parse(readFileSync(pkgPath, "utf-8")).version ?? "0.0.0";
-            const latest = execSync("npm view @songsid/agend version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
+            const latest = await npmVersion("@songsid/agend");
             let target = latest;
             if (currentVersion.includes("-beta")) {
                 // Beta users track the @beta channel (never fall back to @latest, which is
@@ -5452,7 +5577,7 @@ When users create specialized instances, suggest these configurations:
                 // of beta/latest is the newest.
                 let beta = "";
                 try {
-                    beta = execSync("npm view @songsid/agend@beta version", { stdio: "pipe", timeout: 15_000 }).toString().trim();
+                    beta = await npmVersion("@songsid/agend@beta");
                 }
                 catch { /* no beta tag */ }
                 target = beta || latest;
@@ -5577,15 +5702,13 @@ When users create specialized instances, suggest these configurations:
                 }
             }
             if (req.method === "GET" && req.url === "/health") {
-                const instanceCount = this.fleetConfig?.instances
-                    ? Object.keys(this.fleetConfig.instances).length
-                    : 0;
-                res.writeHead(200);
-                res.end(JSON.stringify({
-                    status: "ok",
-                    instances: instanceCount,
-                    uptime: Math.floor((Date.now() - this.startedAt) / 1000),
-                }));
+                const health = this.getFleetHealth();
+                // 503 when the fleet cannot do its job, so an external monitor sees it.
+                // This used to always answer 200 "ok" with a count of CONFIGURED instances,
+                // so every agent could be dead and every adapter down and it still looked
+                // green.
+                res.writeHead(health.status === "ok" ? 200 : 503);
+                res.end(JSON.stringify(health));
                 return;
             }
             if (req.method === "GET" && req.url === "/status") {