@songsid/agend 2.1.0-beta.1 → 2.1.0-beta.11

package/dist/fleet-manager.js

@@ -5,13 +5,13 @@ import { join, dirname, basename } from "node:path";
 import { fileURLToPath } from "node:url";
 import { getAgendHome, ensureWorkspaceGit } from "./paths.js";
 import { sdNotify } from "./sd-notify.js";
-import yaml from "js-yaml";
+import { isScalar, parseDocument } from "yaml";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 /** Fallback access policy for a channel with no `access:` block — open (no gate). */
 const DEFAULT_OPEN_ACCESS = { mode: "open", allowed_users: [], max_pending_codes: 0, code_expiry_minutes: 0 };
 import { isProbeableRouteTarget } from "./fleet-context.js";
-import { loadFleetConfig, DEFAULT_COST_GUARD, DEFAULT_DAILY_SUMMARY, DEFAULT_INSTANCE_CONFIG } from "./config.js";
+import { loadFleetConfig, loadRawFleetConfig, DEFAULT_COST_GUARD, DEFAULT_DAILY_SUMMARY, DEFAULT_INSTANCE_CONFIG } from "./config.js";
 import { EventLog } from "./event-log.js";
 import { AdapterWorld } from "./adapter-world.js";
 import { CostGuard, formatCents } from "./cost-guard.js";
@@ -65,6 +65,9 @@ export class FleetManager {
     /** @deprecated Use lifecycle.daemons — kept for backward compat */
     get daemons() { return this.lifecycle.daemons; }
     fleetConfig = null;
+    rawFleetConfig = {};
+    rawFleetDocument = null;
+    savedFleetConfigSnapshot = null;
     adapter = null;
     worlds = new Map();
     adapters = new Map(); // derived view for backward compat
@@ -93,6 +96,8 @@ export class FleetManager {
     // Topic icon + auto-archive state
     topicIcons = {};
     lastActivity = new Map();
+    /** Latest pane-derived execution snapshot reported by each daemon. */
+    instanceStateCache = new Map();
     lastInboundUser = new Map(); // instanceName → last username
     // Active "🛑 Cancel" buttons, tracked per button (keyed by messageId) rather
     // than one-per-instance. A button is retired (deleted, with bounded retry) on
@@ -160,6 +165,7 @@ export class FleetManager {
         const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
             name,
             status: this.getInstanceStatus(name),
+            state: this.getInstanceExecutionState(name),
             ipc: this.instanceIpcClients.has(name),
             costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
             rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
@@ -174,9 +180,21 @@ export class FleetManager {
     }
     /** Load fleet.yaml and build routing table */
     loadConfig(configPath) {
+        this.configPath = configPath;
+        const source = existsSync(configPath) ? readFileSync(configPath, "utf-8") : "{}\n";
+        this.rawFleetDocument = parseDocument(source, { keepSourceTokens: true });
+        if (this.rawFleetDocument.errors.length > 0) {
+            throw new Error(`Invalid fleet.yaml: ${this.rawFleetDocument.errors[0].message}`);
+        }
+        this.rawFleetConfig = loadRawFleetConfig(configPath);
         this.fleetConfig = loadFleetConfig(configPath);
+        this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
         return this.fleetConfig;
     }
+    /** User-authored fleet.yaml, before defaults are merged into instances. */
+    getRawFleetConfig() {
+        return structuredClone(this.rawFleetConfig);
+    }
     /** Build topic routing table: { topicId -> RouteTarget } */
     buildRoutingTable() {
         if (this.fleetConfig) {
@@ -265,17 +283,24 @@ export class FleetManager {
      */
     bindInstanceAdapter(name, adapterId, fromInbound = false) {
         const cfg = this.fleetConfig?.instances[name];
-        if (fromInbound && (cfg?.general_topic || cfg?.channel_id))
-            return;
-        // A classic instance is bound authoritatively at /start. Don't let an inbound
-        // (seen by every same-guild bot) override an existing binding — but if there
-        // is none yet (e.g. after a restart, before v2.1 persistence), allow inbound
-        // to re-establish it so replies don't fall back to the primary bot forever.
-        if (fromInbound && this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
-            return;
+        if (fromInbound) {
+            // Skip inbound-derived binding for any instance that doesn't have an
+            // explicit channel_id — those default to primary adapter deterministically.
+            // This prevents a non-deterministic race where whichever adapter delivers
+            // first after restart wins the binding.
+            if (cfg?.general_topic || cfg?.channel_id)
+                return;
+            if (cfg && !cfg.channel_id)
+                return; // fleet instance without explicit binding → use primary
+            // Classic instance: don't override an existing binding (authoritative from /start)
+            if (this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
+                return;
+        }
         this.instanceWorldBinding.set(name, adapterId);
     }
     getInstanceStatus(name) {
+        if (this.lifecycle.isPaused(name))
+            return "paused";
         const pidPath = join(this.getInstanceDir(name), "daemon.pid");
         if (!existsSync(pidPath))
             return "stopped";
@@ -288,6 +313,35 @@ export class FleetManager {
             return "crashed";
         }
     }
+    getInstanceExecutionState(name) {
+        if (this.lifecycle.isPaused(name))
+            return null;
+        return this.instanceStateCache.get(name)?.state ?? null;
+    }
+    cacheInstanceExecutionState(name, msg) {
+        const state = msg.state;
+        if (state !== "idle" && state !== "working" && state !== "stuck")
+            return;
+        const previous = this.instanceStateCache.get(name);
+        const now = Date.now();
+        const numberOr = (value, fallback) => typeof value === "number" && Number.isFinite(value) ? value : fallback;
+        this.instanceStateCache.set(name, {
+            state,
+            unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
+            observedAt: numberOr(msg.observedAt, now),
+            stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
+        });
+    }
+    /** Single delivery facade: wake an auto-paused CLI before sending to daemon IPC. */
+    async deliverToInstance(instanceName, payload) {
+        if (this.lifecycle.isPaused(instanceName)) {
+            await this.lifecycle.wake(instanceName, 30_000);
+        }
+        const ipc = this.instanceIpcClients.get(instanceName);
+        if (!ipc?.connected)
+            throw new Error(`Instance '${instanceName}' IPC is unavailable`);
+        ipc.send(payload);
+    }
     async startInstance(name, config, topicMode) {
         if (config.general_topic) {
             this.ensureGeneralInstructions(config.working_directory, config.backend);
@@ -355,6 +409,7 @@ export class FleetManager {
     }
     async stopInstance(name) {
         this.failoverActive.delete(name);
+        this.instanceStateCache.delete(name);
         return this.lifecycle.stop(name);
     }
     /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
@@ -1520,10 +1575,16 @@ export class FleetManager {
                 else if (msg.type === "fleet_set_description") {
                     this.handleSetDescription(name, msg);
                 }
+                else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
+                    this.cacheInstanceExecutionState(name, msg);
+                }
             }, this.logger, `ipc.message[${name}]`));
             // Ask daemon for any sessions that registered before we connected
             // (fixes race condition where mcp_ready was broadcast before fleet manager connected)
             ipc.send({ type: "query_sessions" });
+            // The initial state transition may have happened before FleetManager
+            // connected, so seed the cache instead of waiting for another transition.
+            ipc.send({ type: "query_instance_state", requestId: `fleet-state-${Date.now()}` });
             this.logger.debug({ name }, "Connected to instance IPC");
             if (!this.statuslineWatcher.has(name)) {
                 this.statuslineWatcher.watch(name);
@@ -1926,9 +1987,8 @@ export class FleetManager {
                 }
                 this.warnIfRateLimited(generalInstance, msg);
                 const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
-                const ipc = this.instanceIpcClients.get(generalInstance);
-                if (ipc) {
-                    ipc.send({
+                try {
+                    await this.deliverToInstance(generalInstance, {
                         type: "fleet_inbound",
                         content: text,
                         targetSession: generalInstance,
@@ -1954,6 +2014,9 @@ export class FleetManager {
                     this.trackInboundMsg(generalInstance, msg);
                     void this.sendCancelButton(generalInstance);
                 }
+                catch (err) {
+                    this.logger.warn({ err: err.message, instanceName: generalInstance }, "General wake/delivery failed");
+                }
             }
             return;
         }
@@ -2016,27 +2079,32 @@ export class FleetManager {
         this.setTopicIcon(instanceName, "blue");
         this.warnIfRateLimited(instanceName, msg);
         const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
-        const ipc = this.instanceIpcClients.get(instanceName);
-        if (!ipc) {
-            this.logger.warn({ instanceName }, "No IPC connection to instance");
+        try {
+            await this.deliverToInstance(instanceName, {
+                type: "fleet_inbound",
+                content: text,
+                targetSession: instanceName, // Channel messages → instance's own session
+                meta: {
+                    chat_id: msg.chatId,
+                    message_id: msg.messageId,
+                    user: msg.username,
+                    user_id: msg.userId,
+                    ts: msg.timestamp.toISOString(),
+                    thread_id: msg.threadId ?? "",
+                    source: msg.source,
+                    ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
+                    ...extraMeta,
+                },
+            });
+        }
+        catch (err) {
+            this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
+            if (msg.chatId && msg.messageId) {
+                const reactAdapter = this.getAdapterForInstance(instanceName) ?? inboundAdapter;
+                reactAdapter.react(this.reactTarget(msg), msg.messageId, "❌").catch(() => { });
+            }
             return;
         }
-        ipc.send({
-            type: "fleet_inbound",
-            content: text,
-            targetSession: instanceName, // Channel messages → instance's own session
-            meta: {
-                chat_id: msg.chatId,
-                message_id: msg.messageId,
-                user: msg.username,
-                user_id: msg.userId,
-                ts: msg.timestamp.toISOString(),
-                thread_id: msg.threadId ?? "",
-                source: msg.source,
-                ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
-                ...extraMeta,
-            },
-        });
         this.lastInboundUser.set(instanceName, msg.username);
         this.logger.info(`${msg.username} → ${instanceName}: ${(text ?? "").slice(0, 100)}`);
         this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), instanceName);
@@ -2197,20 +2265,23 @@ export class FleetManager {
         const schedulerDefaults = this.fleetConfig?.defaults.scheduler;
         const retryCount = schedulerDefaults?.retry_count ?? 3;
         const retryInterval = schedulerDefaults?.retry_interval_ms ?? 30_000;
-        const deliver = () => {
-            const ipc = this.instanceIpcClients.get(target);
-            if (!ipc?.connected)
+        const deliver = async () => {
+            try {
+                await this.deliverToInstance(target, {
+                    type: "fleet_schedule_trigger",
+                    payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
+                    meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
+                });
+                // A scheduled trigger also puts the instance to work — show a cancel button.
+                void this.sendCancelButton(target);
+                return true;
+            }
+            catch (err) {
+                this.logger.warn({ err: err.message, target }, "Scheduled wake/delivery attempt failed");
                 return false;
-            ipc.send({
-                type: "fleet_schedule_trigger",
-                payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
-                meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
-            });
-            // A scheduled trigger also puts the instance to work — show a cancel button.
-            void this.sendCancelButton(target);
-            return true;
+            }
         };
-        if (deliver()) {
+        if (await deliver()) {
             this.scheduler.recordRun(id, "delivered");
             if (source !== target)
                 this.notifySourceTopic(schedule);
@@ -2218,7 +2289,7 @@ export class FleetManager {
         }
         for (let i = 0; i < retryCount; i++) {
             await new Promise((r) => setTimeout(r, retryInterval));
-            if (deliver()) {
+            if (await deliver()) {
                 this.scheduler.recordRun(id, "delivered");
                 if (source !== target)
                     this.notifySourceTopic(schedule);
@@ -2642,73 +2713,80 @@ export class FleetManager {
             }
         }, 5 * 60_000);
     }
-    /** Save fleet config back to fleet.yaml */
+    /**
+     * Patch only values changed in the effective config into the original YAML
+     * document. Unknown keys, explicit overrides and comments remain untouched.
+     */
     saveFleetConfig() {
         if (!this.fleetConfig || !this.configPath)
             return;
-        const toSave = {};
-        if (this.fleetConfig.project_roots)
-            toSave.project_roots = this.fleetConfig.project_roots;
-        if (this.fleetConfig.channels && this.fleetConfig.channels.length > 0) {
-            toSave.channels = this.fleetConfig.channels;
-        }
-        else if (this.fleetConfig.channel) {
-            toSave.channel = this.fleetConfig.channel;
-        }
-        if (this.fleetConfig.health_port)
-            toSave.health_port = this.fleetConfig.health_port;
-        if (Object.keys(this.fleetConfig.defaults).length > 0)
-            toSave.defaults = this.fleetConfig.defaults;
-        if (this.fleetConfig.teams && Object.keys(this.fleetConfig.teams).length > 0) {
-            toSave.teams = this.fleetConfig.teams;
-        }
-        if (this.fleetConfig.templates && Object.keys(this.fleetConfig.templates).length > 0) {
-            toSave.templates = this.fleetConfig.templates;
-        }
-        if (this.fleetConfig.profiles && Object.keys(this.fleetConfig.profiles).length > 0) {
-            toSave.profiles = this.fleetConfig.profiles;
-        }
-        toSave.instances = {};
-        for (const [name, inst] of Object.entries(this.fleetConfig.instances)) {
-            const serialized = {
-                working_directory: inst.working_directory,
-                topic_id: inst.topic_id,
-            };
-            // Preserve all optional user-configured fields so saveFleetConfig() never silently drops them
-            if (inst.general_topic)
-                serialized.general_topic = true;
-            if (inst.display_name)
-                serialized.display_name = inst.display_name;
-            if (inst.channel_id)
-                serialized.channel_id = inst.channel_id;
-            if (inst.description)
-                serialized.description = inst.description;
-            if (inst.tags?.length)
-                serialized.tags = inst.tags;
-            if (inst.model)
-                serialized.model = inst.model;
-            if (inst.model_failover?.length)
-                serialized.model_failover = inst.model_failover;
-            if (inst.worktree_source)
-                serialized.worktree_source = inst.worktree_source;
-            if (inst.backend)
-                serialized.backend = inst.backend;
-            if (inst.systemPrompt)
-                serialized.systemPrompt = inst.systemPrompt;
-            if (inst.skipPermissions)
-                serialized.skipPermissions = inst.skipPermissions;
-            if (inst.lightweight)
-                serialized.lightweight = inst.lightweight;
-            if (inst.cost_guard)
-                serialized.cost_guard = inst.cost_guard;
-            if (inst.workflow !== undefined)
-                serialized.workflow = inst.workflow;
-            if (inst.agent_mode)
-                serialized.agent_mode = inst.agent_mode;
-            toSave.instances[name] = serialized;
-        }
-        writeFileSync(this.configPath, yaml.dump(toSave, { lineWidth: 120 }));
-        this.logger.info({ path: this.configPath }, "Saved fleet config");
+        if (!this.savedFleetConfigSnapshot)
+            this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
+        // Re-read immediately before patching so an unrelated concurrent/manual
+        // edit is retained. Invalid concurrent YAML is never overwritten.
+        const source = existsSync(this.configPath) ? readFileSync(this.configPath, "utf-8") : "{}\n";
+        this.rawFleetDocument = parseDocument(source, { keepSourceTokens: true });
+        if (this.rawFleetDocument.errors.length > 0) {
+            throw new Error(`Refusing to overwrite invalid fleet.yaml: ${this.rawFleetDocument.errors[0].message}`);
+        }
+        this.rawFleetConfig = loadRawFleetConfig(this.configPath);
+        this.patchFleetDocument(this.rawFleetDocument, [], this.savedFleetConfigSnapshot, this.fleetConfig);
+        const output = String(this.rawFleetDocument);
+        const tempPath = `${this.configPath}.tmp-${process.pid}`;
+        writeFileSync(tempPath, output, "utf-8");
+        if (existsSync(this.configPath))
+            chmodSync(tempPath, statSync(this.configPath).mode);
+        renameSync(tempPath, this.configPath);
+        this.rawFleetConfig = loadRawFleetConfig(this.configPath);
+        this.savedFleetConfigSnapshot = structuredClone(this.fleetConfig);
+        this.logger.info({ path: this.configPath }, "Saved fleet config (lossless patch)");
+    }
+    patchFleetDocument(document, path, before, after) {
+        if (Object.is(before, after))
+            return;
+        if (Array.isArray(before) && Array.isArray(after)) {
+            const shared = Math.min(before.length, after.length);
+            for (let i = 0; i < shared; i++) {
+                this.patchFleetDocument(document, [...path, i], before[i], after[i]);
+            }
+            // Remove from the end so YAML sequence indexes do not shift underneath us.
+            for (let i = before.length - 1; i >= after.length; i--)
+                document.deleteIn([...path, i]);
+            for (let i = shared; i < after.length; i++)
+                document.setIn([...path, i], after[i]);
+            return;
+        }
+        const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+        if (isRecord(before) && isRecord(after)) {
+            const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
+            for (const key of keys) {
+                // `channel` is a derived alias when the raw file uses `channels`.
+                if (path.length === 0 && key === "channel" && this.rawFleetConfig.channels)
+                    continue;
+                // Conversely, `channels` is a normalized alias for a legacy `channel`.
+                // Keep the user's original shape unless the caller explicitly removed
+                // `channel` (the Settings channels endpoint intentionally migrates it).
+                if (path.length === 0 && key === "channels" && this.rawFleetConfig.channel && !this.rawFleetConfig.channels && after.channel !== undefined)
+                    continue;
+                this.patchFleetDocument(document, [...path, key], before[key], after[key]);
+            }
+            return;
+        }
+        if (after === undefined) {
+            document.deleteIn(path);
+        }
+        else if (path.length === 0) {
+            document.contents = document.createNode(after);
+        }
+        else {
+            const currentNode = document.getIn(path, true);
+            if (isScalar(currentNode) && (after === null || typeof after !== "object")) {
+                currentNode.value = after;
+            }
+            else {
+                document.setIn(path, after);
+            }
+        }
     }
     async removeInstance(name) {
         // Clean up schedules (scheduler is fleet-level, not lifecycle-level)
@@ -2751,6 +2829,10 @@ export class FleetManager {
     startStatuslineWatcher(name) {
         this.statuslineWatcher.watch(name);
     }
+    stopStatuslineWatcher(name) {
+        // Pausing stops I/O but retains the last observed limits for status views.
+        this.statuslineWatcher.unwatch(name, true);
+    }
     reactMessageStatus(instanceName, chatId, messageId, emoji) {
         // React via the adapter BOUND to this instance — NOT the first discord world.
         // Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
@@ -3101,7 +3183,7 @@ export class FleetManager {
             return [];
         }
     }
-    async sendHangNotification(instanceName) {
+    async sendHangNotification(instanceName, unchangedForMs) {
         const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
         if (!adapter)
             return;
@@ -3110,11 +3192,18 @@ export class FleetManager {
         if (!groupId)
             return;
         const threadId = this.fleetConfig?.instances[instanceName]?.topic_id;
+        const instanceHangConfig = this.fleetConfig?.instances[instanceName]?.hang_detector;
+        const configuredMinutes = instanceHangConfig?.timeout_minutes
+            ?? this.fleetConfig?.defaults?.hang_detector?.timeout_minutes
+            ?? 15;
+        const unchangedMinutes = unchangedForMs == null
+            ? configuredMinutes
+            : Math.max(1, Math.floor(unchangedForMs / 60_000));
         this.setTopicIcon(instanceName, "red");
         await adapter.notifyAlert(String(groupId), {
             type: "hang",
             instanceName,
-            message: `⚠️ ${instanceName} appears hung (no activity for 15+ minutes)`,
+            message: `⚠️ ${instanceName} may be stuck — pane unchanged for ${unchangedMinutes}min, ready prompt not recognized`,
             choices: [
                 { id: `hang:restart:${instanceName}`, label: "🔄 Force restart" },
                 { id: `hang:wait:${instanceName}`, label: "⏳ Keep waiting" },
@@ -3615,11 +3704,6 @@ When users create specialized instances, suggest these configurations:
         const fullText = logContext
             ? `[Chat log for context]\n${logContext}\n\n[User message]\n${text}`
             : text;
-        const ipc = this.instanceIpcClients.get(instanceName);
-        if (!ipc) {
-            this.logger.warn({ instanceName }, "Classic channel instance IPC not connected");
-            return;
-        }
         const meta = {
             chat_id: msg.chatId,
             message_id: msg.messageId,
@@ -3641,12 +3725,18 @@ When users create specialized instances, suggest these configurations:
                 meta.image_path = saves[saves.length - 1][1].split(",")[0].trim();
             }
         }
-        ipc.send({
-            type: "fleet_inbound",
-            content: fullText,
-            targetSession: instanceName,
-            meta,
-        });
+        try {
+            await this.deliverToInstance(instanceName, {
+                type: "fleet_inbound",
+                content: fullText,
+                targetSession: instanceName,
+                meta,
+            });
+        }
+        catch (err) {
+            this.logger.warn({ err: err.message, instanceName }, "Classic wake/delivery failed");
+            return;
+        }
         this.lastInboundUser.set(instanceName, msg.username);
         this.logger.info(`${msg.username} → ${instanceName} (classic): ${text.slice(0, 100)}`);
         this.trackInboundMsg(instanceName, msg);
@@ -4326,6 +4416,7 @@ When users create specialized instances, suggest these configurations:
                         lastActivity: this.lastActivityMs(inst.name) || null,
                         currentTask,
                         idle: this.getInstanceIdle(inst.name),
+                        state: this.getInstanceExecutionState(inst.name),
                     };
                 });
                 res.setHeader("Access-Control-Allow-Origin", "*");