@songsid/agend 2.1.0-beta.1 → 2.1.0-beta.10

package/dist/fleet-manager.js

@@ -93,6 +93,8 @@ export class FleetManager {
     // Topic icon + auto-archive state
     topicIcons = {};
     lastActivity = new Map();
+    /** Latest pane-derived execution snapshot reported by each daemon. */
+    instanceStateCache = new Map();
     lastInboundUser = new Map(); // instanceName → last username
     // Active "🛑 Cancel" buttons, tracked per button (keyed by messageId) rather
     // than one-per-instance. A button is retired (deleted, with bounded retry) on
@@ -160,6 +162,7 @@ export class FleetManager {
         const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
             name,
             status: this.getInstanceStatus(name),
+            state: this.getInstanceExecutionState(name),
             ipc: this.instanceIpcClients.has(name),
             costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
             rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
@@ -265,17 +268,24 @@ export class FleetManager {
      */
     bindInstanceAdapter(name, adapterId, fromInbound = false) {
         const cfg = this.fleetConfig?.instances[name];
-        if (fromInbound && (cfg?.general_topic || cfg?.channel_id))
-            return;
-        // A classic instance is bound authoritatively at /start. Don't let an inbound
-        // (seen by every same-guild bot) override an existing binding — but if there
-        // is none yet (e.g. after a restart, before v2.1 persistence), allow inbound
-        // to re-establish it so replies don't fall back to the primary bot forever.
-        if (fromInbound && this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
-            return;
+        if (fromInbound) {
+            // Skip inbound-derived binding for any instance that doesn't have an
+            // explicit channel_id — those default to primary adapter deterministically.
+            // This prevents a non-deterministic race where whichever adapter delivers
+            // first after restart wins the binding.
+            if (cfg?.general_topic || cfg?.channel_id)
+                return;
+            if (cfg && !cfg.channel_id)
+                return; // fleet instance without explicit binding → use primary
+            // Classic instance: don't override an existing binding (authoritative from /start)
+            if (this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
+                return;
+        }
         this.instanceWorldBinding.set(name, adapterId);
     }
     getInstanceStatus(name) {
+        if (this.lifecycle.isPaused(name))
+            return "paused";
         const pidPath = join(this.getInstanceDir(name), "daemon.pid");
         if (!existsSync(pidPath))
             return "stopped";
@@ -288,6 +298,35 @@ export class FleetManager {
             return "crashed";
         }
     }
+    getInstanceExecutionState(name) {
+        if (this.lifecycle.isPaused(name))
+            return null;
+        return this.instanceStateCache.get(name)?.state ?? null;
+    }
+    cacheInstanceExecutionState(name, msg) {
+        const state = msg.state;
+        if (state !== "idle" && state !== "working" && state !== "stuck")
+            return;
+        const previous = this.instanceStateCache.get(name);
+        const now = Date.now();
+        const numberOr = (value, fallback) => typeof value === "number" && Number.isFinite(value) ? value : fallback;
+        this.instanceStateCache.set(name, {
+            state,
+            unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
+            observedAt: numberOr(msg.observedAt, now),
+            stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
+        });
+    }
+    /** Single delivery facade: wake an auto-paused CLI before sending to daemon IPC. */
+    async deliverToInstance(instanceName, payload) {
+        if (this.lifecycle.isPaused(instanceName)) {
+            await this.lifecycle.wake(instanceName, 30_000);
+        }
+        const ipc = this.instanceIpcClients.get(instanceName);
+        if (!ipc?.connected)
+            throw new Error(`Instance '${instanceName}' IPC is unavailable`);
+        ipc.send(payload);
+    }
     async startInstance(name, config, topicMode) {
         if (config.general_topic) {
             this.ensureGeneralInstructions(config.working_directory, config.backend);
@@ -355,6 +394,7 @@ export class FleetManager {
     }
     async stopInstance(name) {
         this.failoverActive.delete(name);
+        this.instanceStateCache.delete(name);
         return this.lifecycle.stop(name);
     }
     /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
@@ -1520,10 +1560,16 @@ export class FleetManager {
                 else if (msg.type === "fleet_set_description") {
                     this.handleSetDescription(name, msg);
                 }
+                else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
+                    this.cacheInstanceExecutionState(name, msg);
+                }
             }, this.logger, `ipc.message[${name}]`));
             // Ask daemon for any sessions that registered before we connected
             // (fixes race condition where mcp_ready was broadcast before fleet manager connected)
             ipc.send({ type: "query_sessions" });
+            // The initial state transition may have happened before FleetManager
+            // connected, so seed the cache instead of waiting for another transition.
+            ipc.send({ type: "query_instance_state", requestId: `fleet-state-${Date.now()}` });
             this.logger.debug({ name }, "Connected to instance IPC");
             if (!this.statuslineWatcher.has(name)) {
                 this.statuslineWatcher.watch(name);
@@ -1926,9 +1972,8 @@ export class FleetManager {
                 }
                 this.warnIfRateLimited(generalInstance, msg);
                 const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
-                const ipc = this.instanceIpcClients.get(generalInstance);
-                if (ipc) {
-                    ipc.send({
+                try {
+                    await this.deliverToInstance(generalInstance, {
                         type: "fleet_inbound",
                         content: text,
                         targetSession: generalInstance,
@@ -1954,6 +1999,9 @@ export class FleetManager {
                     this.trackInboundMsg(generalInstance, msg);
                     void this.sendCancelButton(generalInstance);
                 }
+                catch (err) {
+                    this.logger.warn({ err: err.message, instanceName: generalInstance }, "General wake/delivery failed");
+                }
             }
             return;
         }
@@ -2016,27 +2064,32 @@ export class FleetManager {
         this.setTopicIcon(instanceName, "blue");
         this.warnIfRateLimited(instanceName, msg);
         const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
-        const ipc = this.instanceIpcClients.get(instanceName);
-        if (!ipc) {
-            this.logger.warn({ instanceName }, "No IPC connection to instance");
+        try {
+            await this.deliverToInstance(instanceName, {
+                type: "fleet_inbound",
+                content: text,
+                targetSession: instanceName, // Channel messages → instance's own session
+                meta: {
+                    chat_id: msg.chatId,
+                    message_id: msg.messageId,
+                    user: msg.username,
+                    user_id: msg.userId,
+                    ts: msg.timestamp.toISOString(),
+                    thread_id: msg.threadId ?? "",
+                    source: msg.source,
+                    ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
+                    ...extraMeta,
+                },
+            });
+        }
+        catch (err) {
+            this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
+            if (msg.chatId && msg.messageId) {
+                const reactAdapter = this.getAdapterForInstance(instanceName) ?? inboundAdapter;
+                reactAdapter.react(this.reactTarget(msg), msg.messageId, "❌").catch(() => { });
+            }
             return;
         }
-        ipc.send({
-            type: "fleet_inbound",
-            content: text,
-            targetSession: instanceName, // Channel messages → instance's own session
-            meta: {
-                chat_id: msg.chatId,
-                message_id: msg.messageId,
-                user: msg.username,
-                user_id: msg.userId,
-                ts: msg.timestamp.toISOString(),
-                thread_id: msg.threadId ?? "",
-                source: msg.source,
-                ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
-                ...extraMeta,
-            },
-        });
         this.lastInboundUser.set(instanceName, msg.username);
         this.logger.info(`${msg.username} → ${instanceName}: ${(text ?? "").slice(0, 100)}`);
         this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), instanceName);
@@ -2197,20 +2250,23 @@ export class FleetManager {
         const schedulerDefaults = this.fleetConfig?.defaults.scheduler;
         const retryCount = schedulerDefaults?.retry_count ?? 3;
         const retryInterval = schedulerDefaults?.retry_interval_ms ?? 30_000;
-        const deliver = () => {
-            const ipc = this.instanceIpcClients.get(target);
-            if (!ipc?.connected)
+        const deliver = async () => {
+            try {
+                await this.deliverToInstance(target, {
+                    type: "fleet_schedule_trigger",
+                    payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
+                    meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
+                });
+                // A scheduled trigger also puts the instance to work — show a cancel button.
+                void this.sendCancelButton(target);
+                return true;
+            }
+            catch (err) {
+                this.logger.warn({ err: err.message, target }, "Scheduled wake/delivery attempt failed");
                 return false;
-            ipc.send({
-                type: "fleet_schedule_trigger",
-                payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
-                meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
-            });
-            // A scheduled trigger also puts the instance to work — show a cancel button.
-            void this.sendCancelButton(target);
-            return true;
+            }
         };
-        if (deliver()) {
+        if (await deliver()) {
             this.scheduler.recordRun(id, "delivered");
             if (source !== target)
                 this.notifySourceTopic(schedule);
@@ -2218,7 +2274,7 @@ export class FleetManager {
         }
         for (let i = 0; i < retryCount; i++) {
             await new Promise((r) => setTimeout(r, retryInterval));
-            if (deliver()) {
+            if (await deliver()) {
                 this.scheduler.recordRun(id, "delivered");
                 if (source !== target)
                     this.notifySourceTopic(schedule);
@@ -2751,6 +2807,10 @@ export class FleetManager {
     startStatuslineWatcher(name) {
         this.statuslineWatcher.watch(name);
     }
+    stopStatuslineWatcher(name) {
+        // Pausing stops I/O but retains the last observed limits for status views.
+        this.statuslineWatcher.unwatch(name, true);
+    }
     reactMessageStatus(instanceName, chatId, messageId, emoji) {
         // React via the adapter BOUND to this instance — NOT the first discord world.
         // Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
@@ -3101,7 +3161,7 @@ export class FleetManager {
             return [];
         }
     }
-    async sendHangNotification(instanceName) {
+    async sendHangNotification(instanceName, unchangedForMs) {
         const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
         if (!adapter)
             return;
@@ -3110,11 +3170,18 @@ export class FleetManager {
         if (!groupId)
             return;
         const threadId = this.fleetConfig?.instances[instanceName]?.topic_id;
+        const instanceHangConfig = this.fleetConfig?.instances[instanceName]?.hang_detector;
+        const configuredMinutes = instanceHangConfig?.timeout_minutes
+            ?? this.fleetConfig?.defaults?.hang_detector?.timeout_minutes
+            ?? 15;
+        const unchangedMinutes = unchangedForMs == null
+            ? configuredMinutes
+            : Math.max(1, Math.floor(unchangedForMs / 60_000));
         this.setTopicIcon(instanceName, "red");
         await adapter.notifyAlert(String(groupId), {
             type: "hang",
             instanceName,
-            message: `⚠️ ${instanceName} appears hung (no activity for 15+ minutes)`,
+            message: `⚠️ ${instanceName} may be stuck — pane unchanged for ${unchangedMinutes}min, ready prompt not recognized`,
             choices: [
                 { id: `hang:restart:${instanceName}`, label: "🔄 Force restart" },
                 { id: `hang:wait:${instanceName}`, label: "⏳ Keep waiting" },
@@ -3615,11 +3682,6 @@ When users create specialized instances, suggest these configurations:
         const fullText = logContext
             ? `[Chat log for context]\n${logContext}\n\n[User message]\n${text}`
             : text;
-        const ipc = this.instanceIpcClients.get(instanceName);
-        if (!ipc) {
-            this.logger.warn({ instanceName }, "Classic channel instance IPC not connected");
-            return;
-        }
         const meta = {
             chat_id: msg.chatId,
             message_id: msg.messageId,
@@ -3641,12 +3703,18 @@ When users create specialized instances, suggest these configurations:
                 meta.image_path = saves[saves.length - 1][1].split(",")[0].trim();
             }
         }
-        ipc.send({
-            type: "fleet_inbound",
-            content: fullText,
-            targetSession: instanceName,
-            meta,
-        });
+        try {
+            await this.deliverToInstance(instanceName, {
+                type: "fleet_inbound",
+                content: fullText,
+                targetSession: instanceName,
+                meta,
+            });
+        }
+        catch (err) {
+            this.logger.warn({ err: err.message, instanceName }, "Classic wake/delivery failed");
+            return;
+        }
         this.lastInboundUser.set(instanceName, msg.username);
         this.logger.info(`${msg.username} → ${instanceName} (classic): ${text.slice(0, 100)}`);
         this.trackInboundMsg(instanceName, msg);
@@ -4326,6 +4394,7 @@ When users create specialized instances, suggest these configurations:
                         lastActivity: this.lastActivityMs(inst.name) || null,
                         currentTask,
                         idle: this.getInstanceIdle(inst.name),
+                        state: this.getInstanceExecutionState(inst.name),
                     };
                 });
                 res.setHeader("Access-Control-Allow-Origin", "*");