@songsid/agend 2.0.12 → 2.1.0-beta.10

package/dist/fleet-manager.js

@@ -93,6 +93,8 @@ export class FleetManager {
     // Topic icon + auto-archive state
     topicIcons = {};
     lastActivity = new Map();
+    /** Latest pane-derived execution snapshot reported by each daemon. */
+    instanceStateCache = new Map();
     lastInboundUser = new Map(); // instanceName → last username
     // Active "🛑 Cancel" buttons, tracked per button (keyed by messageId) rather
     // than one-per-instance. A button is retired (deleted, with bounded retry) on
@@ -110,6 +112,8 @@ export class FleetManager {
     ipcStoppingInstances = new Set();
     // Adapter restart: prevents re-entrant restart attempts
     adapterRestarting = new Set();
+    // Adapter isolation: track state per adapter for retry + visibility
+    adapterState = new Map();
     collabInstances = new Set();
     // Health endpoint
     healthServer = null;
@@ -158,6 +162,7 @@ export class FleetManager {
         const instances = Object.keys(this.fleetConfig?.instances ?? {}).map(name => ({
             name,
             status: this.getInstanceStatus(name),
+            state: this.getInstanceExecutionState(name),
             ipc: this.instanceIpcClients.has(name),
             costCents: this.costGuard?.getDailyCostCents(name) ?? 0,
             rateLimits: this.statuslineWatcher.getRateLimits(name) ?? null,
@@ -263,17 +268,24 @@ export class FleetManager {
      */
     bindInstanceAdapter(name, adapterId, fromInbound = false) {
         const cfg = this.fleetConfig?.instances[name];
-        if (fromInbound && (cfg?.general_topic || cfg?.channel_id))
-            return;
-        // A classic instance is bound authoritatively at /start. Don't let an inbound
-        // (seen by every same-guild bot) override an existing binding — but if there
-        // is none yet (e.g. after a restart, before v2.1 persistence), allow inbound
-        // to re-establish it so replies don't fall back to the primary bot forever.
-        if (fromInbound && this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
-            return;
+        if (fromInbound) {
+            // Skip inbound-derived binding for any instance that doesn't have an
+            // explicit channel_id — those default to primary adapter deterministically.
+            // This prevents a non-deterministic race where whichever adapter delivers
+            // first after restart wins the binding.
+            if (cfg?.general_topic || cfg?.channel_id)
+                return;
+            if (cfg && !cfg.channel_id)
+                return; // fleet instance without explicit binding → use primary
+            // Classic instance: don't override an existing binding (authoritative from /start)
+            if (this.classicChannels?.getChannelIdByInstance(name) !== undefined && this.instanceWorldBinding.has(name))
+                return;
+        }
         this.instanceWorldBinding.set(name, adapterId);
     }
     getInstanceStatus(name) {
+        if (this.lifecycle.isPaused(name))
+            return "paused";
         const pidPath = join(this.getInstanceDir(name), "daemon.pid");
         if (!existsSync(pidPath))
             return "stopped";
@@ -286,6 +298,35 @@ export class FleetManager {
             return "crashed";
         }
     }
+    getInstanceExecutionState(name) {
+        if (this.lifecycle.isPaused(name))
+            return null;
+        return this.instanceStateCache.get(name)?.state ?? null;
+    }
+    cacheInstanceExecutionState(name, msg) {
+        const state = msg.state;
+        if (state !== "idle" && state !== "working" && state !== "stuck")
+            return;
+        const previous = this.instanceStateCache.get(name);
+        const now = Date.now();
+        const numberOr = (value, fallback) => typeof value === "number" && Number.isFinite(value) ? value : fallback;
+        this.instanceStateCache.set(name, {
+            state,
+            unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
+            observedAt: numberOr(msg.observedAt, now),
+            stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
+        });
+    }
+    /** Single delivery facade: wake an auto-paused CLI before sending to daemon IPC. */
+    async deliverToInstance(instanceName, payload) {
+        if (this.lifecycle.isPaused(instanceName)) {
+            await this.lifecycle.wake(instanceName, 30_000);
+        }
+        const ipc = this.instanceIpcClients.get(instanceName);
+        if (!ipc?.connected)
+            throw new Error(`Instance '${instanceName}' IPC is unavailable`);
+        ipc.send(payload);
+    }
     async startInstance(name, config, topicMode) {
         if (config.general_topic) {
             this.ensureGeneralInstructions(config.working_directory, config.backend);
@@ -353,6 +394,7 @@ export class FleetManager {
     }
     async stopInstance(name) {
         this.failoverActive.delete(name);
+        this.instanceStateCache.delete(name);
         return this.lifecycle.stop(name);
     }
     /** Restart a single instance, reloading fleet.yaml first to pick up config changes. */
@@ -836,17 +878,81 @@ export class FleetManager {
         const channelConfigs = fleet.channels ?? (fleet.channel ? [fleet.channel] : []);
         if (channelConfigs.length === 0)
             return;
-        // Start primary adapter (first channel) — this.adapter for backward compat.
-        await this.startSingleAdapter(fleet, channelConfigs[0]);
-        // Start additional adapters. Every bot registers its own slash commands —
-        // Discord slash commands are per-application, so a same-guild secondary bot's
-        // /start etc. are distinct entries (labelled with the bot name) and the
-        // interaction only reaches the invoked bot. This is how ClassicBot supports
-        // multiple bots in one guild: the user picks which bot from autocomplete.
-        for (let i = 1; i < channelConfigs.length; i++) {
-            await this.startAdditionalAdapter(channelConfigs[i]);
+        // Start ALL adapters in parallel — any single failure doesn't block others.
+        const results = await Promise.allSettled(channelConfigs.map((cfg, i) => i === 0
+            ? this.startSingleAdapter(fleet, cfg)
+            : this.startAdditionalAdapter(cfg)));
+        // Track state + schedule background retry for failures.
+        for (let i = 0; i < channelConfigs.length; i++) {
+            const adapterId = channelConfigs[i].id ?? channelConfigs[i].type;
+            if (results[i].status === "fulfilled") {
+                this.adapterState.set(adapterId, { status: "connected", retryCount: 0 });
+            }
+            else {
+                const err = results[i].reason;
+                this.logger.error({ adapterId, err: err?.message ?? err }, "Adapter startup failed — scheduling background retry");
+                this.adapterState.set(adapterId, { status: "retrying", retryCount: 0, lastError: err?.message ?? String(err) });
+                this.scheduleAdapterRetry(adapterId, channelConfigs[i], i === 0 ? fleet : undefined);
+                // Notify admin via whichever adapter is already up
+                this.notifyAdapterFailure(adapterId, err?.message ?? String(err));
+            }
         }
     }
+    /** Exponential backoff retry for a single failed adapter (background, non-blocking). */
+    scheduleAdapterRetry(adapterId, channelConfig, fleet) {
+        const MAX_RETRIES = 10;
+        const INITIAL_DELAY_MS = 5_000;
+        const MAX_DELAY_MS = 5 * 60_000;
+        const state = this.adapterState.get(adapterId);
+        if (!state || state.retryCount >= MAX_RETRIES) {
+            if (state) {
+                state.status = "failed";
+                this.logger.error({ adapterId, retries: state.retryCount }, "Adapter retry exhausted — giving up");
+                this.notifyAdapterFailure(adapterId, `Retry exhausted after ${state.retryCount} attempts. Check token/network and restart fleet.`);
+            }
+            return;
+        }
+        const delay = Math.min(INITIAL_DELAY_MS * Math.pow(2, state.retryCount), MAX_DELAY_MS);
+        this.logger.info({ adapterId, attempt: state.retryCount + 1, delay_ms: delay }, "Scheduling adapter retry");
+        state.retryTimer = setTimeout(async () => {
+            state.retryCount++;
+            try {
+                if (fleet) {
+                    await this.startSingleAdapter(fleet, channelConfig);
+                }
+                else {
+                    await this.startAdditionalAdapter(channelConfig);
+                }
+                state.status = "connected";
+                state.lastError = undefined;
+                this.logger.info({ adapterId, attempts: state.retryCount }, "Adapter reconnected on retry");
+                this.notifyAdapterRecovery(adapterId, state.retryCount);
+            }
+            catch (err) {
+                state.lastError = err?.message ?? String(err);
+                this.logger.warn({ adapterId, attempt: state.retryCount, err: state.lastError }, "Adapter retry failed");
+                this.scheduleAdapterRetry(adapterId, channelConfig, fleet);
+            }
+        }, delay);
+    }
+    /** Notify admin about adapter failure (uses any available adapter). */
+    notifyAdapterFailure(adapterId, error) {
+        const generalId = this.findGeneralInstance();
+        if (generalId) {
+            this.notifyInstanceTopic(generalId, `⚠️ Adapter "${adapterId}" failed to start: ${error}\nRetrying in background. Other adapters unaffected.`);
+        }
+    }
+    /** Notify admin that a retried adapter reconnected. */
+    notifyAdapterRecovery(adapterId, attempts) {
+        const generalId = this.findGeneralInstance();
+        if (generalId) {
+            this.notifyInstanceTopic(generalId, `✅ Adapter "${adapterId}" reconnected (after ${attempts} ${attempts === 1 ? "retry" : "retries"}).`);
+        }
+    }
+    /** Get adapter states for /status visibility. */
+    getAdapterStates() {
+        return this.adapterState;
+    }
     /** Start the primary adapter (backward-compatible, sets this.adapter) */
     async startSingleAdapter(fleet, channelConfig) {
         const botToken = process.env[channelConfig.bot_token_env];
@@ -1454,10 +1560,16 @@ export class FleetManager {
                 else if (msg.type === "fleet_set_description") {
                     this.handleSetDescription(name, msg);
                 }
+                else if (msg.type === "instance_state" || msg.type === "instance_state_response") {
+                    this.cacheInstanceExecutionState(name, msg);
+                }
             }, this.logger, `ipc.message[${name}]`));
             // Ask daemon for any sessions that registered before we connected
             // (fixes race condition where mcp_ready was broadcast before fleet manager connected)
             ipc.send({ type: "query_sessions" });
+            // The initial state transition may have happened before FleetManager
+            // connected, so seed the cache instead of waiting for another transition.
+            ipc.send({ type: "query_instance_state", requestId: `fleet-state-${Date.now()}` });
             this.logger.debug({ name }, "Connected to instance IPC");
             if (!this.statuslineWatcher.has(name)) {
                 this.statuslineWatcher.watch(name);
@@ -1860,9 +1972,8 @@ export class FleetManager {
                 }
                 this.warnIfRateLimited(generalInstance, msg);
                 const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, generalInstance);
-                const ipc = this.instanceIpcClients.get(generalInstance);
-                if (ipc) {
-                    ipc.send({
+                try {
+                    await this.deliverToInstance(generalInstance, {
                         type: "fleet_inbound",
                         content: text,
                         targetSession: generalInstance,
@@ -1888,6 +1999,9 @@ export class FleetManager {
                     this.trackInboundMsg(generalInstance, msg);
                     void this.sendCancelButton(generalInstance);
                 }
+                catch (err) {
+                    this.logger.warn({ err: err.message, instanceName: generalInstance }, "General wake/delivery failed");
+                }
             }
             return;
         }
@@ -1950,27 +2064,32 @@ export class FleetManager {
         this.setTopicIcon(instanceName, "blue");
         this.warnIfRateLimited(instanceName, msg);
         const { text, extraMeta } = await processAttachments(msg, inboundAdapter, this.logger, instanceName);
-        const ipc = this.instanceIpcClients.get(instanceName);
-        if (!ipc) {
-            this.logger.warn({ instanceName }, "No IPC connection to instance");
+        try {
+            await this.deliverToInstance(instanceName, {
+                type: "fleet_inbound",
+                content: text,
+                targetSession: instanceName, // Channel messages → instance's own session
+                meta: {
+                    chat_id: msg.chatId,
+                    message_id: msg.messageId,
+                    user: msg.username,
+                    user_id: msg.userId,
+                    ts: msg.timestamp.toISOString(),
+                    thread_id: msg.threadId ?? "",
+                    source: msg.source,
+                    ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
+                    ...extraMeta,
+                },
+            });
+        }
+        catch (err) {
+            this.logger.warn({ err: err.message, instanceName }, "Wake/delivery failed");
+            if (msg.chatId && msg.messageId) {
+                const reactAdapter = this.getAdapterForInstance(instanceName) ?? inboundAdapter;
+                reactAdapter.react(this.reactTarget(msg), msg.messageId, "❌").catch(() => { });
+            }
             return;
         }
-        ipc.send({
-            type: "fleet_inbound",
-            content: text,
-            targetSession: instanceName, // Channel messages → instance's own session
-            meta: {
-                chat_id: msg.chatId,
-                message_id: msg.messageId,
-                user: msg.username,
-                user_id: msg.userId,
-                ts: msg.timestamp.toISOString(),
-                thread_id: msg.threadId ?? "",
-                source: msg.source,
-                ...(msg.replyToText ? { reply_to_text: msg.replyToText } : {}),
-                ...extraMeta,
-            },
-        });
         this.lastInboundUser.set(instanceName, msg.username);
         this.logger.info(`${msg.username} → ${instanceName}: ${(text ?? "").slice(0, 100)}`);
         this.eventLog?.logActivity("message", msg.username, (text ?? "").slice(0, 200), instanceName);
@@ -2131,20 +2250,23 @@ export class FleetManager {
         const schedulerDefaults = this.fleetConfig?.defaults.scheduler;
         const retryCount = schedulerDefaults?.retry_count ?? 3;
         const retryInterval = schedulerDefaults?.retry_interval_ms ?? 30_000;
-        const deliver = () => {
-            const ipc = this.instanceIpcClients.get(target);
-            if (!ipc?.connected)
+        const deliver = async () => {
+            try {
+                await this.deliverToInstance(target, {
+                    type: "fleet_schedule_trigger",
+                    payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
+                    meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
+                });
+                // A scheduled trigger also puts the instance to work — show a cancel button.
+                void this.sendCancelButton(target);
+                return true;
+            }
+            catch (err) {
+                this.logger.warn({ err: err.message, target }, "Scheduled wake/delivery attempt failed");
                 return false;
-            ipc.send({
-                type: "fleet_schedule_trigger",
-                payload: { schedule_id: id, message: `[Scheduled] ${message}`, label },
-                meta: { chat_id: reply_chat_id, thread_id: reply_thread_id, user: "scheduler" },
-            });
-            // A scheduled trigger also puts the instance to work — show a cancel button.
-            void this.sendCancelButton(target);
-            return true;
+            }
         };
-        if (deliver()) {
+        if (await deliver()) {
             this.scheduler.recordRun(id, "delivered");
             if (source !== target)
                 this.notifySourceTopic(schedule);
@@ -2152,7 +2274,7 @@ export class FleetManager {
         }
         for (let i = 0; i < retryCount; i++) {
             await new Promise((r) => setTimeout(r, retryInterval));
-            if (deliver()) {
+            if (await deliver()) {
                 this.scheduler.recordRun(id, "delivered");
                 if (source !== target)
                     this.notifySourceTopic(schedule);
@@ -2685,6 +2807,10 @@ export class FleetManager {
     startStatuslineWatcher(name) {
         this.statuslineWatcher.watch(name);
     }
+    stopStatuslineWatcher(name) {
+        // Pausing stops I/O but retains the last observed limits for status views.
+        this.statuslineWatcher.unwatch(name, true);
+    }
     reactMessageStatus(instanceName, chatId, messageId, emoji) {
         // React via the adapter BOUND to this instance — NOT the first discord world.
         // Otherwise, in a same-channel/same-guild multi-bot setup, the inbound 👀
@@ -3035,7 +3161,7 @@ export class FleetManager {
             return [];
         }
     }
-    async sendHangNotification(instanceName) {
+    async sendHangNotification(instanceName, unchangedForMs) {
         const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
         if (!adapter)
             return;
@@ -3044,11 +3170,18 @@ export class FleetManager {
         if (!groupId)
             return;
         const threadId = this.fleetConfig?.instances[instanceName]?.topic_id;
+        const instanceHangConfig = this.fleetConfig?.instances[instanceName]?.hang_detector;
+        const configuredMinutes = instanceHangConfig?.timeout_minutes
+            ?? this.fleetConfig?.defaults?.hang_detector?.timeout_minutes
+            ?? 15;
+        const unchangedMinutes = unchangedForMs == null
+            ? configuredMinutes
+            : Math.max(1, Math.floor(unchangedForMs / 60_000));
         this.setTopicIcon(instanceName, "red");
         await adapter.notifyAlert(String(groupId), {
             type: "hang",
             instanceName,
-            message: `⚠️ ${instanceName} appears hung (no activity for 15+ minutes)`,
+            message: `⚠️ ${instanceName} may be stuck — pane unchanged for ${unchangedMinutes}min, ready prompt not recognized`,
             choices: [
                 { id: `hang:restart:${instanceName}`, label: "🔄 Force restart" },
                 { id: `hang:wait:${instanceName}`, label: "⏳ Keep waiting" },
@@ -3549,11 +3682,6 @@ When users create specialized instances, suggest these configurations:
         const fullText = logContext
             ? `[Chat log for context]\n${logContext}\n\n[User message]\n${text}`
             : text;
-        const ipc = this.instanceIpcClients.get(instanceName);
-        if (!ipc) {
-            this.logger.warn({ instanceName }, "Classic channel instance IPC not connected");
-            return;
-        }
         const meta = {
             chat_id: msg.chatId,
             message_id: msg.messageId,
@@ -3575,12 +3703,18 @@ When users create specialized instances, suggest these configurations:
                 meta.image_path = saves[saves.length - 1][1].split(",")[0].trim();
             }
         }
-        ipc.send({
-            type: "fleet_inbound",
-            content: fullText,
-            targetSession: instanceName,
-            meta,
-        });
+        try {
+            await this.deliverToInstance(instanceName, {
+                type: "fleet_inbound",
+                content: fullText,
+                targetSession: instanceName,
+                meta,
+            });
+        }
+        catch (err) {
+            this.logger.warn({ err: err.message, instanceName }, "Classic wake/delivery failed");
+            return;
+        }
         this.lastInboundUser.set(instanceName, msg.username);
         this.logger.info(`${msg.username} → ${instanceName} (classic): ${text.slice(0, 100)}`);
         this.trackInboundMsg(instanceName, msg);
@@ -3683,6 +3817,13 @@ When users create specialized instances, suggest these configurations:
             clearInterval(this.watchdogTimer);
             this.watchdogTimer = null;
         }
+        // Cancel adapter retry timers
+        for (const state of this.adapterState.values()) {
+            if (state.retryTimer) {
+                clearTimeout(state.retryTimer);
+                state.retryTimer = undefined;
+            }
+        }
         this.clearStatuslineWatchers();
         this.costGuard?.stop();
         this.dailySummary?.stop();
@@ -4253,6 +4394,7 @@ When users create specialized instances, suggest these configurations:
                         lastActivity: this.lastActivityMs(inst.name) || null,
                         currentTask,
                         idle: this.getInstanceIdle(inst.name),
+                        state: this.getInstanceExecutionState(inst.name),
                     };
                 });
                 res.setHeader("Access-Control-Allow-Origin", "*");