@songsid/agend 2.1.2-beta.17 → 2.1.2-beta.18

package/dist/fleet-manager.d.ts

@@ -138,7 +138,26 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private scheduleReconcile;
     private finishStartup;
     lastActivityMs(name: string): number;
+    /**
+     * Is the instance between turns?
+     *
+     * Prefers the daemon's pane state machine (debounced, busy-pattern aware) over
+     * the control client's raw 2-second output-silence heuristic. The raw heuristic
+     * reads every >2s output lull as idle — and long silent tools (a build, a test
+     * run) or an LLM pause produce those constantly mid-turn. That misreading is
+     * what retired cancel buttons in the middle of long work (the 5-minute backstop
+     * fired during a lull) and froze their progress text (ticker skipped "idle"
+     * ticks). The silence heuristic remains only as the fallback for instances
+     * whose daemon has not reported a state yet.
+     */
     private getInstanceIdle;
+    /**
+     * True when the instance claims working/stuck but nothing has refreshed that
+     * claim for STATE_REPORT_STALE_MS despite the backstop's per-tick queries.
+     * Measures the CACHE's age, not the button's — a healthy multi-hour run
+     * answers every query and never trips this.
+     */
+    private stateReportDead;
     webhookEmit(event: string, name: string, data?: Record<string, unknown>): void;
     getSysInfo(): import("./fleet-context.js").SysInfo;
     /** Load fleet.yaml and build routing table */
@@ -501,6 +520,12 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
      * delete the very cancel button this is trying to keep alive.
      */
     private startProgressTicker;
+    /**
+     * After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
+     * it has not, retire its button. Re-arming replaces the previous timer, so a
+     * burst of replies ends with exactly one pending check.
+     */
+    private armReplyGrace;
     /** Retire (delete) every cancel button belonging to an instance. */
     private retireInstanceButtons;
     /** Begin retiring one button (delete + bounded retry on failure). Idempotent:
@@ -510,6 +535,19 @@ export declare class FleetManager implements FleetContext, LifecycleContext, Arc
     private attemptButtonDelete;
     /** Clear an entry's timers (retry + idle-check) and drop it from the map. */
     private discardButton;
+    /**
+     * Mirror the live buttons to disk. The map is memory-only, so before this a
+     * fleet restart orphaned every button on screen: frozen "處理中…" text and a
+     * click that did nothing, forever. The ledger is tiny (a handful of rows) and
+     * written on every add/remove — no debounce needed at that rate.
+     */
+    private persistCancelButtons;
+    /**
+     * Delete the previous process's buttons. Runs once adapters are up: nothing
+     * from a previous fleet process can still be mid-turn from this process's
+     * point of view, so every ledger row is an orphan by definition.
+     */
+    private sweepOrphanedCancelButtons;
     /** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
     private scheduleButtonRetry;
     /** Delete one button's message via its own adapter. Resolves on success,

package/dist/fleet-manager.js

@@ -85,6 +85,32 @@ const CANCEL_BTN_MAX_RETRIES = 3;
  * buttons no clear trigger reached (e.g. a scheduled/HTTP turn that never called
  * reply). 5min (not the old 2s idle-watch) so Thinking isn't misread as idle. */
 const CANCEL_BTN_IDLE_CHECK_INTERVAL_MS = 5 * 60_000;
+/**
+ * How long after a reply an instance gets to resume working before its cancel
+ * button is retired. A short turn ends with a reply and never works again → the
+ * button disappears ~2 minutes after the answer. A multi-step run replies
+ * mid-flight and keeps going → the grace check sees "working" and leaves the
+ * button alone (the idle edge retires it when the run really ends).
+ */
+const REPLY_RETIRE_GRACE_MS = 2 * 60_000;
+/**
+ * The daemon only broadcasts execution state on TRANSITIONS, so a long
+ * single-state run sends nothing for hours. The idle backstop therefore pokes a
+ * query each tick; a live daemon answers within milliseconds and refreshes the
+ * cache. When nothing has refreshed it for this long despite those pokes, the
+ * reporting chain (daemon, IPC, or state monitor) is dead and a "working" state
+ * from 30 minutes ago proves nothing — the button may be retired.
+ */
+const STATE_REPORT_STALE_MS = 30 * 60_000;
+/**
+ * Unconditional ceiling on a cancel button's life. Deliberately far beyond any
+ * legitimate run (multi-hour tasks are normal on this fleet): everything below
+ * this is decided by real state; a button that somehow survives a full day is
+ * wreckage, stuck or not.
+ */
+const CANCEL_BTN_MAX_LIFETIME_MS = 24 * 60 * 60_000;
+/** Orphaned-button ledger, swept at startup. Lives in the fleet data dir. */
+const CANCEL_BTN_LEDGER_FILE = "cancel-buttons.json";
 /**
  * How often the cancel button's text is refreshed with elapsed working time.
  *
@@ -273,12 +299,35 @@ export class FleetManager {
         this.startupComplete = true;
         if (this.reloadPending)
             this.scheduleReconcile();
+        void this.sweepOrphanedCancelButtons();
     }
     // ── ArchiverContext bridge ────────────────────────────────────────────
     lastActivityMs(name) {
         return this.lastActivity.get(name) ?? 0;
     }
+    /**
+     * Is the instance between turns?
+     *
+     * Prefers the daemon's pane state machine (debounced, busy-pattern aware) over
+     * the control client's raw 2-second output-silence heuristic. The raw heuristic
+     * reads every >2s output lull as idle — and long silent tools (a build, a test
+     * run) or an LLM pause produce those constantly mid-turn. That misreading is
+     * what retired cancel buttons in the middle of long work (the 5-minute backstop
+     * fired during a lull) and froze their progress text (ticker skipped "idle"
+     * ticks). The silence heuristic remains only as the fallback for instances
+     * whose daemon has not reported a state yet.
+     */
     getInstanceIdle(name) {
+        // A daemon that is not running cannot be mid-turn. This is what a stale
+        // "working" cache after a hard daemon kill (SIGKILL/OOM — no IPC crash
+        // report ever arrives) must not override.
+        if (this.getInstanceStatus(name) !== "running")
+            return true;
+        const state = this.getInstanceExecutionState(name);
+        if (state === "working" || state === "stuck")
+            return false;
+        if (state === "idle")
+            return true;
         try {
             const widFile = join(this.getInstanceDir(name), "window-id");
             if (!existsSync(widFile))
@@ -290,6 +339,18 @@ export class FleetManager {
             return true;
         }
     }
+    /**
+     * True when the instance claims working/stuck but nothing has refreshed that
+     * claim for STATE_REPORT_STALE_MS despite the backstop's per-tick queries.
+     * Measures the CACHE's age, not the button's — a healthy multi-hour run
+     * answers every query and never trips this.
+     */
+    stateReportDead(name) {
+        const cached = this.instanceStateCache.get(name);
+        if (!cached)
+            return false; // no claim to distrust — getInstanceIdle owns this case
+        return Date.now() - cached.receivedAt > STATE_REPORT_STALE_MS;
+    }
     // ── LifecycleContext bridge methods ──────────────────────────────────────
     webhookEmit(event, name, data) {
         this.webhookEmitter?.emit(event, name, data);
@@ -518,6 +579,9 @@ export class FleetManager {
             unchangedForMs: numberOr(msg.unchangedForMs, previous?.unchangedForMs ?? 0),
             observedAt: numberOr(msg.observedAt, now),
             stateChangedAt: numberOr(msg.stateChangedAt, previous?.state === state ? previous.stateChangedAt : now),
+            // Fleet-manager receipt time, NOT the daemon's observation time: staleness
+            // asks "is anyone still reporting", which only the receiver can date.
+            receivedAt: now,
         });
         for (const check of this.instanceIdleWaiters.get(name) ?? [])
             check();
@@ -3016,17 +3080,18 @@ export class FleetManager {
         // Route standard channel tools (reply, react, edit_message, download_attachment)
         if (routeToolCall(outAdapter, tool, args, threadId, respond)) {
             if (tool === "reply") {
-                // A reply is NOT proof the turn is over: on multi-step work an agent
-                // replies ("starting…") and keeps going for many minutes. Retiring the
-                // button here left the channel looking idle with no way to cancel and no
-                // sign anything was happening (#410). Idle state owns retirement now; if the
-                // instance is still working, move the button below the new reply so it stays
-                // the last thing in the channel.
+                // A reply is NOT proof the turn is over (#410) — but it is not proof of
+                // more work either. Split the difference: an instance that is clearly
+                // idle loses the button now; one that looks busy keeps it (re-posted
+                // below the reply so it stays last in the channel), with a 2-minute
+                // grace check — if it has NOT resumed working by then, the reply was the
+                // end of the turn and the button goes. A multi-step run that keeps
+                // working sails through the check and keeps its button.
                 if (this.getInstanceIdle(instanceName)) {
                     this.clearCancelButton(instanceName);
                 }
                 else {
-                    void this.sendCancelButton(instanceName);
+                    void this.sendCancelButton(instanceName).then(() => this.armReplyGrace(instanceName));
                 }
                 this.reactDone(instanceName);
                 const replyTo = this.lastInboundUser.get(instanceName) ?? "user";
@@ -4019,8 +4084,13 @@ export class FleetManager {
         const adapter = this.getAdapterForInstance(instanceName) ?? this.adapter;
         if (!adapter)
             return;
+        // Resolve the group through the world fallback (first world when unbound),
+        // NOT through getChannelConfig(binding)?.group_id: on a fleet configured with
+        // `channels:` worlds the primary `channel:` block is empty, so an instance
+        // with no world binding yet (fresh restart, cross-instance delegation)
+        // resolved group_id to undefined and the button silently never appeared.
         const adapterId = this.instanceWorldBinding.get(instanceName);
-        const groupId = this.getChannelConfig(adapterId)?.group_id;
+        const groupId = this.getGroupIdForInstance(instanceName) || undefined;
         const topicId = this.fleetConfig?.instances[instanceName]?.topic_id;
         let chatId;
         let threadId;
@@ -4036,8 +4106,12 @@ export class FleetManager {
             if (!chatId && groupId)
                 chatId = String(groupId);
         }
-        if (!chatId)
+        if (!chatId) {
+            // A button that cannot be addressed must say so — this exact silence is how
+            // "the cancel button sometimes never appears" stayed unreported-in-logs.
+            this.logger.warn({ instanceName, topicId, groupId }, "Cannot address cancel button (no chat id resolved)");
             return;
+        }
         try {
             const sent = await adapter.notifyAlert(chatId, {
                 type: "cancel",
@@ -4064,6 +4138,10 @@ export class FleetManager {
                 // when the work was handed over — not from the pane's working transition,
                 // which resets if the CLI blips idle mid-turn.
                 startedAt: Date.now(),
+                // Matches the text notifyAlert just posted, so the first 60s tick does
+                // not re-edit identical text — which put a "(edited)" mark on Discord
+                // with nothing visibly changed.
+                lastProgressText: "👀 處理中…",
             };
             this.startProgressTicker(entry);
             // Idle-check backstop: every 5min, if the instance is idle, retire the
@@ -4074,12 +4152,24 @@ export class FleetManager {
                     clearInterval(entry.idleCheckTimer);
                     return;
                 }
-                if (this.getInstanceIdle(instanceName)) {
-                    this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button idle backstop retiring");
+                const reason = this.getInstanceIdle(instanceName) ? "idle"
+                    : this.stateReportDead(instanceName) ? "state reports stopped"
+                        : Date.now() - (entry.startedAt ?? 0) > CANCEL_BTN_MAX_LIFETIME_MS ? "24h ceiling"
+                            : null;
+                if (reason) {
+                    this.logger.info({ instanceName, messageId: entry.messageId, reason }, "Cancel button backstop retiring");
                     this.retireButton(entry);
+                    return;
                 }
+                // Still looks busy. The daemon only broadcasts on transitions, so ask for
+                // a fresh snapshot — a live daemon's answer refreshes receivedAt and keeps
+                // the staleness check honest; a dead one's silence is the evidence.
+                this.instanceIpcClients.get(instanceName)?.send({
+                    type: "query_instance_state", requestId: `cancel-btn-${Date.now()}`,
+                });
             }, CANCEL_BTN_IDLE_CHECK_INTERVAL_MS);
             this.cancelButtons.set(sent.messageId, entry);
+            this.persistCancelButtons();
             this.logger.info({ instanceName, messageId: sent.messageId }, "Cancel button sent");
         }
         catch (e) {
@@ -4152,9 +4242,6 @@ export class FleetManager {
                 clearInterval(entry.progressTimer);
                 return;
             }
-            // Idle means the turn ended; the idle-edge handler retires the button.
-            if (this.getInstanceIdle(entry.instanceName))
-                return;
             const text = FleetManager.progressText(Date.now() - (entry.startedAt ?? Date.now()), this.instanceActivity.get(entry.instanceName));
             if (text === entry.lastProgressText)
                 return; // nothing changed — skip the API call
@@ -4177,6 +4264,29 @@ export class FleetManager {
         }, PROGRESS_UPDATE_INTERVAL_MS);
         entry.progressTimer.unref?.();
     }
+    /**
+     * After a reply: give the instance REPLY_RETIRE_GRACE_MS to resume working; if
+     * it has not, retire its button. Re-arming replaces the previous timer, so a
+     * burst of replies ends with exactly one pending check.
+     */
+    armReplyGrace(instanceName) {
+        for (const entry of this.cancelButtons.values()) {
+            if (entry.instanceName !== instanceName)
+                continue;
+            if (entry.replyGraceTimer)
+                clearTimeout(entry.replyGraceTimer);
+            entry.replyGraceTimer = setTimeout(() => {
+                entry.replyGraceTimer = undefined;
+                if (!this.cancelButtons.has(entry.messageId))
+                    return;
+                if (!this.getInstanceIdle(instanceName))
+                    return; // resumed — a long run keeps its button
+                this.logger.info({ instanceName, messageId: entry.messageId }, "Cancel button retired — no work resumed after reply");
+                this.retireButton(entry);
+            }, REPLY_RETIRE_GRACE_MS);
+            entry.replyGraceTimer.unref?.();
+        }
+    }
     /** Retire (delete) every cancel button belonging to an instance. */
     retireInstanceButtons(instanceName) {
         // Snapshot first — retireButton may delete entries from the map on success.
@@ -4210,7 +4320,68 @@ export class FleetManager {
             clearInterval(entry.idleCheckTimer);
         if (entry.progressTimer)
             clearInterval(entry.progressTimer);
+        if (entry.replyGraceTimer)
+            clearTimeout(entry.replyGraceTimer);
         this.cancelButtons.delete(entry.messageId);
+        this.persistCancelButtons();
+    }
+    /**
+     * Mirror the live buttons to disk. The map is memory-only, so before this a
+     * fleet restart orphaned every button on screen: frozen "處理中…" text and a
+     * click that did nothing, forever. The ledger is tiny (a handful of rows) and
+     * written on every add/remove — no debounce needed at that rate.
+     */
+    persistCancelButtons() {
+        try {
+            const rows = [...this.cancelButtons.values()].map(e => ({
+                instanceName: e.instanceName,
+                adapterId: e.adapterId,
+                chatId: e.chatId,
+                messageId: e.messageId,
+                threadId: e.threadId,
+            }));
+            writeFileSync(join(this.dataDir, CANCEL_BTN_LEDGER_FILE), JSON.stringify(rows));
+        }
+        catch (err) {
+            this.logger.debug({ err }, "Cancel button ledger write failed");
+        }
+    }
+    /**
+     * Delete the previous process's buttons. Runs once adapters are up: nothing
+     * from a previous fleet process can still be mid-turn from this process's
+     * point of view, so every ledger row is an orphan by definition.
+     */
+    async sweepOrphanedCancelButtons() {
+        const ledgerPath = join(this.dataDir, CANCEL_BTN_LEDGER_FILE);
+        let rows;
+        try {
+            if (!existsSync(ledgerPath))
+                return;
+            rows = JSON.parse(readFileSync(ledgerPath, "utf-8"));
+        }
+        catch {
+            try {
+                unlinkSync(ledgerPath);
+            }
+            catch { /* corrupt ledger — drop it */ }
+            return;
+        }
+        for (const row of rows) {
+            const adapter = (row.adapterId ? this.worlds.get(row.adapterId)?.adapter : undefined)
+                ?? this.getAdapterForInstance?.(row.instanceName) ?? this.adapter;
+            if (!adapter?.deleteMessage)
+                continue;
+            try {
+                await adapter.deleteMessage(row.chatId, row.messageId, row.threadId);
+                this.logger.info({ instanceName: row.instanceName, messageId: row.messageId }, "Swept orphaned cancel button from previous run");
+            }
+            catch (err) {
+                // Best effort: the message may already be gone, or too old to delete.
+                this.logger.debug({ err, messageId: row.messageId }, "Orphaned cancel button sweep failed");
+            }
+        }
+        // The current process owns the ledger from here on.
+        this.persistCancelButtons();
     }
     /** Re-attempt a failed button delete up to CANCEL_BTN_MAX_RETRIES times. */
     scheduleButtonRetry(entry, err) {