@songsid/agend 2.0.8-beta.2 → 2.0.8-beta.20

package/dist/daemon.js

@@ -385,11 +385,27 @@ export class Daemon extends EventEmitter {
                     scheduleNext();
                     return;
                 }
-                const paneStatus = await this.tmux.getPaneStatus();
+                // Human-readable backend label for logs (e.g. "claude", "kiro-cli")
+                const cliLabel = this.backend?.binaryName ?? "CLI";
+                let paneStatus = await this.tmux.getPaneStatus();
                 if (paneStatus?.alive) {
                     scheduleNext();
                     return;
                 }
+                // A null status is ambiguous: it can be a transient `tmux list-panes`
+                // failure (e.g. tmux busy during a fleet-restart storm) rather than a
+                // real exit. Re-confirm once after a short delay before treating it as
+                // a crash. A non-null {alive:false} is a definite dead pane (real exit)
+                // and needs no recheck.
+                if (paneStatus === null) {
+                    await new Promise(r => setTimeout(r, 1500));
+                    paneStatus = await this.tmux.getPaneStatus();
+                    if (paneStatus?.alive) {
+                        this.logger.debug(`[health] ${cliLabel} pane reported gone then alive on recheck — transient query failure, ignoring`);
+                        scheduleNext();
+                        return;
+                    }
+                }
                 // paneStatus === null → window gone entirely (e.g. tmux server crash)
                 // paneStatus.alive === false → pane dead, exit code available
                 const exitCode = paneStatus?.exitCode;
@@ -401,13 +417,17 @@ export class Daemon extends EventEmitter {
                     this.healthCheckPaused = true;
                     return;
                 }
-                // Distinguish tmux server crash from single window crash
+                // Distinguish tmux server crash from single window crash.
+                // nullReason records *why* getPaneStatus returned null (for diagnosing
+                // whether this was a real window loss or a transient query failure).
                 let crashType = "window";
+                let nullReason;
                 if (!paneStatus) {
                     const serverAlive = await TmuxManager.sessionExists(this.tmuxSessionName);
                     if (!serverAlive) {
                         crashType = "server";
-                        this.logger.error("tmux server died — all windows lost");
+                        nullReason = "server_gone";
+                        this.logger.error(`tmux server died — all ${cliLabel} windows lost`);
                         // Fleet-level circuit breaker: pause all instances on repeated tmux server crashes
                         Daemon.tmuxServerCrashTimestamps.push(Date.now());
                         const cutoff = Date.now() - 5 * 60_000;
@@ -428,23 +448,34 @@ export class Daemon extends EventEmitter {
                         await new Promise(r => setTimeout(r, 2_000)); // let session stabilize
                     }
                     else {
-                        this.logger.warn({ exitCode }, "Claude window died (tmux server alive)");
+                        // null but server alive: window-level disappearance. Probe whether
+                        // the window truly no longer exists vs a transient query glitch.
+                        nullReason = "no_window";
+                        try {
+                            const windows = await TmuxManager.listWindows(this.tmuxSessionName);
+                            if (windows.some(w => w.name === this.name))
+                                nullReason = "window_present_query_glitch";
+                        }
+                        catch {
+                            nullReason = "query_error";
+                        }
+                        this.logger.warn({ exitCode, nullReason }, `${cliLabel} window not found (tmux server alive)`);
                     }
                 }
                 else {
-                    this.logger.warn({ exitCode }, "Claude process exited");
+                    this.logger.warn({ exitCode }, `${cliLabel} process exited`);
                 }
-                // Capture last output from dead pane before killing
+                // Capture last output before killing. Best-effort even when the pane is
+                // gone (paneStatus null) — gives the crash record something to diagnose
+                // from instead of an empty lastOutput.
                 let lastOutput;
-                if (paneStatus) {
-                    try {
-                        const raw = await this.tmux.capturePaneWithHistory(50);
-                        // Strip ANSI escape codes for readability
-                        const cleaned = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-                        lastOutput = cleaned.trimEnd() || undefined;
-                    }
-                    catch { /* best effort */ }
+                try {
+                    const raw = await this.tmux.capturePaneWithHistory(50);
+                    // Strip ANSI escape codes for readability
+                    const cleaned = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
+                    lastOutput = cleaned.trimEnd() || undefined;
                 }
+                catch { /* best effort — pane may already be gone */ }
                 // Kill the dead window (remain-on-exit keeps it around) before respawn
                 if (paneStatus) {
                     await this.tmux.killWindow();
@@ -473,8 +504,21 @@ export class Daemon extends EventEmitter {
                     }
                     // Already attempted recovery — fall through to normal crash handling
                 }
+                // Detect a --continue/--resume failure (no conversation to resume). The
+                // session-id file persists across the crash, so a blind respawn would add
+                // --continue again and crash in the same way → loop. Clear the session id
+                // and skip resume so the next spawn starts fresh. (skipResume also stops
+                // saveSessionId below from resurrecting the id from statusline.json.)
+                if (lastOutput && /no conversation found|no conversation to (continue|resume)|no previous (session|conversation)|--continue/i.test(lastOutput)) {
+                    this.logger.warn("Detected --continue/resume failure — clearing session-id; next spawn starts fresh");
+                    try {
+                        unlinkSync(join(this.instanceDir, "session-id"));
+                    }
+                    catch { /* may not exist */ }
+                    this.skipResume = true;
+                }
                 // Append to crash history
-                this.appendCrashHistory({ exitCode, lastOutput, crashType });
+                this.appendCrashHistory({ exitCode, lastOutput, crashType, reason: nullReason });
                 // Detect rapid crash: sliding window — 3+ crashes in 5 minutes
                 this.crashTimestamps.push(Date.now());
                 const crashWindowMs = 5 * 60_000;
@@ -508,7 +552,7 @@ export class Daemon extends EventEmitter {
                 const delay = backoff === "exponential"
                     ? Math.min(1000 * Math.pow(2, this.crashCount - 1), 60_000)
                     : 1000 * this.crashCount;
-                this.logger.warn({ crashCount: this.crashCount, delay }, "Claude window died — respawning after backoff");
+                this.logger.warn({ crashCount: this.crashCount, delay }, `${cliLabel} window died — respawning after backoff`);
                 await new Promise(r => setTimeout(r, delay));
                 try {
                     this.saveSessionId();
@@ -550,11 +594,11 @@ export class Daemon extends EventEmitter {
                         }
                         catch { /* may not exist */ }
                     }
-                    this.logger.info({ resumed }, "Respawned Claude window after crash");
+                    this.logger.info({ resumed }, `Respawned ${cliLabel} window after crash`);
                     this.emit("crash_respawn", this.name);
                 }
                 catch (err) {
-                    this.logger.error({ err }, "Failed to respawn Claude window");
+                    this.logger.error({ err }, `Failed to respawn ${cliLabel} window`);
                 }
                 scheduleNext();
             }, this.config.restart_policy.health_check_interval_ms ?? 30_000);
@@ -669,6 +713,15 @@ export class Daemon extends EventEmitter {
             }
         }, 5_000); // Check every 5 seconds (runtime dialogs need fast response)
     }
+    /**
+     * Interrupt the CLI's current generation (cancel button / `/cancel`).
+     * Direct tmux key event (not a paste) so it registers as the interrupt key.
+     * kiro-cli interrupts on Ctrl+C; the others (claude-code, codex, …) on Escape.
+     */
+    async sendEscape() {
+        const cancelKey = this.backend?.binaryName === "kiro-cli" ? "C-c" : "Escape";
+        await this.tmux?.sendSpecialKey(cancelKey);
+    }
     async stop() {
         this.logger.info("Stopping daemon instance");
         if (this.healthCheckTimer) {
@@ -704,9 +757,10 @@ export class Daemon extends EventEmitter {
                 // instances stop in parallel (same pattern as pasteText).
                 await new Promise(r => setTimeout(r, 150));
                 await this.tmux.sendSpecialKey("Enter");
-                // Wait up to 10s for graceful exit
-                for (let i = 0; i < 20; i++) {
-                    await new Promise(r => setTimeout(r, 500));
+                // Wait up to 3s for graceful exit, polling every 200ms. A healthy CLI
+                // exits within ~1s; a longer wait just delays the force-kill fallback.
+                for (let i = 0; i < 15; i++) {
+                    await new Promise(r => setTimeout(r, 200));
                     const status = await this.tmux.getPaneStatus();
                     if (!status || !status.alive) {
                         killed = true;
@@ -715,7 +769,7 @@ export class Daemon extends EventEmitter {
                 }
             }
             if (!killed)
-                this.logger.warn("CLI did not exit gracefully within 10s, force killing window");
+                this.logger.warn("CLI did not exit gracefully within 3s, force killing window");
             // Always kill window — remain-on-exit keeps dead panes around after CLI exits
             await this.tmux.killWindow();
             const windowIdFile = join(this.instanceDir, "window-id");
@@ -857,9 +911,9 @@ export class Daemon extends EventEmitter {
             formatted += `\n(reply_to: "${meta.reply_to_text}")`;
         }
         // Serialize deliveries: each message waits for the previous to complete,
-        // and each waits for the CLI to be idle before pasting.
-        const enqueuedAt = Date.now();
-        const isFromInstance = !!meta.from_instance;
+        // and each waits for the CLI to be idle before pasting. Messages are never
+        // dropped for age — a long-busy CLI just queues them until it frees up
+        // (the user can press Cancel to interrupt and let the queue drain sooner).
         const chatId = meta.chat_id;
         const messageId = meta.message_id;
         const wasQueued = this.pasteQueueDepth > 0;
@@ -872,11 +926,6 @@ export class Daemon extends EventEmitter {
         }
         this.pasteLock = this.pasteLock.then(async () => {
             try {
-                // Drop stale user messages (>60s in queue), but never drop cross-instance messages
-                if (!isFromInstance && Date.now() - enqueuedAt > 60_000) {
-                    this.logger.warn({ age: Date.now() - enqueuedAt, user: meta.user }, "Dropping stale message");
-                    return;
-                }
                 if (this.config.pre_task_command) {
                     await this.deliverMessage(this.config.pre_task_command);
                 }
@@ -884,10 +933,10 @@ export class Daemon extends EventEmitter {
                     this.pendingInstructionsNotice = false;
                     await this.deliverMessage("[system] Your instructions/steering files have been updated. Please re-read them for the latest guidelines.");
                 }
-                await this.deliverMessage(formatted);
-                if (chatId && messageId) {
-                    this.emit("message_delivered", { chatId: meta.thread_id || chatId, messageId });
-                }
+                const status = (chatId && messageId)
+                    ? { chatId: meta.thread_id || chatId, messageId }
+                    : undefined;
+                await this.deliverMessage(formatted, status);
             }
             finally {
                 this.pasteQueueDepth--;
@@ -897,41 +946,109 @@ export class Daemon extends EventEmitter {
         });
         this.logger.debug({ user: meta.user, text: content.slice(0, 100) }, "Queued channel message for delivery");
     }
-    /** Deliver a single message: wait for idle, then paste */
-    async deliverMessage(formatted) {
+    /**
+     * Deliver a single message and drive its status reactions:
+     *   ⏳ message_queued    — CLI busy; queued, waiting for idle
+     *   👀 message_delivered — pasted + Enter sent; agent now has it
+     *   ✅ message_confirmed — idle→busy transition observed; agent is processing
+     *   ❌ message_failed    — tmux window gone, paste retries exhausted
+     * Returns true once the message is in the CLI, false only on real delivery failure.
+     *
+     * Bug A (silent message loss): paste failures retry with backoff (window recovery)
+     * and emit `message_failed` if all attempts fail.
+     * Busy handling (UX): we never force-paste into a busy CLI and never give up on a
+     * busy one — we show ⏳ and wait for idle indefinitely (a genuinely hung CLI is the
+     * hang detector's job; a user who can't wait presses Cancel → Escape → idle). The
+     * pasteLock is serial, so later messages naturally queue behind this wait.
+     */
+    async deliverMessage(formatted, status) {
         // Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
         const fenceCount = (formatted.match(/```/g) || []).length;
         if (fenceCount % 2 !== 0) {
             // Odd number of fences = unclosed. Remove all code fences from the message.
             formatted = formatted.replace(/```/g, "");
         }
-        const windowId = this.getWindowId();
-        if (windowId && this.controlClient) {
-            const idle = await this.controlClient.waitForIdle(windowId, this.config.lightweight ? 30_000 : 120_000);
-            if (!idle) {
-                this.logger.warn("Delivering message after idle timeout (CLI may be busy)");
+        let windowId = this.getWindowId();
+        // If the CLI is busy, show ⏳ and wait for it to go idle — no timeout, no force.
+        if (windowId && this.controlClient && !this.controlClient.isIdle(windowId)) {
+            if (status)
+                this.emit("message_queued", status);
+            this.logger.debug("CLI busy — queuing message until idle");
+            await this.controlClient.waitUntilIdle(windowId);
+        }
+        // Bug A: paste with backoff. Transient failures are usually a stale window id
+        // after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
+        const maxAttempts = 3;
+        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+            const pasted = await this.tmux.pasteBuffer(formatted);
+            if (!pasted) {
+                this.logger.warn({ attempt }, "pasteBuffer failed — recovering window and backing off");
+                windowId = (await this.recoverWindow()) ?? windowId;
+                if (attempt < maxAttempts)
+                    await new Promise(r => setTimeout(r, 2000));
+                continue;
             }
-        }
-        const ok = await this.tmux.pasteText(formatted);
-        if (!ok) {
-            // Window ID may be stale after crash/respawn — try to find by name
-            this.logger.warn("pasteText failed, looking up window by name");
-            try {
-                const windows = await TmuxManager.listWindows(this.tmuxSessionName);
-                const match = windows.find(w => w.name === this.name);
-                if (match) {
-                    this.tmux = new TmuxManager(this.tmuxSessionName, match.id);
-                    writeFileSync(join(this.instanceDir, "window-id"), match.id);
-                    await this.controlClient?.registerWindow(match.id);
-                    await this.tmux.pasteText(formatted);
-                    this.logger.info({ windowId: match.id }, "Recovered window ID and delivered message");
+            // Settle the bracketed paste, then submit.
+            await new Promise(r => setTimeout(r, 500));
+            const enterAt = Date.now();
+            await this.tmux.sendSpecialKey("Enter");
+            if (status)
+                this.emit("message_delivered", status); // 👀
+            // Confirm the CLI accepted the message by transitioning idle→busy (new output
+            // after Enter). If still idle after ~2s the Enter was likely swallowed while
+            // the TUI was redrawing — re-send Enter once and re-check.
+            if (windowId && this.controlClient) {
+                let becameBusy = await this.confirmBusyAfterEnter(windowId, enterAt);
+                if (!becameBusy) {
+                    this.logger.warn("No idle→busy transition after Enter — re-sending Enter once");
+                    const retryAt = Date.now();
+                    await this.tmux.sendSpecialKey("Enter");
+                    becameBusy = await this.confirmBusyAfterEnter(windowId, retryAt);
                 }
+                if (becameBusy && status)
+                    this.emit("message_confirmed", status); // ✅
             }
-            catch (retryErr) {
-                this.logger.error({ err: retryErr }, "Failed to recover window for message delivery");
+            else {
+                // No control client to observe output: fall back to the legacy double-Enter.
+                await new Promise(r => setTimeout(r, 1000));
+                await this.tmux.sendSpecialKey("Enter");
+                if (status)
+                    this.emit("message_confirmed", status); // ✅ (best-effort)
             }
+            return true;
+        }
+        this.logger.error("Message delivery failed after retries — window not ready");
+        if (status)
+            this.emit("message_failed", status); // ❌
+        return false;
+    }
+    /** Re-resolve this instance's tmux window by name (stale id after crash/respawn). */
+    async recoverWindow() {
+        try {
+            const windows = await TmuxManager.listWindows(this.tmuxSessionName);
+            const match = windows.find(w => w.name === this.name);
+            if (!match)
+                return undefined;
+            this.tmux = new TmuxManager(this.tmuxSessionName, match.id);
+            writeFileSync(join(this.instanceDir, "window-id"), match.id);
+            await this.controlClient?.registerWindow(match.id);
+            this.logger.info({ windowId: match.id }, "Recovered window ID for message delivery");
+            return match.id;
+        }
+        catch (retryErr) {
+            this.logger.error({ err: retryErr }, "Failed to recover window for message delivery");
+            return undefined;
         }
     }
+    /** Poll up to ~2s (200ms × 10) for the pane to emit output after `since`. */
+    async confirmBusyAfterEnter(windowId, since) {
+        for (let i = 0; i < 10; i++) {
+            await new Promise(r => setTimeout(r, 200));
+            if (this.controlClient.hasOutputSince(windowId, since))
+                return true;
+        }
+        return false;
+    }
     getWindowId() {
         try {
             return readFileSync(join(this.instanceDir, "window-id"), "utf-8").trim() || undefined;
@@ -1504,6 +1621,10 @@ export class Daemon extends EventEmitter {
         return true;
     }
     saveSessionId() {
+        // When a resume failure has forced a fresh start, don't persist the stale id
+        // back from statusline.json — that would re-arm --continue and re-loop.
+        if (this.skipResume)
+            return;
         const sid = this.backend?.getSessionId();
         if (sid) {
             writeFileSync(join(this.instanceDir, "session-id"), sid);
@@ -1642,6 +1763,7 @@ export class Daemon extends EventEmitter {
                 instance: this.name,
                 crashType: data.crashType,
                 exitCode: data.exitCode,
+                reason: data.reason,
                 lastOutput: data.lastOutput,
                 crashCount: this.crashCount + 1,
                 crashesInWindow: this.crashTimestamps.length,