@songsid/agend 2.1.6-beta.5 → 2.1.6-beta.7

package/dist/daemon.js

@@ -437,7 +437,23 @@ export class BackendUnreachableStartupError extends Error {
 /** Bounded wait (under the pane lock) for the prompt to return before retrying a dropped Enter. */
 const STRANDED_RETRY_READY_WAIT_MS = 30_000;
 /** A passive startup phase must clear within this bound before an Enter is sent. */
-const INPUT_TRANSIENT_WAIT_MS = 30_000;
+/**
+ * How long a passive startup/resume transient may sit WITHOUT REPAINTING before
+ * delivery gives up. It is a stall budget, not a total: a CLI that is still
+ * painting its resume is making progress, and #826 is what happens when the two
+ * are confused — codex's resume is the only declared transient in the fleet
+ * (backend/codex.ts), a woken paused codex routinely takes longer than this to
+ * come back, and the flat 30s cap turned that into ❌ plus a discarded message
+ * on a pane that was seconds from ready.
+ */
+const INPUT_TRANSIENT_STALL_MS = 30_000;
+/**
+ * The absolute ceiling for the same wait, so a transient that repaints forever
+ * (a spinner on a wedged resume) still ends in an honest failure. Bounded, as
+ * every other delivery wait is — just bounded by "no progress", not by a guess
+ * at how long a resume takes.
+ */
+const INPUT_TRANSIENT_WAIT_MS = 10 * 60_000;
 const INPUT_TRANSIENT_POLL_MS = 250;
 /** Max "stranded text → submit → wait for prompt → re-check" rounds per delivery; each may send one recovery Enter. */
 const STRANDED_INPUT_MAX_ROUNDS = 3;
@@ -3834,6 +3850,28 @@ export class Daemon extends EventEmitter {
         }
         return formatted;
     }
+    /**
+     * Whether ONE delivery reached a verdict, carried by that delivery.
+     *
+     * `deliverMessage` returns false for two unrelated things: a delivery that
+     * cannot land (window gone, a pane that never came back, an unanswered
+     * dialog) and a delivery that was never attempted (user cancel, a tmux storm,
+     * shutdown). Only the first is a failure to tell the sender about; reporting
+     * the second told a sending agent its message was lost while the fleet was
+     * merely standing down (#826).
+     *
+     * It is per call and not a field on the daemon because deliveries do NOT all
+     * share one lock: an ordinary delivery is serialised on `pasteLock`, a steer
+     * on `steerLock`, and the two run concurrently by design. A field would let a
+     * steer's verdict decide whether a queued message's sender is told its
+     * delivery failed — a false ❌ carrying the wrong correlation id.
+     */
+    failDelivery(verdict, status) {
+        verdict.reached = true;
+        if (status)
+            this.emit("message_failed", status); // ❌
+        return false;
+    }
     /** Tell the fleet when an already-accepted cross-instance pane write failed. */
     reportCrossInstanceDeliveryFailure(meta, error) {
         if (!meta.from_instance)
@@ -3889,10 +3927,14 @@ export class Daemon extends EventEmitter {
             await this.wake();
             if (!this.isDeliveryEpochCurrent(deliveryEpoch))
                 return;
-            if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch, submissionId: meta.message_id })) {
+            const verdict = { reached: false };
+            if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch, submissionId: meta.message_id, verdict })) {
                 this.markTurnStarted(meta, formatted);
             }
-            else if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
+            else if (verdict.reached && this.isDeliveryEpochCurrent(deliveryEpoch)) {
+                // Same rule as the queued path: a steer that never got to try is not a
+                // delivery failure. This holder is the steer's own, which is the point
+                // — it runs on steerLock while a queued delivery runs on pasteLock.
                 this.reportCrossInstanceDeliveryFailure(meta);
             }
         }).catch(err => {
@@ -4022,10 +4064,16 @@ export class Daemon extends EventEmitter {
                 // A fresh delivery begins a fresh turn — its bubble must not inherit
                 // the previous turn's tool list.
                 this.resetToolProgress();
-                if (await this.deliverMessage(formatted, status, { deliveryEpoch, submissionId: meta.message_id })) {
+                const verdict = { reached: false };
+                if (await this.deliverMessage(formatted, status, { deliveryEpoch, submissionId: meta.message_id, verdict })) {
                     this.markTurnStarted(meta, formatted);
                 }
-                else if (meta.from_instance && this.isDeliveryEpochCurrent(deliveryEpoch)) {
+                else if (meta.from_instance && verdict.reached
+                    && this.isDeliveryEpochCurrent(deliveryEpoch)) {
+                    // Only a delivery that reached a verdict is reported. A pane that is
+                    // not ready yet, a cancel, a storm hold or a shutdown all return
+                    // false without one, and telling the sender its message was lost
+                    // there would be the false ❌ of #826 in its other form.
                     this.reportCrossInstanceDeliveryFailure(meta);
                 }
             }
@@ -4057,6 +4105,9 @@ export class Daemon extends EventEmitter {
      * serial in both cases so separate PTY writes can never overlap.
      */
     async deliverMessage(formatted, status, opts) {
+        // The caller passes its own holder when it needs the answer; a system paste
+        // that ignores the outcome gets a throwaway.
+        const verdict = opts?.verdict ?? { reached: false };
         const cancelled = () => opts?.deliveryEpoch !== undefined
             && !this.isDeliveryEpochCurrent(opts.deliveryEpoch);
         if (cancelled() || this.stormWindow?.isStopped())
@@ -4080,7 +4131,7 @@ export class Daemon extends EventEmitter {
         await this.waitForSpawnToSettle();
         if (cancelled())
             return false;
-        if (this.refuseFatalStartupDelivery(status))
+        if (this.refuseFatalStartupDelivery(verdict, status))
             return false;
         let windowId = this.getWindowId();
         const supportsQueuedInput = this.backend?.supportsQueuedInput?.() === true;
@@ -4122,9 +4173,7 @@ export class Daemon extends EventEmitter {
                     // wedged CLI (where the text would sit unsubmitted and the next message
                     // would land on top of it) — and instead of holding the queue silently.
                     this.logger.error("Pane still busy after the idle wait — reporting delivery failure");
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
             }
         }
@@ -4145,9 +4194,7 @@ export class Daemon extends EventEmitter {
                 // recovery Enter, so exactly STRANDED_INPUT_MAX_ROUNDS of them go out.
                 if (round >= STRANDED_INPUT_MAX_ROUNDS) {
                     this.logger.error({ round }, "Input row still not clear after the stranded-text recovery budget — reporting delivery failure");
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
                 const stranded = await this.submitStrandedInputIfAny(windowId);
                 if (cancelled())
@@ -4156,18 +4203,14 @@ export class Daemon extends EventEmitter {
                     break;
                 if (stranded === "failed") {
                     this.logger.error({ round }, "Could not submit the stranded text — reporting delivery failure");
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
                 const readyAgain = await this.waitForPaneReadyForDelivery(windowId);
                 if (cancelled())
                     return false;
                 if (!readyAgain) {
                     this.logger.error("Pane never returned to its prompt after submitting stranded input — reporting delivery failure");
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
             }
         }
@@ -4179,15 +4222,13 @@ export class Daemon extends EventEmitter {
             const outcome = await this.paneWriteLock.run(async () => {
                 if (cancelled())
                     return false;
-                if (this.refuseFatalStartupDelivery(status))
+                if (this.refuseFatalStartupDelivery(verdict, status))
                     return false;
                 // A passive startup phase may paint after the outer readiness probe.
                 // It clears without input, so waiting under the pane lock cannot starve
                 // a dialog dismisser and closes the final clear→paste TOCTOU window.
                 if (!(await this.waitForInputTransientToClear("pre-write"))) {
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
                 // TOCTOU: the probes above ran outside this lock, and the CLI repaints
                 // whenever it likes — a resume prompt can be painted between "clear" and
@@ -4198,7 +4239,7 @@ export class Daemon extends EventEmitter {
                     if (probe.state !== "clear")
                         return "dialog";
                 }
-                return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId);
+                return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId, verdict);
             });
             if (outcome !== "dialog")
                 return outcome;
@@ -4207,18 +4248,14 @@ export class Daemon extends EventEmitter {
             // an honest failure rather than a message that never lands.
             if (round + 1 >= LATE_DIALOG_WRITE_ROUNDS) {
                 this.logger.error({ rounds: round + 1 }, "A dialog kept appearing before the pane write — reporting delivery failure");
-                if (status)
-                    this.emit("message_failed", status); // ❌
-                return false;
+                return this.failDelivery(verdict, status);
             }
             this.logger.info("Dialog appeared before the pane write — waiting for it to clear");
             const clear = gateWindowId ? await this.waitForPaneReadyForDelivery(gateWindowId) : false;
             if (cancelled())
                 return false;
             if (!clear) {
-                if (status)
-                    this.emit("message_failed", status); // ❌
-                return false;
+                return this.failDelivery(verdict, status);
             }
         }
     }
@@ -4253,13 +4290,47 @@ export class Daemon extends EventEmitter {
         return null;
     }
     /** Capture failure is unknown, never evidence that input is available. */
+    /**
+     * The budget for waiting out a passive transient, measured in progress rather
+     * than in wall clock.
+     *
+     * `observe(pane)` answers "may I keep waiting?". The stall timer restarts
+     * every time the pane text changes, so a resume that is still painting never
+     * runs out; a screen that has not changed for INPUT_TRANSIENT_STALL_MS, or a
+     * transient that outlives the ceiling, does. Both ends stay bounded — the
+     * point of #826 was never that the wait should be unlimited, it was that a
+     * flat 30s could not tell "slow" from "stuck".
+     */
+    transientProgressBudget(overallDeadline) {
+        const ceiling = Math.min(overallDeadline, Date.now() + INPUT_TRANSIENT_WAIT_MS);
+        let lastPane = null;
+        let stallDeadline = 0;
+        const budget = {
+            stalled: false,
+            observe(pane) {
+                const now = Date.now();
+                if (pane !== lastPane) {
+                    lastPane = pane;
+                    stallDeadline = now + INPUT_TRANSIENT_STALL_MS;
+                }
+                if (now >= ceiling)
+                    return false;
+                budget.stalled = now >= stallDeadline;
+                return !budget.stalled;
+            },
+        };
+        return budget;
+    }
     async probeInputTransient() {
         if (this.guardedInputTransients().length === 0 || !this.tmux)
             return { state: "clear" };
         try {
             const pane = await this.tmux.capturePane();
             const transient = this.inputTransientInPane(pane);
-            return transient ? { state: "active", transient } : { state: "clear" };
+            // The pane text comes back with the verdict: a transient that is still
+            // repainting is still working, and that is the only thing separating a
+            // slow resume from a wedged one.
+            return transient ? { state: "active", transient, pane } : { state: "clear" };
         }
         catch (err) {
             this.logger.debug({ err }, "capture-pane failed during the input-transient probe — pane state unknown");
@@ -4275,7 +4346,12 @@ export class Daemon extends EventEmitter {
         const generation = this.inputTransientGuardGeneration;
         if (generation === null || generation !== this.spawnGeneration)
             return true;
-        const deadline = Date.now() + timeoutMs;
+        // Same progress rule as the outer readiness wait, for the same reason: this
+        // runs under the pane write lock, so a flat cap here turned a resume that
+        // repainted one second too late into a discarded message. Holding the lock
+        // while the pane is still painting costs nothing — every other writer is
+        // waiting on the same screen.
+        const budget = this.transientProgressBudget(Date.now() + timeoutMs);
         let observedDescription = null;
         for (;;) {
             if (this.inputTransientGuardGeneration !== generation || this.spawnGeneration !== generation) {
@@ -4293,12 +4369,18 @@ export class Daemon extends EventEmitter {
                 observedDescription = probe.transient.description;
                 this.logger.info({ phase, transient: probe.transient.description, generation }, "CLI is still completing startup — waiting before sending Enter");
             }
-            const remaining = deadline - Date.now();
-            if (remaining <= 0) {
-                this.logger.error({ phase, generation, timeoutMs, transient: observedDescription }, "CLI input stayed unavailable — refusing to send Enter");
+            // "unknown" (capture-pane failed) has to consume the budget too, or an
+            // unreadable pane would loop forever now that the flat deadline is gone.
+            // It never "changes", so it stalls out on the same rule.
+            if (!budget.observe(probe.state === "active" ? probe.pane : "\u0000unreadable")) {
+                this.logger.error({
+                    phase, generation, transient: observedDescription,
+                    stalledForMs: budget.stalled ? INPUT_TRANSIENT_STALL_MS : undefined,
+                    ceilingMs: timeoutMs,
+                }, "CLI input stayed unavailable — refusing to send Enter");
                 return false;
             }
-            await new Promise(r => setTimeout(r, Math.min(INPUT_TRANSIENT_POLL_MS, remaining)));
+            await new Promise(r => setTimeout(r, INPUT_TRANSIENT_POLL_MS));
         }
     }
     static dialogKey(dialog) {
@@ -4473,7 +4555,7 @@ export class Daemon extends EventEmitter {
         const deadline = Date.now() + timeoutMs;
         const bottomGated = this.backend?.dropsEnterWhileBusy?.() === true;
         let unknownStreak = 0;
-        let transientDeadline = 0;
+        let transientBudget = null;
         for (;;) {
             const remaining = deadline - Date.now();
             if (remaining <= 0)
@@ -4486,15 +4568,19 @@ export class Daemon extends EventEmitter {
             const transient = await this.probeInputTransient();
             if (transient.state === "active") {
                 unknownStreak = 0;
-                transientDeadline ||= Math.min(deadline, Date.now() + INPUT_TRANSIENT_WAIT_MS);
-                if (Date.now() >= transientDeadline) {
-                    this.logger.error({ transient: transient.transient.description, timeoutMs: INPUT_TRANSIENT_WAIT_MS }, "CLI input stayed unavailable during the delivery-readiness wait");
+                transientBudget ||= this.transientProgressBudget(deadline);
+                if (!transientBudget.observe(transient.pane)) {
+                    this.logger.error({
+                        transient: transient.transient.description,
+                        stalledForMs: transientBudget.stalled ? INPUT_TRANSIENT_STALL_MS : undefined,
+                        ceilingMs: INPUT_TRANSIENT_WAIT_MS,
+                    }, "CLI input stayed unavailable during the delivery-readiness wait");
                     return false;
                 }
                 await new Promise(r => setTimeout(r, BOTTOM_READY_POLL_MS));
                 continue;
             }
-            transientDeadline = 0;
+            transientBudget = null;
             if (transient.state === "unknown") {
                 if (++unknownStreak >= DIALOG_PROBE_UNKNOWN_MAX) {
                     this.logger.error({ probes: unknownStreak }, "Pane stayed unreadable during the input-transient probe — refusing to deliver blind");
@@ -4640,12 +4726,11 @@ export class Daemon extends EventEmitter {
      * what to fix, and holding would only build a queue that floods the pane the
      * moment the flag clears.
      */
-    refuseFatalStartupDelivery(status) {
+    refuseFatalStartupDelivery(verdict, status) {
         if (!this.fatalStartupBlocked)
             return false;
         this.logger.error("Delivery refused — CLI is parked on a fatal startup screen");
-        if (status)
-            this.emit("message_failed", status); // ❌
+        this.failDelivery(verdict, status);
         return true;
     }
     /**
@@ -4724,7 +4809,10 @@ export class Daemon extends EventEmitter {
         }
         return sent;
     }
-    async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status, submissionId) {
+    async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status, submissionId, 
+    // Last and defaulted so the positional callers that ignore the outcome stay
+    // readable; deliverMessage, the only one that reports, always passes its own.
+    verdict = { reached: false }) {
         const signature = this.submissionSignature(formatted, submissionId);
         let windowId = initialWindowId;
         // Bug A: paste with backoff. Transient failures are usually a stale window id
@@ -4743,9 +4831,9 @@ export class Daemon extends EventEmitter {
                     ? "pasteBuffer failed — recovering window and backing off"
                     : "pasteBuffer failed — non-retryable tmux error");
                 if (!recoverable) {
-                    if (status)
-                        this.emit("message_failed", status);
-                    return false;
+                    // A tmux error the window cannot be recovered from: the text will
+                    // never reach this pane, so it is a verdict like the others.
+                    return this.failDelivery(verdict, status);
                 }
                 windowId = (await this.recoverWindow()) ?? windowId;
                 if (attempt < maxAttempts)
@@ -4775,9 +4863,7 @@ export class Daemon extends EventEmitter {
             }
             let enterAt = Date.now();
             if (!(await this.sendDeliveryEnter("initial-submit"))) {
-                if (status)
-                    this.emit("message_failed", status); // ❌
-                return false;
+                return this.failDelivery(verdict, status);
             }
             // Kiro's legacy TUI can swallow Enter while it is still processing a large
             // paste — not only during the post-ready redraw (#479): on slower hosts it
@@ -4860,9 +4946,7 @@ export class Daemon extends EventEmitter {
                     this.logger.warn("Message still in the input row after idle — submitting the existing text instead of pasting it again");
                     const strandedAt = Date.now();
                     if (!(await this.sendDeliveryEnter("native-queue-stranded-submit"))) {
-                        if (status)
-                            this.emit("message_failed", status); // ❌
-                        return false;
+                        return this.failDelivery(verdict, status);
                     }
                     const afterEnter = await this.confirmSubmitted(signature, pasteBaseline);
                     if (afterEnter === "submitted") {
@@ -4877,9 +4961,7 @@ export class Daemon extends EventEmitter {
                     // message nobody submitted. Output is corroboration; text sitting in
                     // the input row is disqualifying, and disqualifying evidence wins.
                     this.logger.error({ afterEnter, strandedAt }, "Stranded message could not be submitted by Enter");
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
                 // "unproven": nothing of ours is on screen — the paste itself was lost,
                 // so pasting it again cannot duplicate anything.
@@ -4888,16 +4970,12 @@ export class Daemon extends EventEmitter {
                     this.logger.error({
                         tmuxError: this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure",
                     }, "Idle-gated redelivery paste failed after native-queue silent loss");
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
                 await new Promise(r => setTimeout(r, NORMAL_ENTER_SETTLE_MS));
                 const retryAt = Date.now();
                 if (!(await this.sendDeliveryEnter("native-queue-idle-redelivery"))) {
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
                 if (windowId && this.controlClient) {
                     if (await this.confirmAfterEnter(windowId, retryAt, signature, pasteBaseline, "native-queue-idle-redelivery-retry")) {
@@ -4912,9 +4990,7 @@ export class Daemon extends EventEmitter {
                     return true;
                 }
                 this.logger.error("Idle-gated redelivery also failed after native-queue silent loss");
-                if (status)
-                    this.emit("message_failed", status); // ❌
-                return false;
+                return this.failDelivery(verdict, status);
             }
             if (windowId && this.controlClient && this.backend?.dropsEnterWhileBusy?.() === true) {
                 // F2: output after Enter is necessary but not sufficient — the paste
@@ -4928,9 +5004,7 @@ export class Daemon extends EventEmitter {
                     const promptBack = await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS);
                     const retryAt = Date.now();
                     if (promptBack && !(await this.sendDeliveryEnter("stranded-text-retry"))) {
-                        if (status)
-                            this.emit("message_failed", status); // ❌
-                        return false;
+                        return this.failDelivery(verdict, status);
                     }
                     submitted = promptBack && await this.confirmSubmittedAfterEnter(windowId, retryAt, formatted);
                 }
@@ -4940,9 +5014,7 @@ export class Daemon extends EventEmitter {
                 }
                 else {
                     this.logger.error("Message pasted but never submitted (text still in the input row after Enter retry)");
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
             }
             else if (windowId && this.controlClient) {
@@ -4967,9 +5039,7 @@ export class Daemon extends EventEmitter {
                     // forever and the next delivery pasted on top — submitting two messages
                     // as one. Say so instead.
                     this.logger.error("Message pasted but never submitted (no idle→busy after two Enters)");
-                    if (status)
-                        this.emit("message_failed", status); // ❌
-                    return false;
+                    return this.failDelivery(verdict, status);
                 }
             }
             else {
@@ -4982,9 +5052,7 @@ export class Daemon extends EventEmitter {
             return true;
         }
         this.logger.error("Message delivery failed after retries — window not ready");
-        if (status)
-            this.emit("message_failed", status); // ❌
-        return false;
+        return this.failDelivery(verdict, status);
     }
     /**
      * The single place a pasted message is judged submitted, shared by the