@songsid/agend 2.1.5-beta.12 → 2.1.5-beta.14

package/dist/daemon.js

@@ -23,7 +23,7 @@ import { writeSecretFile } from "./secret-file.js";
 import { PaneWriteLock } from "./pane-write-lock.js";
 import { buildFleetInstructions } from "./instructions.js";
 import { formatCrossInstanceInboundMessage, renderCrossInstanceHandoffMetadata, } from "./cross-instance-envelope.js";
-import { bottomRowIsReady, pasteLeftInInput, strandedAgendMessageInInput } from "./pane-input-residue.js";
+import { bottomRowIsReady, inputAreaText, inputShowsPastedText, pastedTextSignature, pasteLeftInInput, strandedAgendMessageInInput } from "./pane-input-residue.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 // Tool routing sets — module-level to avoid re-creation on every handleToolCall
@@ -386,6 +386,18 @@ export class PendingWorkTracker {
     }
 }
 const NORMAL_ENTER_SETTLE_MS = 500;
+/** Attempts to read the pre-paste pane before a delivery gives up on a baseline. */
+const BASELINE_CAPTURE_ATTEMPTS = 3;
+const BASELINE_CAPTURE_RETRY_MS = 150;
+/** Occurrences of `needle` in `haystack` (plain text, no regex semantics). */
+function countOccurrences(haystack, needle) {
+    if (!needle)
+        return 0;
+    let n = 0;
+    for (let i = haystack.indexOf(needle); i !== -1; i = haystack.indexOf(needle, i + needle.length))
+        n++;
+    return n;
+}
 /** Bottom-ready re-poll cadence for Enter-dropping TUIs once the pane is quiet. */
 const BOTTOM_READY_POLL_MS = 250;
 /** Consecutive unreadable pane probes tolerated by the delivery gate before the delivery is failed (≈10s at the poll cadence). */
@@ -1059,6 +1071,33 @@ export class Daemon extends EventEmitter {
     static errorPatternKey(ep) {
         return `${ep.type}:${ep.pattern.source}`;
     }
+    /**
+     * Undo the auth suspicion a pattern match armed, once the backend's token-free
+     * probe has said the credentials are fine.
+     *
+     * emitErrorPattern arms these BEFORE the pty_error reaches the lifecycle, and
+     * the lifecycle's "valid" verdict used to just drop the incident — leaving the
+     * daemon permanently suspicious of an auth failure that never existed. That
+     * state is not inert: authFailureUnresolved suppresses the stuck/hang
+     * notification and holds MCP auto-restart (a later real MCP death is then read
+     * as already-confirmed auth trouble rather than re-verified), and the recovery
+     * gate suppresses further error detection until a ready pattern shows up.
+     *
+     * Only what an auth match armed is rolled back. If some other pattern has
+     * since armed the recovery gate, that one is still live and stays.
+     */
+    clearSuspectedAuthFailure() {
+        if (!this.authFailureUnresolved && !this.loginScreenReported)
+            return false;
+        this.authFailureUnresolved = false;
+        this.loginScreenReported = false;
+        if (this.lastDetectedErrorType === "auth_error") {
+            this.clearErrorRecoveryGate();
+            this.lastDetectedErrorType = null;
+        }
+        this.logger.info("Auth suspicion withdrawn — the token-free probe reports valid credentials");
+        return true;
+    }
     clearErrorRecoveryGate() {
         this.errorWaitingForRecovery = false;
         this.errorDetectedAt = 0;
@@ -3539,7 +3578,7 @@ export class Daemon extends EventEmitter {
             await this.wake();
             if (!this.isDeliveryEpochCurrent(deliveryEpoch))
                 return;
-            if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
+            if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch, submissionId: meta.message_id })) {
                 this.markTurnStarted(meta, formatted);
             }
             else if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
@@ -3577,7 +3616,7 @@ export class Daemon extends EventEmitter {
             await this.wake();
             if (!this.isDeliveryEpochCurrent(deliveryEpoch))
                 return;
-            if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch })) {
+            if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch, submissionId: meta.message_id })) {
                 this.markTurnStarted(meta, formatted);
             }
         }).catch(err => {
@@ -3673,7 +3712,7 @@ export class Daemon extends EventEmitter {
                 // A fresh delivery begins a fresh turn — its bubble must not inherit
                 // the previous turn's tool list.
                 this.resetToolProgress();
-                if (await this.deliverMessage(formatted, status, { deliveryEpoch })) {
+                if (await this.deliverMessage(formatted, status, { deliveryEpoch, submissionId: meta.message_id })) {
                     this.markTurnStarted(meta, formatted);
                 }
                 else if (meta.from_instance && this.isDeliveryEpochCurrent(deliveryEpoch)) {
@@ -3841,7 +3880,7 @@ export class Daemon extends EventEmitter {
                     if (probe.state !== "clear")
                         return "dialog";
                 }
-                return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status);
+                return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId);
             });
             if (outcome !== "dialog")
                 return outcome;
@@ -4250,13 +4289,17 @@ export class Daemon extends EventEmitter {
         }
         return sent;
     }
-    async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status) {
+    async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status, submissionId) {
+        const signature = this.submissionSignature(formatted, submissionId);
         let windowId = initialWindowId;
         // Bug A: paste with backoff. Transient failures are usually a stale window id
         // after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
         const maxAttempts = 3;
         for (let attempt = 1; attempt <= maxAttempts; attempt++) {
             const pasteStartedAt = Date.now();
+            // Read the pane BEFORE writing to it, so the submission check can require
+            // evidence this paste ADDED rather than evidence that was already there.
+            const pasteBaseline = await this.capturePaneEvidence(signature);
             const pasted = await this.tmux.pasteBuffer(formatted);
             if (!pasted) {
                 const tmuxError = this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure";
@@ -4338,16 +4381,70 @@ export class Daemon extends EventEmitter {
             // can silently swallow it). Idle submissions keep the swallowed-Enter path.
             if (handingOffToNativeQueue) {
                 await new Promise(r => setTimeout(r, NATIVE_QUEUE_PASTE_VERIFY_MS));
-                if (await this.nativeQueuePasteVisible(formatted)) {
+                const proof = await this.confirmSubmitted(signature, pasteBaseline);
+                if (proof === "submitted") {
                     if (status)
                         this.emit("message_confirmed", status); // ✅ native queue accepted
                     return true;
                 }
-                // Silent loss: fall back once to the normal idle-gated path.
-                this.logger.warn("Native-queue paste not visible in pane — retrying via idle-gated delivery");
+                if (proof === "unverifiable") {
+                    // Our text is on the pane but cannot be shown to have left the input
+                    // row — either the backend exposes no input row (only /steer reaches
+                    // this path on such a backend) or the pre-paste pane was unreadable.
+                    // Re-pasting on that would deliver the message twice, so accept as
+                    // this path always has, and record that it is unproven.
+                    this.logger.warn("Paste reached the pane but could not be verified as submitted — accepting without proof");
+                    if (status)
+                        this.emit("message_confirmed", status); // ✅ (best-effort)
+                    return true;
+                }
+                // Not submitted (or not provably submitted): fall back once to the
+                // normal idle-gated path, which is the only one with a full
+                // confirmation ladder. "stranded" is the case that used to be a silent
+                // ✅ — the text was visible, so the old check called it delivered.
+                this.logger.warn({ proof }, "Native-queue paste not confirmed as submitted — retrying via idle-gated delivery");
                 if (windowId && this.controlClient) {
                     await this.controlClient.waitUntilIdle(windowId);
                 }
+                // Re-read before doing anything: the CLI had a whole turn to flush its
+                // queue while we waited, and the right recovery differs per outcome.
+                // Re-pasting a payload that is STILL in the input row appends it to
+                // itself — paste-buffer writes at the cursor — and the next Enter then
+                // submits the message twice over. That is the trap this fix would walk
+                // into if it treated "not confirmed" as one state.
+                const settled = await this.confirmSubmitted(signature, pasteBaseline);
+                if (settled === "submitted") {
+                    this.logger.info("Native-queue message submitted itself while we waited for idle");
+                    if (status)
+                        this.emit("message_confirmed", status); // ✅
+                    return true;
+                }
+                if (settled === "stranded") {
+                    // The text is already there in full: it needs submitting, not sending
+                    // again. The prompt is back now, so an Enter can land.
+                    this.logger.warn("Message still in the input row after idle — submitting the existing text instead of pasting it again");
+                    const strandedAt = Date.now();
+                    if (!(await this.sendDeliveryEnter("native-queue-stranded-submit"))) {
+                        if (status)
+                            this.emit("message_failed", status); // ❌
+                        return false;
+                    }
+                    const afterEnter = await this.confirmSubmitted(signature, pasteBaseline);
+                    const turnStarted = windowId && this.controlClient
+                        ? await this.confirmBusyAfterEnter(windowId, strandedAt)
+                        : false;
+                    if (afterEnter === "submitted" || turnStarted) {
+                        if (status)
+                            this.emit("message_confirmed", status); // ✅
+                        return true;
+                    }
+                    this.logger.error("Stranded message could not be submitted by Enter");
+                    if (status)
+                        this.emit("message_failed", status); // ❌
+                    return false;
+                }
+                // "unproven": nothing of ours is on screen — the paste itself was lost,
+                // so pasting it again cannot duplicate anything.
                 const repasted = await this.tmux.pasteBuffer(formatted);
                 if (!repasted) {
                     this.logger.error({
@@ -4382,7 +4479,7 @@ export class Daemon extends EventEmitter {
                         return true;
                     }
                 }
-                else if (await this.nativeQueuePasteVisible(formatted)) {
+                else if (await this.confirmSubmitted(signature, pasteBaseline) === "submitted") {
                     if (status)
                         this.emit("message_confirmed", status); // ✅
                     return true;
@@ -4463,31 +4560,195 @@ export class Daemon extends EventEmitter {
         return false;
     }
     /**
-     * True when a busy native-queue paste appears to have landed: Codex shows a
-     * `↳` queue marker, or a distinctive slice of the pasted text is on screen.
-     * Used only to detect silent paste loss — not as a general ready check.
+     * The single place a pasted message is judged submitted, shared by the
+     * native-queue handoff, the idle-gated redelivery and the system pastes.
+     *
+     * It replaces three divergent weak checks, the weakest of which accepted "the
+     * pasted text is visible somewhere in the pane". That one was satisfied BY
+     * the failure it was meant to catch: text stranded in the input row is on
+     * screen, so a message nobody submitted was confirmed ✅ with no warning
+     * anywhere. Evidence is weighed in order of what it can prove, and the
+     * disqualifying evidence is checked first.
      */
-    async nativeQueuePasteVisible(formatted) {
+    async confirmSubmitted(signature, baseline) {
         if (!this.tmux)
-            return false;
+            return "unproven";
+        let pane;
         try {
-            const pane = await this.tmux.capturePane();
-            if (pane.includes("↳"))
-                return true;
-            for (const line of formatted.split(/\r?\n/)) {
-                const t = line.trim();
-                if (t.length >= 8 && pane.includes(t))
-                    return true;
+            pane = await this.tmux.capturePane();
+        }
+        catch {
+            return "unproven";
+        }
+        // Without a way to tell the input row from the transcript, "the text is on
+        // screen" cannot distinguish submitted from stranded — that ambiguity IS
+        // the bug. Such a backend gets an explicit verdict rather than a guess:
+        // "unverifiable" (something of ours appeared, but not provably out of the
+        // input row) or "unproven" (nothing of ours appeared, so the paste itself
+        // was lost). The caller keeps that backend's existing best-effort handling
+        // for the first and recovers on the second, which is what it did before —
+        // the fix here is for backends that CAN be read, not a new guess for those
+        // that cannot.
+        const prompt = this.backend?.getBottomReadyPattern?.();
+        if (!prompt) {
+            const seen = this.paneEvidence(pane, signature);
+            if (signature.unique && seen.payload > 0)
+                return "unverifiable";
+            return seen.payload > (baseline?.payload ?? Infinity) || seen.queued > (baseline?.queued ?? Infinity)
+                ? "unverifiable"
+                : "unproven";
+        }
+        const after = this.paneEvidence(pane, signature);
+        // 1. Disqualifying evidence, checked FIRST and never overridden by the
+        //    corroborating evidence below: our text is sitting in the input row, so
+        //    it was not submitted — whatever else is on screen.
+        //
+        //    It must be OUR text. A unique signature settles that on its own; a
+        //    body-derived one cannot, so it is only attributed to this delivery
+        //    when the input row did not already show it before we pasted.
+        //    Otherwise an older stranded message with the same opening would be
+        //    read as ours, we would press Enter to "recover" it, and the turn IT
+        //    starts would confirm a message that never reached the pane.
+        if (after.strandedInput && (signature.unique || baseline?.strandedInput === false))
+            return "stranded";
+        // 2. Positive evidence. A unique signature needs no baseline: no earlier
+        //    message can carry this delivery's message_id, so finding it outside
+        //    the input row is proof in itself — which also means a momentary
+        //    failure to read the pane BEFORE pasting cannot turn a delivered
+        //    message into a re-paste.
+        if (signature.unique && after.payload > 0)
+            return "submitted";
+        // 3. Otherwise the evidence must be NEW relative to the pane as it was
+        //    before we pasted: a queue marker left by an earlier message, or an
+        //    older transcript entry that opens the same way, are on screen either
+        //    way and would otherwise confirm a paste that never landed.
+        if (baseline) {
+            if (after.queued > baseline.queued)
+                return "submitted";
+            if (after.payload > baseline.payload)
+                return "submitted";
+            // 4. Nothing new of ours anywhere: the paste was swallowed by a redraw.
+            return "unproven";
+        }
+        // No baseline and nothing that identifies this delivery on its own. "The
+        // paste was lost" is a guess, not a finding, and acting on it re-pastes a
+        // message that may well have been submitted — so say the evidence is
+        // missing instead of inventing a verdict.
+        this.logger.warn("Could not read the pane before pasting — this delivery cannot be verified either way");
+        return "unverifiable";
+    }
+    /**
+     * What the pane currently shows of a given message. Taken once before the
+     * paste and once after, so confirmSubmitted can require a NEW marker or a NEW
+     * echo rather than accepting whatever was already there.
+     */
+    async capturePaneEvidence(signature) {
+        if (!this.tmux)
+            return null;
+        // Bounded re-probe: a capture that fails here costs the delivery its only
+        // "before" picture, and a momentary failure should not decide anything.
+        for (let attempt = 1; attempt <= BASELINE_CAPTURE_ATTEMPTS; attempt++) {
+            try {
+                const pane = await this.tmux.capturePane();
+                const evidence = this.paneEvidence(pane, signature);
+                const prompt = this.backend?.getBottomReadyPattern?.();
+                if (prompt && strandedAgendMessageInInput(pane, prompt)) {
+                    // Whatever we paste now lands after it, and one Enter submits both as
+                    // a single message. Nothing here can undo that; saying so beats
+                    // letting two messages silently merge.
+                    this.logger.warn("An unsubmitted message is already in the input row — this delivery will be appended to it");
+                }
+                return evidence;
             }
-            const compact = formatted.replace(/\s+/g, " ").trim();
-            if (compact.length >= 8 && pane.includes(compact.slice(0, Math.min(80, compact.length)))) {
-                return true;
+            catch {
+                if (attempt < BASELINE_CAPTURE_ATTEMPTS)
+                    await new Promise(r => setTimeout(r, BASELINE_CAPTURE_RETRY_MS));
             }
+        }
+        // Still unreadable: confirmSubmitted falls back to evidence that stands on
+        // its own (a unique message_id) and refuses to guess without it.
+        this.logger.warn("Could not read the pane before pasting — no baseline for this delivery");
+        return null;
+    }
+    paneEvidence(pane, signature) {
+        const marker = this.backend?.getQueuedInputMarker?.();
+        const prompt = this.backend?.getBottomReadyPattern?.();
+        const input = prompt ? inputAreaText(pane, prompt) : null;
+        return {
+            queued: marker ? pane.split(/\r?\n/).filter(row => marker.test(row)).length : 0,
+            payload: countOccurrences(pane.replace(/\s+/g, ""), signature.value),
+            strandedInput: input != null && inputShowsPastedText(input, signature.value),
+        };
+    }
+    /**
+     * What identifies THIS message on screen. The routing envelope carries a
+     * message_id that no earlier transcript entry can share, so prefer it: the
+     * body alone is not distinctive enough — short repeated instructions ("carry
+     * on", a status ping) recur verbatim, and an older copy would vouch for a
+     * paste that never landed. System pastes carry no envelope and fall back to
+     * the body signature, which the baseline comparison still ties to this paste.
+     */
+    submissionSignature(formatted, trustedId) {
+        // The id must come from the message's own metadata, never from scanning the
+        // rendered text: agents and users discuss message ids in the body all the
+        // time ("check message_id: abc"), and a body-scanned value would match an
+        // older transcript entry quoting the same thing — then be trusted as unique
+        // and confirm a paste that never landed. Only the value AgEnD itself put in
+        // the handoff metadata identifies THIS delivery.
+        const id = trustedId?.trim();
+        if (id)
+            return { value: `message_id:${id}`, unique: true };
+        return { value: pastedTextSignature(formatted), unique: false };
+    }
+    /**
+     * Paste + submit + confirm for the SYSTEM messages that used to go out via
+     * tmux.pasteText: one Enter (two for queue-less backends) and no verification
+     * at all, so a swallowed Enter left the notice sitting in the input row where
+     * the next delivery submitted both as one message. Shares confirmSubmitted
+     * with the delivery path — having one primitive is the point.
+     *
+     * Best effort by design: these notices must never fail a delivery or throw.
+     * The caller already holds paneWriteLock.
+     */
+    async submitSystemPaste(text, label) {
+        if (!this.tmux)
+            return false;
+        const signature = this.submissionSignature(text);
+        const baseline = await this.capturePaneEvidence(signature);
+        if (!(await this.tmux.pasteBuffer(text))) {
+            this.logger.warn({ label }, "System paste failed to reach the pane");
             return false;
         }
-        catch {
+        await new Promise(r => setTimeout(r, NORMAL_ENTER_SETTLE_MS));
+        if (!(await this.sendDeliveryEnter(label)))
+            return false;
+        let proof = await this.confirmSubmitted(signature, baseline);
+        if (proof === "unverifiable") {
+            // No input row to read: this backend gets exactly what the old pasteText
+            // path gave it, including the unconditional second Enter for queue-less
+            // TUIs that swallow the first. Narrowing that to one Enter on the grounds
+            // that "the text is visible" would be the very inference this change
+            // exists to remove — visible text is also what a strand looks like.
+            if (this.systemPasteOptions().retryEnter) {
+                await new Promise(r => setTimeout(r, 1_000));
+                await this.sendDeliveryEnter(`${label}-defensive-retry`);
+            }
+            return true; // best effort, exactly as before — nothing here is verified
+        }
+        if (proof !== "submitted") {
+            // A bare Enter is a no-op at an empty prompt, so this is safe even if the
+            // first one did land; when the text is still in the input row it is the
+            // submit it never got. Re-pasting would append the text to itself.
+            await new Promise(r => setTimeout(r, NORMAL_ENTER_SETTLE_MS));
+            if (!(await this.sendDeliveryEnter(`${label}-retry`)))
+                return false;
+            proof = await this.confirmSubmitted(signature, baseline);
+        }
+        if (proof !== "submitted") {
+            this.logger.warn({ label, proof }, "System paste may not have been submitted");
             return false;
         }
+        return true;
     }
     /** Re-resolve this instance's tmux window by name (stale id after crash/respawn). */
     async recoverWindow() {
@@ -4969,7 +5230,16 @@ export class Daemon extends EventEmitter {
         try {
             // Messages can arrive during a restart and be queued on pasteLock before the
             // snapshot lands; both write to the pane, so both go through the same lock.
-            await this.paneWriteLock.run(() => this.tmux.pasteText(`[system:session-snapshot]\n${snapshot}\n\nThis is a background context restore — do NOT reply to or acknowledge this message. Simply resume normal operation when the next user or instance message arrives.`, this.systemPasteOptions()));
+            const restoreNotice = `[system:session-snapshot]\n${snapshot}\n\nThis is a background context restore — do NOT reply to or acknowledge this message. Simply resume normal operation when the next user or instance message arrives.`;
+            const injected = await this.paneWriteLock.run(() => this.submitSystemPaste(restoreNotice, "session-snapshot-restore"));
+            if (!injected) {
+                // rotation-state.json was deleted when the prompt was built, so there is
+                // nothing left to retry from: the restore is gone either way. Say so —
+                // reporting it as injected is how a lost context restore stays invisible.
+                this.logger.error("Session snapshot could not be submitted — session continues without context");
+                this.emit("snapshot_failed", this.name);
+                return;
+            }
             this.logger.info("Injected session snapshot as first message");
             this.emit("snapshot_injected", this.name);
         }
@@ -5391,9 +5661,16 @@ export class Daemon extends EventEmitter {
             // This path only runs when pasteQueueDepth > 0 — i.e. exactly when a real
             // delivery is already in flight or queued. Without the lock the notice and
             // that delivery race into the same pane.
-            await this.paneWriteLock.run(async () => {
-                await this.tmux?.pasteText(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), this.systemPasteOptions());
-            });
+            const told = await this.paneWriteLock.run(() => this.submitSystemPaste(buildInstructionReloadNotice(this.backend?.binaryName ?? "unknown", this.name, this.instanceDir), "instruction-reload-notice"));
+            if (!told) {
+                // Recording the new instructions here would mark the agent as told about
+                // a notice it never received, and every later restart would then skip
+                // the reload. Leave the snapshot behind and hand it to the deferred
+                // notice, which fires on the next real message.
+                this.logger.warn("Instruction reload notice not confirmed — deferring it to the next message");
+                this.pendingInstructionsNotice = true;
+                return;
+            }
             // Record the value the agent has now been told about so the next
             // unchanged restart skips the reload.
             try {