@songsid/agend 2.1.7-beta.3 → 2.1.7-beta.5

package/dist/daemon.js

@@ -1,7 +1,7 @@
 import { join, dirname, basename, resolve } from "node:path";
 import { mkdirSync, writeFileSync, readFileSync, readdirSync, existsSync, unlinkSync, rmSync, appendFileSync, statSync, renameSync } from "node:fs";
 import { fileURLToPath } from "node:url";
-import { createHash, randomBytes } from "node:crypto";
+import { createHash, randomBytes, randomUUID } from "node:crypto";
 import { EventEmitter } from "node:events";
 import { rotateLogIfNeeded } from "./logger.js";
 import { mcpServerState } from "./mcp-liveness.js";
@@ -30,10 +30,11 @@ import { bottomRowIsReady, inputAreaText, inputShowsPastedText, pastedTextSignat
 import { TurnReplyGuard } from "./turn-reply-guard.js";
 import { t } from "./locale.js";
 import { MuseUsageRelay, clearMuseUsageSnapshot } from "./muse-usage-relay.js";
+import { queueResumePolicyForAttempt } from "./delivery-queue-evidence.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 // Tool routing sets — module-level to avoid re-creation on every handleToolCall
-const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "list_instances", "start_instance", "restart_instance", "create_instance", "delete_instance", "replace_instance", "request_information", "delegate_task", "report_result", "describe_instance"]);
+const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "broadcast", "list_instances", "start_instance", "restart_instance", "create_instance", "delete_instance", "replace_instance", "request_information", "delegate_task", "report_result", "delivery_status", "describe_instance"]);
 const SCHEDULE_TOOLS = new Set(["create_schedule", "list_schedules", "update_schedule", "delete_schedule"]);
 const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decision"]);
 const TASK_TOOL = "task";
@@ -95,6 +96,8 @@ export const DEFAULT_STATE_SAFETY_SWEEP_MS = 60_000;
 export const PERIODIC_REDRAW_PROBE_MS = 25;
 /** Do not turn a streaming Codex turn into one capture-pane per burst. */
 export const STRUCTURED_IDLE_PROBE_MS = 500;
+/** How long an unrecognised-but-empty composer must stay unchanged before it counts as idle (#978). */
+export const UNKNOWN_LAYOUT_STABLE_MS = 10_000;
 /** A foreground server/port-forward should hand control back or be acknowledged. */
 export const DEFAULT_BLOCKING_PROCESS_GRACE_MS = 2 * 60_000;
 const LAST_INBOUND_FILE = "last-inbound-at";
@@ -326,9 +329,12 @@ export class PaneStateMachine {
      *   the caller has them.
      * @param opts.forceBusy the backend has positively identified a still-running
      *   foreground tool even though its TUI keeps an old ready footer visible.
+     * @param opts.forceReady the backend has positively proved an idle composer
+     *   the ready pattern cannot read (an unknown footer layout); the busy
+     *   pattern still vetoes it.
      */
     observe(pane, now = Date.now(), opts = {}) {
-        const { settled = false, changeAt = now, forceBusy = false } = opts;
+        const { settled = false, changeAt = now, forceBusy = false, forceReady = false } = opts;
         const paneHash = createHash("sha256").update(pane).digest("hex");
         const firstObservation = this.lastPaneHash === null;
         const paneChanged = this.lastPaneHash !== paneHash;
@@ -340,7 +346,9 @@ export class PaneStateMachine {
         // Some TUIs keep their ready footer visible while a foreground tool owns
         // stdin. Backends can positively identify that tool from the pane; in that
         // case the old prompt must not clear pending work or retire Cancel.
-        const ready = !forceBusy && this.isReady(pane);
+        // forceReady: the backend proved readiness itself (#978 stable unknown
+        // footer); a visible busy marker still vetoes it.
+        const ready = !forceBusy && (forceReady ? !this.busyPattern?.test(pane) : this.isReady(pane));
         const nextState = firstObservation
             ? ready ? "idle" : "working"
             : paneChanged && !settled
@@ -908,6 +916,9 @@ export class Daemon extends EventEmitter {
     spawnGate;
     stormWindow;
     backendOutage;
+    /** Identity of this live Daemon object; changes on object stop/restart. */
+    bootId = randomUUID();
+    deliveryOutbox;
     logger;
     tmuxSessionName;
     tmux = null;
@@ -996,6 +1007,10 @@ export class Daemon extends EventEmitter {
     /** Consecutive structural idle captures for a noisy, periodically-redrawing
      * pane.  Two are required before a working -> idle transition. */
     instanceStatePeriodicIdleConfirmations = 0;
+    /** #978: last unrecognised-but-empty-composer capture and since when it has been unchanged. */
+    unknownLayoutPaneKey = null;
+    unknownLayoutStableSince = 0;
+    unknownLayoutWarned = false;
     instanceStateStuckTimer = null;
     instanceStateOutputListener = null;
     instanceStateOutputEventName = null;
@@ -1081,6 +1096,7 @@ export class Daemon extends EventEmitter {
     /** Mutual exclusion for *every* write into the pane, whichever subsystem it
      *  comes from. See PaneWriteLock for why interleaving is destructive. */
     paneWriteLock = new PaneWriteLock();
+    deliveryWritesStopping = false;
     pendingInstructionsUpdate;
     pendingInstructionsNotice = false;
     // Whether the warmup steering-reload notice should be injected after spawn.
@@ -1313,6 +1329,109 @@ export class Daemon extends EventEmitter {
         const autoPauseMinutes = isGeneral ? 0 : (typeof config.auto_pause_after === "number" ? config.auto_pause_after : 0); // default: disabled
         this.autoPauseController = new AutoPauseController(Math.max(0, autoPauseMinutes) * 60_000, readLastInboundAt(instanceDir) ?? Date.now());
     }
+    /** FleetManager and this daemon share a process, so durable begin/ACK is a direct synchronous store call. */
+    setDeliveryOutboxPort(port) {
+        this.deliveryOutbox = port;
+    }
+    /** The fleet's other same-backend instances' working directories, read at spawn (#984). */
+    peerWorkingDirectories;
+    setPeerWorkingDirectories(peers) {
+        this.peerWorkingDirectories = peers;
+    }
+    /** Fence pane writers before lifecycle captures pre-kill reconciliation evidence. */
+    fenceDeliveryWritesForStop() {
+        if (this.deliveryWritesStopping)
+            return;
+        this.deliveryWritesStopping = true;
+        this.deliveryEpoch++;
+    }
+    /** Bounded drain: timeout remains safe because Enter has a write-ahead fence. */
+    waitForDeliveryWritesToDrain(timeoutMs) {
+        return this.paneWriteLock.waitForIdle(timeoutMs);
+    }
+    durableDeliveryAttempt(meta) {
+        const deliveryId = meta.delivery_id;
+        if (!deliveryId)
+            return null;
+        const attemptNo = Number(meta.delivery_attempt);
+        if (!Number.isSafeInteger(attemptNo) || attemptNo < 1)
+            return false;
+        return { deliveryId, attemptNo };
+    }
+    beginDurableDelivery(delivery, evidence) {
+        if (!this.deliveryOutbox) {
+            this.emit("durable_delivery_deferred", { ...delivery, targetBootId: this.bootId, reason: "outbox port unavailable before pane write" });
+            return false;
+        }
+        const outcome = this.deliveryOutbox.begin(delivery.deliveryId, this.bootId, delivery.attemptNo, evidence);
+        // A repeated begin is idempotent at the store but must not authorize a
+        // second pane write. Only the first caller that acquired the permit acts.
+        return outcome === "begun";
+    }
+    async durableAttemptEvidence(windowId, handingOffToNativeQueue, steer, rawPaste = false) {
+        let checkpoint = null;
+        try {
+            checkpoint = await this.transcriptMonitor?.reconciliationCheckpoint() ?? null;
+        }
+        catch (err) {
+            this.logger.debug({ err }, "Could not checkpoint transcript before durable delivery");
+        }
+        const submissionMode = rawPaste ? "raw_paste" : steer
+            ? "steer"
+            : handingOffToNativeQueue ? "native_queue_handoff" : "idle_submit";
+        const backend = this.config.backend ?? this.backend?.binaryName ?? "unknown";
+        // The daemon does not synchronously probe CLI versions on the delivery
+        // path. Until lifecycle supplies a version tied to this exact process,
+        // queue handoffs persist an unknown resume contract and reconciliation
+        // must fail closed rather than infer loss from transcript absence.
+        const backendVersion = null;
+        return {
+            backend,
+            backendVersion,
+            windowId: windowId ?? this.tmux?.getWindowId() ?? null,
+            transcriptPath: checkpoint?.path ?? null,
+            transcriptOffset: checkpoint?.offset ?? null,
+            transcriptSessionId: checkpoint?.sessionId ?? null,
+            submissionMode,
+            queueResumePolicy: queueResumePolicyForAttempt(backend, backendVersion, submissionMode),
+        };
+    }
+    abortDurableDelivery(delivery, reason) {
+        if (!delivery)
+            return;
+        this.deliveryOutbox?.abort(delivery.deliveryId, this.bootId, delivery.attemptNo, reason);
+    }
+    retryDurableDeliveryBeforeBegin(delivery, reason) {
+        if (!delivery)
+            return;
+        if (this.deliveryOutbox) {
+            this.deliveryOutbox.retryBeforeBegin(delivery.deliveryId, this.bootId, delivery.attemptNo, reason, 5_000);
+        }
+        else {
+            this.emit("durable_delivery_deferred", { ...delivery, targetBootId: this.bootId, reason });
+        }
+    }
+    finishDurableDelivery(delivery, outcome, evidence) {
+        if (!delivery)
+            return;
+        this.deliveryOutbox?.complete(delivery.deliveryId, this.bootId, delivery.attemptNo, outcome, evidence);
+    }
+    finishDurableSubmission(delivery, verdict) {
+        // Legacy backends may accept a pane write without enough pane evidence to
+        // claim submission. Keep that row inspectable instead of recording a false
+        // delivered state; reconciliation can classify it in a later phase.
+        if (verdict.proof === "unverified") {
+            this.finishDurableDelivery(delivery, "uncertain", `${verdict.phase ?? "submission"}:unverified`);
+            return;
+        }
+        this.finishDurableDelivery(delivery, "delivered", "positive submission proof");
+    }
+    deferDurableDelivery(meta, reason) {
+        const delivery = this.durableDeliveryAttempt(meta);
+        if (delivery === null || delivery === false)
+            return;
+        this.retryDurableDeliveryBeforeBegin(delivery, reason);
+    }
     async start() {
         mkdirSync(this.instanceDir, { recursive: true });
         writeFileSync(join(this.instanceDir, "daemon.pid"), String(process.pid));
@@ -1411,12 +1530,12 @@ export class Daemon extends EventEmitter {
                 void this.wake().then(() => {
                     this.pushChannelMessage(msg.content, meta, targetSession, deliveryEpoch);
                 }).catch(err => {
+                    this.deferDurableDelivery(meta, `target wake failed before pane submission: ${err.message}`);
                     this.logger.error({ err: err.message }, "Wake failed for inbound delivery");
                 });
             }
             else if (msg.type === "raw_paste") {
-                // Paste raw text directly to CLI without [user:] wrapping.
-                this.queueRawPaste(msg.content, this.captureDeliveryEpoch(msg.delivery_epoch), msg.confirm_clear === true);
+                this.handleRawPasteIpcMessage(msg);
             }
             else if (msg.type === "config_update") {
                 this.applyConfigUpdate(msg.config);
@@ -2691,24 +2810,76 @@ export class Daemon extends EventEmitter {
     isDeliveryEpochCurrent(epoch) {
         return epoch === this.deliveryEpoch;
     }
-    queueRawPaste(rawText, deliveryEpoch = this.deliveryEpoch, confirmClear = false) {
-        if (!this.tmux || !this.isDeliveryEpochCurrent(deliveryEpoch))
+    queueRawPaste(rawText, deliveryEpoch = this.deliveryEpoch, confirmClear = false, durableAttempt) {
+        if (!this.tmux || !this.isDeliveryEpochCurrent(deliveryEpoch)) {
+            if (durableAttempt)
+                this.retryDurableDeliveryBeforeBegin(durableAttempt, "raw_paste target unavailable before pane queue");
             return;
+        }
         this.pasteLock = this.pasteLock.then(async () => {
             if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
                 this.logger.info("Pending raw delivery dropped by user cancel");
+                if (durableAttempt)
+                    this.retryDurableDeliveryBeforeBegin(durableAttempt, "raw_paste epoch changed while queued");
                 return;
             }
-            await this.wake();
-            if (!this.isDeliveryEpochCurrent(deliveryEpoch))
+            const verdict = { reached: false };
+            try {
+                await this.wake();
+                if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
+                    if (durableAttempt)
+                        this.retryDurableDeliveryBeforeBegin(durableAttempt, "raw_paste epoch changed while waking");
+                    return;
+                }
+                const delivered = await this.deliverMessage(rawText, undefined, {
+                    deliveryEpoch,
+                    verdict,
+                    durableAttempt,
+                    preserveExactBytes: true,
+                });
+                if (delivered)
+                    this.finishDurableSubmission(durableAttempt ?? null, verdict);
+                else if (durableAttempt && verdict.paneWriteStarted) {
+                    this.finishDurableDelivery(durableAttempt, "uncertain", `${verdict.phase ?? "raw-paste"}:${verdict.proof ?? "unverified"}`);
+                }
+                else if (durableAttempt && verdict.durableBeginCommitted) {
+                    this.abortDurableDelivery(durableAttempt, `${verdict.phase ?? "raw-paste"}:${verdict.proof ?? "no-pane-write"}`);
+                }
+                else if (durableAttempt) {
+                    this.retryDurableDeliveryBeforeBegin(durableAttempt, `${verdict.phase ?? "raw-paste"}:${verdict.proof ?? "deferred"}`);
+                }
+            }
+            catch (err) {
+                if (durableAttempt && verdict.paneWriteStarted)
+                    this.finishDurableDelivery(durableAttempt, "uncertain", err.message);
+                else if (durableAttempt && verdict.durableBeginCommitted)
+                    this.abortDurableDelivery(durableAttempt, err.message);
+                else if (durableAttempt)
+                    this.retryDurableDeliveryBeforeBegin(durableAttempt, err.message);
+                this.logger.warn({ err: err.message }, "raw_paste delivery error");
                 return;
-            await this.deliverMessage(rawText, undefined, { deliveryEpoch });
+            }
             if (confirmClear)
                 await this.confirmBackendClearDialog();
             this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
-        }).catch(err => {
-            this.logger.warn({ err: err.message }, "raw_paste delivery error");
         });
+        this.pasteLock = this.pasteLock.catch(err => {
+            this.logger.warn({ err: err.message }, "raw_paste delivery queue error");
+        });
+    }
+    /** Keep the IPC raw route's durable mode attached to the parsed attempt. */
+    handleRawPasteIpcMessage(msg) {
+        // Paste raw text directly to CLI without [user:] wrapping.
+        const deliveryMeta = {
+            delivery_id: typeof msg.delivery_id === "string" ? msg.delivery_id : "",
+            delivery_attempt: typeof msg.delivery_attempt === "string" ? msg.delivery_attempt : "",
+        };
+        const durableAttempt = this.durableDeliveryAttempt(deliveryMeta);
+        if (durableAttempt === false) {
+            this.deferDurableDelivery(deliveryMeta, "invalid raw_paste delivery attempt metadata");
+            return;
+        }
+        this.queueRawPaste(msg.content, this.captureDeliveryEpoch(msg.delivery_epoch), msg.confirm_clear === true, durableAttempt ? { ...durableAttempt, submissionMode: "raw_paste" } : undefined);
     }
     /**
      * Confirm a backend's terminal-level /clear prompt after the channel-level
@@ -2856,6 +3027,7 @@ export class Daemon extends EventEmitter {
     }
     async stop() {
         this.logger.info("Stopping daemon instance");
+        this.fenceDeliveryWritesForStop();
         this.turnReplyGuard.reset();
         // Invalidate any bounded pre-Enter wait from the process generation being
         // stopped. It must fail closed, not press Enter in a replacement pane.
@@ -3699,9 +3871,39 @@ export class Daemon extends EventEmitter {
                 }
                 else {
                     // A live busy marker or an unfamiliar layout outranks the broad Codex
-                    // ready pattern during the noisy-redraw path. Unknown is working.
+                    // ready pattern during the noisy-redraw path. Unknown is working —
+                    // but not forever (#978): a footer the proof does not recognise
+                    // would otherwise pin an idle CLI in `working` and queue every
+                    // delivery indefinitely. If the backend still sees its own live,
+                    // empty composer and the pane has not changed (cosmetic glyphs
+                    // aside) for UNKNOWN_LAYOUT_STABLE_MS, read it as settled and ready.
                     this.instanceStatePeriodicIdleConfirmations = 0;
-                    snapshot = this.instanceStateMachine.recordOutput(observedChangeAt);
+                    const stableFallback = this.backend?.isStableUnknownLayoutIdlePane;
+                    const now = Date.now();
+                    if (stableFallback?.call(this.backend, pane)) {
+                        const key = pane.replace(/⋆/gu, "").replace(/[ \t]+$/gmu, "");
+                        if (key !== this.unknownLayoutPaneKey) {
+                            this.unknownLayoutPaneKey = key;
+                            this.unknownLayoutStableSince = now;
+                        }
+                    }
+                    else {
+                        this.unknownLayoutPaneKey = null;
+                    }
+                    if (this.unknownLayoutPaneKey !== null && now - this.unknownLayoutStableSince >= UNKNOWN_LAYOUT_STABLE_MS) {
+                        if (!this.unknownLayoutWarned) {
+                            this.unknownLayoutWarned = true;
+                            this.logger.warn({ backend: this.backend?.binaryName }, "Unrecognised idle footer layout; falling back to a stable empty composer for readiness");
+                        }
+                        snapshot = this.instanceStateMachine.observe(pane, now, {
+                            settled: true,
+                            changeAt: observedChangeAt,
+                            forceReady: true,
+                        });
+                    }
+                    else {
+                        snapshot = this.instanceStateMachine.recordOutput(observedChangeAt);
+                    }
                 }
             }
             else if (structuredPeriodicIdle && reason !== "idle_debounce" && !settled) {
@@ -4182,6 +4384,9 @@ export class Daemon extends EventEmitter {
         if (meta.reply_to_text) {
             formatted += `\n(reply_to: "${meta.reply_to_text}")`;
         }
+        if (meta.delivery_id) {
+            formatted = `[agend-delivery-id:${meta.delivery_id}]\n${formatted}`;
+        }
         return formatted;
     }
     /**
@@ -4204,7 +4409,10 @@ export class Daemon extends EventEmitter {
         verdict.reached = true;
         verdict.phase = phase;
         verdict.proof = proof;
-        if (status)
+        // Durable cross-instance rows are still retryable before a pane write. Their
+        // terminal state is announced by the outbox only after its retry/TTL policy
+        // is exhausted; emitting ❌ here would expose a transient gate timeout.
+        if (status && !verdict.durableAttempt)
             this.emit("message_failed", status); // ❌
         return false;
     }
@@ -4240,8 +4448,10 @@ export class Daemon extends EventEmitter {
      * normal inbound message, with a steering notice prepended for context.
      */
     steerMessage(content, meta, deliveryEpoch = this.deliveryEpoch) {
-        if (!this.isDeliveryEpochCurrent(deliveryEpoch))
+        if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
+            this.deferDurableDelivery(meta, "delivery epoch changed before steer began");
             return;
+        }
         // The fleet facade prevents a steer from overtaking an item still in its
         // idle queue. Close the second race window here: an earlier item may already
         // have reached this daemon's paste queue by the time the steer IPC arrives.
@@ -4256,25 +4466,60 @@ export class Daemon extends EventEmitter {
         const formatted = "[STEERING — mid-task course correction. Fold this into the CURRENT work if one is active.]\n"
             + this.formatInboundMessage(content, meta);
         const status = channelStatus(meta);
+        const durableAttempt = this.durableDeliveryAttempt(meta);
+        if (durableAttempt === false) {
+            this.deferDurableDelivery(meta, "invalid durable delivery attempt metadata");
+            return;
+        }
         this.steerLock = this.steerLock.then(async () => {
-            if (!this.isDeliveryEpochCurrent(deliveryEpoch))
+            if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
+                this.deferDurableDelivery(meta, "delivery epoch changed before steer began");
                 return;
+            }
             await this.wake();
-            if (!this.isDeliveryEpochCurrent(deliveryEpoch))
+            if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
+                this.deferDurableDelivery(meta, "delivery epoch changed while waking for steer");
                 return;
+            }
             const verdict = { reached: false };
-            if (await this.deliverMessage(formatted, status, { steer: true, deliveryEpoch, submissionId: meta.message_id, verdict })) {
-                this.markTurnStarted(meta, formatted);
+            try {
+                if (await this.deliverMessage(formatted, status, {
+                    steer: true,
+                    deliveryEpoch,
+                    submissionId: meta.message_id,
+                    verdict,
+                    durableAttempt: durableAttempt || undefined,
+                })) {
+                    this.finishDurableSubmission(durableAttempt, verdict);
+                    this.markTurnStarted(meta, formatted);
+                }
+                else if (durableAttempt && verdict.paneWriteStarted) {
+                    this.finishDurableDelivery(durableAttempt, "uncertain", `${verdict.phase ?? "post-submit"}:${verdict.proof ?? "unproven"}`);
+                }
+                else if (durableAttempt && verdict.durableBeginCommitted) {
+                    this.abortDurableDelivery(durableAttempt, `${verdict.phase ?? "delivery"}:${verdict.proof ?? "no-pane-write"}`);
+                }
+                else if (durableAttempt) {
+                    this.retryDurableDeliveryBeforeBegin(durableAttempt, `${verdict.phase ?? "delivery"}:${verdict.proof ?? "deferred"}`);
+                }
+                else if (verdict.reached && this.isDeliveryEpochCurrent(deliveryEpoch)) {
+                    // Legacy non-outbox path retains its existing failure surface.
+                    this.reportCrossInstanceDeliveryFailure(meta, verdict);
+                }
             }
-            else if (verdict.reached && this.isDeliveryEpochCurrent(deliveryEpoch)) {
-                // Same rule as the queued path: a steer that never got to try is not a
-                // delivery failure. This holder is the steer's own, which is the point
-                // — it runs on steerLock while a queued delivery runs on pasteLock.
-                this.reportCrossInstanceDeliveryFailure(meta, verdict);
+            catch (err) {
+                if (durableAttempt && verdict.paneWriteStarted)
+                    this.finishDurableDelivery(durableAttempt, "uncertain", err.message);
+                else if (durableAttempt && verdict.durableBeginCommitted)
+                    this.abortDurableDelivery(durableAttempt, err.message);
+                else if (durableAttempt)
+                    this.retryDurableDeliveryBeforeBegin(durableAttempt, err.message);
+                throw err;
             }
         }).catch(err => {
+            this.deferDurableDelivery(meta, `target wake/steer failed before pane submission: ${err.message}`);
             this.logger.warn({ err: err.message }, "steer delivery error");
-            if (this.isDeliveryEpochCurrent(deliveryEpoch)) {
+            if (!durableAttempt && this.isDeliveryEpochCurrent(deliveryEpoch)) {
                 this.reportCrossInstanceDeliveryFailure(meta, undefined, err.message);
             }
         });
@@ -4315,10 +4560,12 @@ export class Daemon extends EventEmitter {
     pushChannelMessage(content, meta, _targetSession, deliveryEpoch = this.deliveryEpoch) {
         if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
             this.logger.info("Pending channel delivery dropped by user cancel");
+            this.deferDurableDelivery(meta, "delivery epoch changed before queue admission");
             return;
         }
         if (!this.tmux) {
             this.logger.warn("Cannot push channel message: tmux not running");
+            this.deferDurableDelivery(meta, "target tmux is unavailable");
             return;
         }
         // Remember (and persist) the reply target. Only real channel messages have a
@@ -4375,9 +4622,12 @@ export class Daemon extends EventEmitter {
                 this.emit("message_queued", queuedStatus);
         }
         this.pasteLock = this.pasteLock.then(async () => {
+            let durable = null;
+            let verdict = { reached: false };
             try {
                 if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
                     this.logger.info("Pending channel delivery dropped by user cancel");
+                    this.deferDurableDelivery(meta, "delivery epoch changed while queued");
                     return;
                 }
                 if (this.config.pre_task_command) {
@@ -4395,18 +4645,60 @@ export class Daemon extends EventEmitter {
                 // A fresh delivery begins a fresh turn — its bubble must not inherit
                 // the previous turn's tool list.
                 this.resetToolProgress();
-                const verdict = { reached: false };
-                if (await this.deliverMessage(formatted, status, { deliveryEpoch, submissionId: meta.message_id, verdict })) {
+                verdict = { reached: false };
+                const durableAttempt = this.durableDeliveryAttempt(meta);
+                if (durableAttempt === false) {
+                    this.deferDurableDelivery(meta, "invalid durable delivery attempt metadata");
+                    return;
+                }
+                durable = durableAttempt;
+                if (await this.deliverMessage(formatted, status, {
+                    deliveryEpoch,
+                    submissionId: meta.message_id,
+                    verdict,
+                    durableAttempt: durableAttempt || undefined,
+                })) {
+                    this.finishDurableSubmission(durable, verdict);
                     this.markTurnStarted(meta, formatted);
                 }
+                else if (durableAttempt && verdict.paneWriteStarted) {
+                    this.finishDurableDelivery(durable, "uncertain", `${verdict.phase ?? "post-submit"}:${verdict.proof ?? "unproven"}`);
+                }
+                else if (durableAttempt && verdict.durableBeginCommitted) {
+                    this.abortDurableDelivery(durable, `${verdict.phase ?? "delivery"}:${verdict.proof ?? "no-pane-write"}`);
+                }
+                else if (durableAttempt) {
+                    this.retryDurableDeliveryBeforeBegin(durableAttempt, `${verdict.phase ?? "delivery"}:${verdict.proof ?? "deferred"}`);
+                }
                 else if (meta.from_instance && verdict.reached
                     && this.isDeliveryEpochCurrent(deliveryEpoch)) {
                     // Only a delivery that reached a verdict is reported. A pane that is
                     // not ready yet, a cancel, a storm hold or a shutdown all return
                     // false without one, and telling the sender its message was lost
                     // there would be the false ❌ of #826 in its other form.
+                    this.finishDurableDelivery(durable, "failed", `${verdict.phase ?? "delivery"}:${verdict.proof ?? "failed"}`);
                     this.reportCrossInstanceDeliveryFailure(meta, verdict);
                 }
+                else if (verdict.reached) {
+                    this.finishDurableDelivery(durable, "failed", `${verdict.phase ?? "delivery"}:${verdict.proof ?? "failed"}`);
+                }
+                else if (verdict.paneWriteStarted) {
+                    this.finishDurableDelivery(durable, "uncertain", `${verdict.phase ?? "post-submit"}:${verdict.proof ?? "unproven"}`);
+                }
+                else {
+                    this.abortDurableDelivery(durable, "delivery did not reach pane submission");
+                }
+            }
+            catch (err) {
+                if (durable && verdict.paneWriteStarted)
+                    this.finishDurableDelivery(durable, "uncertain", err.message);
+                else if (durable && verdict.durableBeginCommitted)
+                    this.abortDurableDelivery(durable, err.message);
+                else if (durable)
+                    this.retryDurableDeliveryBeforeBegin(durable, err.message);
+                else
+                    throw err;
+                this.logger.warn({ err: err.message }, "Durable pane delivery deferred after exception");
             }
             finally {
                 this.pasteQueueDepth--;
@@ -4439,13 +4731,15 @@ export class Daemon extends EventEmitter {
         // The caller passes its own holder when it needs the answer; a system paste
         // that ignores the outcome gets a throwaway.
         const verdict = opts?.verdict ?? { reached: false };
+        if (opts?.durableAttempt)
+            verdict.durableAttempt = true;
         const cancelled = () => opts?.deliveryEpoch !== undefined
             && !this.isDeliveryEpochCurrent(opts.deliveryEpoch);
         if (cancelled() || this.stormWindow?.isStopped())
             return false;
         // Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
         const fenceCount = (formatted.match(/```/g) || []).length;
-        if (fenceCount % 2 !== 0) {
+        if (!opts?.preserveExactBytes && fenceCount % 2 !== 0) {
             // Odd number of fences = unclosed. Remove all code fences from the message.
             formatted = formatted.replace(/```/g, "");
         }
@@ -4513,7 +4807,9 @@ export class Daemon extends EventEmitter {
                     // The pane never freed up. Report the failure instead of pasting into a
                     // wedged CLI (where the text would sit unsubmitted and the next message
                     // would land on top of it) — and instead of holding the queue silently.
-                    this.logger.error("Pane still busy after the idle wait — reporting delivery failure");
+                    this.logger[verdict.durableAttempt ? "warn" : "error"](verdict.durableAttempt
+                        ? "Pane still busy after the idle wait — durable delivery remains retryable"
+                        : "Pane still busy after the idle wait — reporting delivery failure");
                     return this.failDelivery(verdict, status, "readiness", "timeout-before-write");
                 }
             }
@@ -4591,7 +4887,7 @@ export class Daemon extends EventEmitter {
                 }
                 if (!(await this.hasPositiveDeliveryInput()))
                     return "dialog";
-                return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId, verdict);
+                return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId, verdict, opts?.durableAttempt, opts?.steer === true);
             });
             if (outcome === "spawn-started") {
                 // The pane changed under this delivery: its queued paste must not land
@@ -5345,13 +5641,20 @@ export class Daemon extends EventEmitter {
      * precisely so the lock's scope is visible at the call site rather than being
      * an invariant maintained by comments.
      */
-    async sendDeliveryEnter(phase, stillCurrent) {
+    async sendDeliveryEnter(phase, stillCurrent, durableAttempt) {
         if (!(await this.waitForInputTransientToClear(phase)))
             return false;
         // A recovery Enter may have waited for a transient while cancel or spawn
         // replaced the delivery. Check at the last point before the tmux write.
-        if (stillCurrent && !stillCurrent())
+        if (this.deliveryWritesStopping || (stillCurrent && !stillCurrent()))
+            return false;
+        // W1: persist the "Enter may have happened" boundary before tmux receives
+        // the key. A crash after this commit is never auto-replayed without a
+        // positive transcript marker, even if the stale pane still shows composer.
+        if (durableAttempt && !this.deliveryOutbox?.markEnterStarted(durableAttempt.deliveryId, this.bootId, durableAttempt.attemptNo)) {
+            this.logger.error({ phase, deliveryId: durableAttempt.deliveryId }, "Could not commit durable Enter write-ahead fence");
             return false;
+        }
         const sent = await this.tmux.sendSpecialKey("Enter");
         if (!sent) {
             this.logger.error({
@@ -5364,19 +5667,46 @@ export class Daemon extends EventEmitter {
     async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status, submissionId, 
     // Last and defaulted so the positional callers that ignore the outcome stay
     // readable; deliverMessage, the only one that reports, always passes its own.
-    verdict = { reached: false }) {
+    verdict = { reached: false }, durableAttempt, steer = false) {
         const signature = this.submissionSignature(formatted, submissionId);
+        const rawPaste = durableAttempt?.submissionMode === "raw_paste";
         let windowId = initialWindowId;
         // Bug A: paste with backoff. Transient failures are usually a stale window id
         // after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
         const maxAttempts = 3;
+        let beginChecked = false;
         for (let attempt = 1; attempt <= maxAttempts; attempt++) {
             const pasteStartedAt = Date.now();
             // Read the pane BEFORE writing to it, so the submission check can require
             // evidence this paste ADDED rather than evidence that was already there.
             const pasteBaseline = await this.capturePaneEvidence(signature);
+            // Readiness, idle-gate, dialog, spawn and pane-lock waits have all ended.
+            // Commit the submission fence at the last possible point before the
+            // first side effect; a crash during those waits remains safely replayable.
+            if (durableAttempt && !beginChecked) {
+                beginChecked = true;
+                const attemptEvidence = await this.durableAttemptEvidence(windowId, handingOffToNativeQueue, steer, durableAttempt.submissionMode === "raw_paste");
+                if (!this.beginDurableDelivery(durableAttempt, attemptEvidence)) {
+                    verdict.durableBeginRejected = true;
+                    verdict.phase = "submission-begin";
+                    verdict.proof = "permit-rejected";
+                    return false;
+                }
+                verdict.durableBeginCommitted = true;
+            }
+            if (rawPaste)
+                verdict.paneWriteStarted = true;
+            // For raw commands the paste itself may alter the live composer. Treat a
+            // failed/throwing write as uncertain rather than retrying bytes into the
+            // same pane; the write-ahead Enter marker later separates safe pre-Enter
+            // recovery from a possibly executed command.
             const pasted = await this.tmux.pasteBuffer(formatted);
             if (!pasted) {
+                if (rawPaste) {
+                    verdict.phase = "raw-paste-write";
+                    verdict.proof = "paste-result-unknown";
+                    return false;
+                }
                 const tmuxError = this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure";
                 const recoverable = this.tmux.isLastPasteFailureRecoverable?.() ?? true;
                 this.logger[recoverable ? "warn" : "error"]({ attempt, tmuxError, recoverable }, recoverable
@@ -5392,6 +5722,10 @@ export class Daemon extends EventEmitter {
                     await new Promise(r => setTimeout(r, 2000));
                 continue;
             }
+            // From this point a process/window transition can leave the user payload
+            // in the pane even if later proof is cancelled or unreadable. Such a row
+            // must be uncertain, never aborted back to replayable queued state.
+            verdict.paneWriteStarted = true;
             // Settle the bracketed paste, then submit. When control mode can observe
             // the pane, wait for the paste's own render to go quiet instead of
             // trusting a fixed delay (see waitForPasteSettle). The fixed delays —
@@ -5414,7 +5748,7 @@ export class Daemon extends EventEmitter {
                 settle = { settleMs: fallbackMs, observedPostPasteOutput: false, capHit: false, usedFallback: true };
             }
             let enterAt = Date.now();
-            if (!(await this.sendDeliveryEnter("initial-submit"))) {
+            if (!(await this.sendDeliveryEnter("initial-submit", undefined, durableAttempt))) {
                 return this.failDelivery(verdict, status, "submit-enter", "tmux-send-keys-failed");
             }
             // Kiro's legacy TUI can swallow Enter while it is still processing a large
@@ -5430,7 +5764,7 @@ export class Daemon extends EventEmitter {
             // re-baselined to the second Enter so leftover paste-render output between
             // the two cannot be what "confirms" the submission.
             let enterRetry = false;
-            if (this.backend?.requiresDeliveryEnterRetry?.() === true) {
+            if (!rawPaste && this.backend?.requiresDeliveryEnterRetry?.() === true) {
                 await new Promise(r => setTimeout(r, 1_000));
                 const retryAt = Date.now();
                 if (await this.sendDeliveryEnter("queue-less-defensive-retry")) {
@@ -5475,10 +5809,19 @@ export class Daemon extends EventEmitter {
                     // Re-pasting on a backend without structured Codex pane evidence
                     // would risk duplication; retain its legacy best-effort behavior.
                     this.logger.warn("Paste reached the pane but could not be verified as submitted — accepting without proof");
+                    verdict.phase = "best-effort-submission";
+                    verdict.proof = "unverified";
                     if (status)
                         this.emit("message_confirmed", status); // ✅ (best-effort)
                     return true;
                 }
+                // raw_paste has no unique marker and is not allowed to re-paste or
+                // issue another Enter after its one committed submission attempt.
+                if (rawPaste) {
+                    verdict.phase = "raw-paste-native-queue-proof";
+                    verdict.proof = proof;
+                    return false;
+                }
                 // Not submitted (or not provably submitted): fall back once to the
                 // normal idle-gated path, which is the only one with a full
                 // confirmation ladder. "stranded" is the case that used to be a silent
@@ -5588,6 +5931,11 @@ export class Daemon extends EventEmitter {
                     }
                 }
                 if (!submitted) {
+                    if (rawPaste) {
+                        verdict.phase = "raw-paste-submit-proof";
+                        verdict.proof = "no-positive-acceptance-proof";
+                        return false;
+                    }
                     // Legacy system pastes have no unique envelope id. Preserve their
                     // existing bounded retry until they can be tied to a specific turn.
                     this.logger.warn("Message not submitted after Enter — waiting for the prompt, then re-sending Enter once");
@@ -5618,7 +5966,7 @@ export class Daemon extends EventEmitter {
                 // is disqualifying and no amount of output may override it. Where it
                 // cannot (a backend with no prompt pattern), the output signal remains
                 // exactly as before — this must not regress backends we cannot read.
-                const becameBusy = await this.confirmAfterEnter(windowId, enterAt, signature, pasteBaseline, "idle-to-busy-retry");
+                const becameBusy = await this.confirmAfterEnter(windowId, enterAt, signature, pasteBaseline, "idle-to-busy-retry", !rawPaste);
                 if (becameBusy) {
                     if (status)
                         this.emit("message_confirmed", status); // ✅
@@ -5647,6 +5995,14 @@ export class Daemon extends EventEmitter {
                 }
             }
             else {
+                // No control client means a raw command has no acceptance proof. The
+                // write-ahead Enter fence keeps it uncertain after this point; never
+                // send a second Enter that could execute a command twice.
+                if (rawPaste) {
+                    verdict.phase = "raw-paste-submit-proof";
+                    verdict.proof = "unverified";
+                    return true;
+                }
                 // No control client to observe output: fall back to the legacy double-Enter.
                 await new Promise(r => setTimeout(r, 1000));
                 await this.sendDeliveryEnter("unobserved-defensive-retry");
@@ -5702,10 +6058,12 @@ export class Daemon extends EventEmitter {
      * cannot reappear in just one of them (it did: three copies, two of which
      * still confirmed on output alone).
      */
-    async confirmAfterEnter(windowId, enterAt, signature, baseline, retryPhase) {
+    async confirmAfterEnter(windowId, enterAt, signature, baseline, retryPhase, allowRecoveryEnter = true) {
         if (!this.canProveSubmission()) {
             let busy = await this.confirmBusyAfterEnter(windowId, enterAt);
             if (!busy) {
+                if (!allowRecoveryEnter)
+                    return false;
                 this.logger.warn("No idle→busy transition after Enter — re-sending Enter once");
                 const retryAt = Date.now();
                 if (!(await this.sendDeliveryEnter(retryPhase)))
@@ -5729,7 +6087,7 @@ export class Daemon extends EventEmitter {
                     break;
                 await new Promise(r => setTimeout(r, POST_ENTER_PROOF_POLL_MS));
             }
-            if (proof !== "stranded")
+            if (proof !== "stranded" || !allowRecoveryEnter)
                 return false;
             if (!(await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS)))
                 return false;
@@ -5756,6 +6114,8 @@ export class Daemon extends EventEmitter {
         // how we got here. Then poll briefly rather than judging on one capture:
         // the key takes a moment to be digested, and a single early look would
         // report a failure for a message that did go out.
+        if (!allowRecoveryEnter)
+            return false;
         this.logger.warn("Enter did not submit — waiting for the prompt, then re-sending once");
         if (!(await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS)))
             return false;
@@ -6081,6 +6441,7 @@ export class Daemon extends EventEmitter {
         const tool = msg.tool;
         const args = (msg.args ?? {});
         const requestId = msg.requestId;
+        const operationId = typeof msg.operationId === "string" ? msg.operationId : undefined;
         this.logger.debug({ tool, requestId }, "Tool call from MCP server");
         // A tool call can only come from a live MCP server of ours — proof of life
         // that outranks whatever the pid slot says.
@@ -6103,10 +6464,13 @@ export class Daemon extends EventEmitter {
             if (!error && result != null && TURN_OUTBOUND_TOOLS.has(tool)) {
                 this.turnReplyGuard.settleToolAttempt(replyAttempt, true);
             }
-            const sent = this.ipcServer?.send(socket, { requestId, result, error }) ?? false;
+            const sent = this.ipcServer?.send(socket, { requestId, result, error, operationId }) ?? false;
             if (!sent) {
                 this.logger.warn({ tool, requestId }, "Tool result could not be returned — originating MCP socket is closed");
             }
+            else if (operationId) {
+                this.emit("mcp_tool_response_written", { operationId, daemonBootId: this.bootId });
+            }
         };
         // Repo checkout — handled locally in daemon (no fleet-manager)
         if (tool === "checkout_repo") {
@@ -6231,6 +6595,7 @@ export class Daemon extends EventEmitter {
                     type: "fleet_outbound",
                     tool,
                     args,
+                    ...(operationId ? { operationId, sourceDaemonBootId: this.bootId } : {}),
                     fleetRequestId: fleetReqId,
                     senderSessionName,
                 });
@@ -6289,6 +6654,7 @@ export class Daemon extends EventEmitter {
                 type: "fleet_outbound",
                 tool,
                 args,
+                ...(operationId ? { operationId, sourceDaemonBootId: this.bootId } : {}),
                 fleetRequestId: fleetReqId,
                 // Preserve the exact adapter world that supplied the chat context. An
                 // instance binding is only a fallback: persisted/runtime context can be
@@ -6481,6 +6847,7 @@ export class Daemon extends EventEmitter {
             agentMode: isCliMode ? "cli" : "mcp",
             agentPort: isCliMode ? agentPort : undefined,
             backendOptions,
+            peerWorkingDirectories: this.peerWorkingDirectories,
         };
     }
     /**
@@ -7186,6 +7553,13 @@ export class Daemon extends EventEmitter {
             }
         }
         const cmd = `${envPrefix} ` + this.backend.buildCommand(launchConfig);
+        // e.g. Codex could not read its session DB and chose a fallback (#984):
+        // the operator should know which conversation this launch continues.
+        const launchWarning = this.backend.consumeLaunchWarning?.() ?? null;
+        if (launchWarning) {
+            this.logger.warn({ warning: launchWarning }, "Backend launch fell back");
+            this.emit("backend_launch_warning", { name: this.name, message: launchWarning });
+        }
         // Ensure tmux session exists (may have been destroyed if all windows died)
         await TmuxManager.ensureSession(this.tmuxSessionName);
         if (this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName))) {