@songsid/agend 2.1.7-beta.4 → 2.1.7-beta.5

package/dist/daemon.js

@@ -30,10 +30,11 @@ import { bottomRowIsReady, inputAreaText, inputShowsPastedText, pastedTextSignat
 import { TurnReplyGuard } from "./turn-reply-guard.js";
 import { t } from "./locale.js";
 import { MuseUsageRelay, clearMuseUsageSnapshot } from "./muse-usage-relay.js";
+import { queueResumePolicyForAttempt } from "./delivery-queue-evidence.js";
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 // Tool routing sets — module-level to avoid re-creation on every handleToolCall
-const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "broadcast", "list_instances", "start_instance", "restart_instance", "create_instance", "delete_instance", "replace_instance", "request_information", "delegate_task", "report_result", "describe_instance"]);
+const CROSS_INSTANCE_TOOLS = new Set(["send_to_instance", "broadcast", "list_instances", "start_instance", "restart_instance", "create_instance", "delete_instance", "replace_instance", "request_information", "delegate_task", "report_result", "delivery_status", "describe_instance"]);
 const SCHEDULE_TOOLS = new Set(["create_schedule", "list_schedules", "update_schedule", "delete_schedule"]);
 const DECISION_TOOLS = new Set(["post_decision", "list_decisions", "update_decision"]);
 const TASK_TOOL = "task";
@@ -1095,6 +1096,7 @@ export class Daemon extends EventEmitter {
     /** Mutual exclusion for *every* write into the pane, whichever subsystem it
      *  comes from. See PaneWriteLock for why interleaving is destructive. */
     paneWriteLock = new PaneWriteLock();
+    deliveryWritesStopping = false;
     pendingInstructionsUpdate;
     pendingInstructionsNotice = false;
     // Whether the warmup steering-reload notice should be injected after spawn.
@@ -1331,6 +1333,22 @@ export class Daemon extends EventEmitter {
     setDeliveryOutboxPort(port) {
         this.deliveryOutbox = port;
     }
+    /** The fleet's other same-backend instances' working directories, read at spawn (#984). */
+    peerWorkingDirectories;
+    setPeerWorkingDirectories(peers) {
+        this.peerWorkingDirectories = peers;
+    }
+    /** Fence pane writers before lifecycle captures pre-kill reconciliation evidence. */
+    fenceDeliveryWritesForStop() {
+        if (this.deliveryWritesStopping)
+            return;
+        this.deliveryWritesStopping = true;
+        this.deliveryEpoch++;
+    }
+    /** Bounded drain: timeout remains safe because Enter has a write-ahead fence. */
+    waitForDeliveryWritesToDrain(timeoutMs) {
+        return this.paneWriteLock.waitForIdle(timeoutMs);
+    }
     durableDeliveryAttempt(meta) {
         const deliveryId = meta.delivery_id;
         if (!deliveryId)
@@ -1340,16 +1358,44 @@ export class Daemon extends EventEmitter {
             return false;
         return { deliveryId, attemptNo };
     }
-    beginDurableDelivery(delivery) {
+    beginDurableDelivery(delivery, evidence) {
         if (!this.deliveryOutbox) {
             this.emit("durable_delivery_deferred", { ...delivery, targetBootId: this.bootId, reason: "outbox port unavailable before pane write" });
             return false;
         }
-        const outcome = this.deliveryOutbox.begin(delivery.deliveryId, this.bootId, delivery.attemptNo);
+        const outcome = this.deliveryOutbox.begin(delivery.deliveryId, this.bootId, delivery.attemptNo, evidence);
         // A repeated begin is idempotent at the store but must not authorize a
         // second pane write. Only the first caller that acquired the permit acts.
         return outcome === "begun";
     }
+    async durableAttemptEvidence(windowId, handingOffToNativeQueue, steer, rawPaste = false) {
+        let checkpoint = null;
+        try {
+            checkpoint = await this.transcriptMonitor?.reconciliationCheckpoint() ?? null;
+        }
+        catch (err) {
+            this.logger.debug({ err }, "Could not checkpoint transcript before durable delivery");
+        }
+        const submissionMode = rawPaste ? "raw_paste" : steer
+            ? "steer"
+            : handingOffToNativeQueue ? "native_queue_handoff" : "idle_submit";
+        const backend = this.config.backend ?? this.backend?.binaryName ?? "unknown";
+        // The daemon does not synchronously probe CLI versions on the delivery
+        // path. Until lifecycle supplies a version tied to this exact process,
+        // queue handoffs persist an unknown resume contract and reconciliation
+        // must fail closed rather than infer loss from transcript absence.
+        const backendVersion = null;
+        return {
+            backend,
+            backendVersion,
+            windowId: windowId ?? this.tmux?.getWindowId() ?? null,
+            transcriptPath: checkpoint?.path ?? null,
+            transcriptOffset: checkpoint?.offset ?? null,
+            transcriptSessionId: checkpoint?.sessionId ?? null,
+            submissionMode,
+            queueResumePolicy: queueResumePolicyForAttempt(backend, backendVersion, submissionMode),
+        };
+    }
     abortDurableDelivery(delivery, reason) {
         if (!delivery)
             return;
@@ -1489,8 +1535,7 @@ export class Daemon extends EventEmitter {
                 });
             }
             else if (msg.type === "raw_paste") {
-                // Paste raw text directly to CLI without [user:] wrapping.
-                this.queueRawPaste(msg.content, this.captureDeliveryEpoch(msg.delivery_epoch), msg.confirm_clear === true);
+                this.handleRawPasteIpcMessage(msg);
             }
             else if (msg.type === "config_update") {
                 this.applyConfigUpdate(msg.config);
@@ -2765,25 +2810,77 @@ export class Daemon extends EventEmitter {
     isDeliveryEpochCurrent(epoch) {
         return epoch === this.deliveryEpoch;
     }
-    queueRawPaste(rawText, deliveryEpoch = this.deliveryEpoch, confirmClear = false) {
-        if (!this.tmux || !this.isDeliveryEpochCurrent(deliveryEpoch))
+    queueRawPaste(rawText, deliveryEpoch = this.deliveryEpoch, confirmClear = false, durableAttempt) {
+        if (!this.tmux || !this.isDeliveryEpochCurrent(deliveryEpoch)) {
+            if (durableAttempt)
+                this.retryDurableDeliveryBeforeBegin(durableAttempt, "raw_paste target unavailable before pane queue");
             return;
+        }
         this.pasteLock = this.pasteLock.then(async () => {
             if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
                 this.logger.info("Pending raw delivery dropped by user cancel");
+                if (durableAttempt)
+                    this.retryDurableDeliveryBeforeBegin(durableAttempt, "raw_paste epoch changed while queued");
                 return;
             }
-            await this.wake();
-            if (!this.isDeliveryEpochCurrent(deliveryEpoch))
+            const verdict = { reached: false };
+            try {
+                await this.wake();
+                if (!this.isDeliveryEpochCurrent(deliveryEpoch)) {
+                    if (durableAttempt)
+                        this.retryDurableDeliveryBeforeBegin(durableAttempt, "raw_paste epoch changed while waking");
+                    return;
+                }
+                const delivered = await this.deliverMessage(rawText, undefined, {
+                    deliveryEpoch,
+                    verdict,
+                    durableAttempt,
+                    preserveExactBytes: true,
+                });
+                if (delivered)
+                    this.finishDurableSubmission(durableAttempt ?? null, verdict);
+                else if (durableAttempt && verdict.paneWriteStarted) {
+                    this.finishDurableDelivery(durableAttempt, "uncertain", `${verdict.phase ?? "raw-paste"}:${verdict.proof ?? "unverified"}`);
+                }
+                else if (durableAttempt && verdict.durableBeginCommitted) {
+                    this.abortDurableDelivery(durableAttempt, `${verdict.phase ?? "raw-paste"}:${verdict.proof ?? "no-pane-write"}`);
+                }
+                else if (durableAttempt) {
+                    this.retryDurableDeliveryBeforeBegin(durableAttempt, `${verdict.phase ?? "raw-paste"}:${verdict.proof ?? "deferred"}`);
+                }
+            }
+            catch (err) {
+                if (durableAttempt && verdict.paneWriteStarted)
+                    this.finishDurableDelivery(durableAttempt, "uncertain", err.message);
+                else if (durableAttempt && verdict.durableBeginCommitted)
+                    this.abortDurableDelivery(durableAttempt, err.message);
+                else if (durableAttempt)
+                    this.retryDurableDeliveryBeforeBegin(durableAttempt, err.message);
+                this.logger.warn({ err: err.message }, "raw_paste delivery error");
                 return;
-            await this.deliverMessage(rawText, undefined, { deliveryEpoch });
+            }
             if (confirmClear)
                 await this.confirmBackendClearDialog();
             this.logger.debug({ text: rawText.slice(0, 100) }, "Raw paste delivered");
-        }).catch(err => {
-            this.logger.warn({ err: err.message }, "raw_paste delivery error");
+        });
+        this.pasteLock = this.pasteLock.catch(err => {
+            this.logger.warn({ err: err.message }, "raw_paste delivery queue error");
         });
     }
+    /** Keep the IPC raw route's durable mode attached to the parsed attempt. */
+    handleRawPasteIpcMessage(msg) {
+        // Paste raw text directly to CLI without [user:] wrapping.
+        const deliveryMeta = {
+            delivery_id: typeof msg.delivery_id === "string" ? msg.delivery_id : "",
+            delivery_attempt: typeof msg.delivery_attempt === "string" ? msg.delivery_attempt : "",
+        };
+        const durableAttempt = this.durableDeliveryAttempt(deliveryMeta);
+        if (durableAttempt === false) {
+            this.deferDurableDelivery(deliveryMeta, "invalid raw_paste delivery attempt metadata");
+            return;
+        }
+        this.queueRawPaste(msg.content, this.captureDeliveryEpoch(msg.delivery_epoch), msg.confirm_clear === true, durableAttempt ? { ...durableAttempt, submissionMode: "raw_paste" } : undefined);
+    }
     /**
      * Confirm a backend's terminal-level /clear prompt after the channel-level
      * admin confirmation has already succeeded. Poll instead of pasting `/clear\ny`
@@ -2930,6 +3027,7 @@ export class Daemon extends EventEmitter {
     }
     async stop() {
         this.logger.info("Stopping daemon instance");
+        this.fenceDeliveryWritesForStop();
         this.turnReplyGuard.reset();
         // Invalidate any bounded pre-Enter wait from the process generation being
         // stopped. It must fail closed, not press Enter in a replacement pane.
@@ -4641,7 +4739,7 @@ export class Daemon extends EventEmitter {
             return false;
         // Sanitize unclosed code fences — they cause CLI to wait for closure on Enter
         const fenceCount = (formatted.match(/```/g) || []).length;
-        if (fenceCount % 2 !== 0) {
+        if (!opts?.preserveExactBytes && fenceCount % 2 !== 0) {
             // Odd number of fences = unclosed. Remove all code fences from the message.
             formatted = formatted.replace(/```/g, "");
         }
@@ -4789,7 +4887,7 @@ export class Daemon extends EventEmitter {
                 }
                 if (!(await this.hasPositiveDeliveryInput()))
                     return "dialog";
-                return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId, verdict, opts?.durableAttempt);
+                return this.writeMessageToPane(formatted, windowId, handingOffToNativeQueue, status, opts?.submissionId, verdict, opts?.durableAttempt, opts?.steer === true);
             });
             if (outcome === "spawn-started") {
                 // The pane changed under this delivery: its queued paste must not land
@@ -5543,13 +5641,20 @@ export class Daemon extends EventEmitter {
      * precisely so the lock's scope is visible at the call site rather than being
      * an invariant maintained by comments.
      */
-    async sendDeliveryEnter(phase, stillCurrent) {
+    async sendDeliveryEnter(phase, stillCurrent, durableAttempt) {
         if (!(await this.waitForInputTransientToClear(phase)))
             return false;
         // A recovery Enter may have waited for a transient while cancel or spawn
         // replaced the delivery. Check at the last point before the tmux write.
-        if (stillCurrent && !stillCurrent())
+        if (this.deliveryWritesStopping || (stillCurrent && !stillCurrent()))
+            return false;
+        // W1: persist the "Enter may have happened" boundary before tmux receives
+        // the key. A crash after this commit is never auto-replayed without a
+        // positive transcript marker, even if the stale pane still shows composer.
+        if (durableAttempt && !this.deliveryOutbox?.markEnterStarted(durableAttempt.deliveryId, this.bootId, durableAttempt.attemptNo)) {
+            this.logger.error({ phase, deliveryId: durableAttempt.deliveryId }, "Could not commit durable Enter write-ahead fence");
             return false;
+        }
         const sent = await this.tmux.sendSpecialKey("Enter");
         if (!sent) {
             this.logger.error({
@@ -5562,8 +5667,9 @@ export class Daemon extends EventEmitter {
     async writeMessageToPane(formatted, initialWindowId, handingOffToNativeQueue, status, submissionId, 
     // Last and defaulted so the positional callers that ignore the outcome stay
     // readable; deliverMessage, the only one that reports, always passes its own.
-    verdict = { reached: false }, durableAttempt) {
+    verdict = { reached: false }, durableAttempt, steer = false) {
         const signature = this.submissionSignature(formatted, submissionId);
+        const rawPaste = durableAttempt?.submissionMode === "raw_paste";
         let windowId = initialWindowId;
         // Bug A: paste with backoff. Transient failures are usually a stale window id
         // after a crash/respawn — recover by name and retry (max 3 attempts, 2s apart).
@@ -5579,7 +5685,8 @@ export class Daemon extends EventEmitter {
             // first side effect; a crash during those waits remains safely replayable.
             if (durableAttempt && !beginChecked) {
                 beginChecked = true;
-                if (!this.beginDurableDelivery(durableAttempt)) {
+                const attemptEvidence = await this.durableAttemptEvidence(windowId, handingOffToNativeQueue, steer, durableAttempt.submissionMode === "raw_paste");
+                if (!this.beginDurableDelivery(durableAttempt, attemptEvidence)) {
                     verdict.durableBeginRejected = true;
                     verdict.phase = "submission-begin";
                     verdict.proof = "permit-rejected";
@@ -5587,8 +5694,19 @@ export class Daemon extends EventEmitter {
                 }
                 verdict.durableBeginCommitted = true;
             }
+            if (rawPaste)
+                verdict.paneWriteStarted = true;
+            // For raw commands the paste itself may alter the live composer. Treat a
+            // failed/throwing write as uncertain rather than retrying bytes into the
+            // same pane; the write-ahead Enter marker later separates safe pre-Enter
+            // recovery from a possibly executed command.
             const pasted = await this.tmux.pasteBuffer(formatted);
             if (!pasted) {
+                if (rawPaste) {
+                    verdict.phase = "raw-paste-write";
+                    verdict.proof = "paste-result-unknown";
+                    return false;
+                }
                 const tmuxError = this.tmux.getLastPasteError?.() ?? "unknown tmux paste failure";
                 const recoverable = this.tmux.isLastPasteFailureRecoverable?.() ?? true;
                 this.logger[recoverable ? "warn" : "error"]({ attempt, tmuxError, recoverable }, recoverable
@@ -5630,7 +5748,7 @@ export class Daemon extends EventEmitter {
                 settle = { settleMs: fallbackMs, observedPostPasteOutput: false, capHit: false, usedFallback: true };
             }
             let enterAt = Date.now();
-            if (!(await this.sendDeliveryEnter("initial-submit"))) {
+            if (!(await this.sendDeliveryEnter("initial-submit", undefined, durableAttempt))) {
                 return this.failDelivery(verdict, status, "submit-enter", "tmux-send-keys-failed");
             }
             // Kiro's legacy TUI can swallow Enter while it is still processing a large
@@ -5646,7 +5764,7 @@ export class Daemon extends EventEmitter {
             // re-baselined to the second Enter so leftover paste-render output between
             // the two cannot be what "confirms" the submission.
             let enterRetry = false;
-            if (this.backend?.requiresDeliveryEnterRetry?.() === true) {
+            if (!rawPaste && this.backend?.requiresDeliveryEnterRetry?.() === true) {
                 await new Promise(r => setTimeout(r, 1_000));
                 const retryAt = Date.now();
                 if (await this.sendDeliveryEnter("queue-less-defensive-retry")) {
@@ -5697,6 +5815,13 @@ export class Daemon extends EventEmitter {
                         this.emit("message_confirmed", status); // ✅ (best-effort)
                     return true;
                 }
+                // raw_paste has no unique marker and is not allowed to re-paste or
+                // issue another Enter after its one committed submission attempt.
+                if (rawPaste) {
+                    verdict.phase = "raw-paste-native-queue-proof";
+                    verdict.proof = proof;
+                    return false;
+                }
                 // Not submitted (or not provably submitted): fall back once to the
                 // normal idle-gated path, which is the only one with a full
                 // confirmation ladder. "stranded" is the case that used to be a silent
@@ -5806,6 +5931,11 @@ export class Daemon extends EventEmitter {
                     }
                 }
                 if (!submitted) {
+                    if (rawPaste) {
+                        verdict.phase = "raw-paste-submit-proof";
+                        verdict.proof = "no-positive-acceptance-proof";
+                        return false;
+                    }
                     // Legacy system pastes have no unique envelope id. Preserve their
                     // existing bounded retry until they can be tied to a specific turn.
                     this.logger.warn("Message not submitted after Enter — waiting for the prompt, then re-sending Enter once");
@@ -5836,7 +5966,7 @@ export class Daemon extends EventEmitter {
                 // is disqualifying and no amount of output may override it. Where it
                 // cannot (a backend with no prompt pattern), the output signal remains
                 // exactly as before — this must not regress backends we cannot read.
-                const becameBusy = await this.confirmAfterEnter(windowId, enterAt, signature, pasteBaseline, "idle-to-busy-retry");
+                const becameBusy = await this.confirmAfterEnter(windowId, enterAt, signature, pasteBaseline, "idle-to-busy-retry", !rawPaste);
                 if (becameBusy) {
                     if (status)
                         this.emit("message_confirmed", status); // ✅
@@ -5865,6 +5995,14 @@ export class Daemon extends EventEmitter {
                 }
             }
             else {
+                // No control client means a raw command has no acceptance proof. The
+                // write-ahead Enter fence keeps it uncertain after this point; never
+                // send a second Enter that could execute a command twice.
+                if (rawPaste) {
+                    verdict.phase = "raw-paste-submit-proof";
+                    verdict.proof = "unverified";
+                    return true;
+                }
                 // No control client to observe output: fall back to the legacy double-Enter.
                 await new Promise(r => setTimeout(r, 1000));
                 await this.sendDeliveryEnter("unobserved-defensive-retry");
@@ -5920,10 +6058,12 @@ export class Daemon extends EventEmitter {
      * cannot reappear in just one of them (it did: three copies, two of which
      * still confirmed on output alone).
      */
-    async confirmAfterEnter(windowId, enterAt, signature, baseline, retryPhase) {
+    async confirmAfterEnter(windowId, enterAt, signature, baseline, retryPhase, allowRecoveryEnter = true) {
         if (!this.canProveSubmission()) {
             let busy = await this.confirmBusyAfterEnter(windowId, enterAt);
             if (!busy) {
+                if (!allowRecoveryEnter)
+                    return false;
                 this.logger.warn("No idle→busy transition after Enter — re-sending Enter once");
                 const retryAt = Date.now();
                 if (!(await this.sendDeliveryEnter(retryPhase)))
@@ -5947,7 +6087,7 @@ export class Daemon extends EventEmitter {
                     break;
                 await new Promise(r => setTimeout(r, POST_ENTER_PROOF_POLL_MS));
             }
-            if (proof !== "stranded")
+            if (proof !== "stranded" || !allowRecoveryEnter)
                 return false;
             if (!(await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS)))
                 return false;
@@ -5974,6 +6114,8 @@ export class Daemon extends EventEmitter {
         // how we got here. Then poll briefly rather than judging on one capture:
         // the key takes a moment to be digested, and a single early look would
         // report a failure for a message that did go out.
+        if (!allowRecoveryEnter)
+            return false;
         this.logger.warn("Enter did not submit — waiting for the prompt, then re-sending once");
         if (!(await this.waitForPaneReadyForDelivery(windowId, STRANDED_RETRY_READY_WAIT_MS)))
             return false;
@@ -6705,6 +6847,7 @@ export class Daemon extends EventEmitter {
             agentMode: isCliMode ? "cli" : "mcp",
             agentPort: isCliMode ? agentPort : undefined,
             backendOptions,
+            peerWorkingDirectories: this.peerWorkingDirectories,
         };
     }
     /**
@@ -7410,6 +7553,13 @@ export class Daemon extends EventEmitter {
             }
         }
         const cmd = `${envPrefix} ` + this.backend.buildCommand(launchConfig);
+        // e.g. Codex could not read its session DB and chose a fallback (#984):
+        // the operator should know which conversation this launch continues.
+        const launchWarning = this.backend.consumeLaunchWarning?.() ?? null;
+        if (launchWarning) {
+            this.logger.warn({ warning: launchWarning }, "Backend launch fell back");
+            this.emit("backend_launch_warning", { name: this.name, message: launchWarning });
+        }
         // Ensure tmux session exists (may have been destroyed if all windows died)
         await TmuxManager.ensureSession(this.tmuxSessionName);
         if (this.stormWindow?.observeServerAlive(await TmuxManager.getServerPid(this.tmuxSessionName))) {