@cello-protocol/daemon 0.0.123 → 0.0.125

package/dist/session-node-manager.js

@@ -229,9 +229,45 @@ export class SessionNodeManager {
     // still fires and the ACK still resolves — only the durable crash-backstop is skipped.
     #onAwaitingPersisted = null;
     #onAwaitingTtf = null;
+    // M12-P12 verification: force the next N park deposits to be REFUSED, so the failure this unit
+    // fixes can be produced on demand instead of waited for. The real failure is a race — the deposit
+    // is refused only in the seconds-long window while the sender's standing receiver rebuilds — and
+    // no CLI lever reaches that window: set-agent-offline leaves an open session's node serving, and
+    // the CLI refuses a send from an offline agent. Without this the fix ships unwatched.
+    // INERT unless the daemon is started with CELLO_FAULT_INJECTION=1; the IPC handler that sets it
+    // refuses outright otherwise, so a normal daemon cannot be talked into dropping messages.
+    #parkFaultRemaining = 0;
+    #parkFaultCause = "standing_receiver_creating";
+    // The incident needs BOTH halves: the direct dial has to fail (or the park path is never entered
+    // — measured, the counterparty's session node accepts the frame and reports delivered:true even
+    // with its agent away), and the park deposit that follows has to be refused. One without the
+    // other reproduces nothing.
+    #sendFaultRemaining = 0;
+    /** Arm the park-deposit fault. Returns the count now armed. */
+    injectParkFault(count, cause) {
+        this.#parkFaultRemaining = Math.max(0, count);
+        if (cause)
+            this.#parkFaultCause = cause;
+        return this.#parkFaultRemaining;
+    }
+    /** Arm the direct-send fault — makes the next N sends take the dial-failure path. */
+    injectSendFault(count) {
+        this.#sendFaultRemaining = Math.max(0, count);
+        return this.#sendFaultRemaining;
+    }
+    getSendFaultRemaining() {
+        return this.#sendFaultRemaining;
+    }
+    /** Remaining armed park faults — so a test can assert the fault was actually consumed. */
+    getParkFaultRemaining() {
+        return this.#parkFaultRemaining;
+    }
     // M12-P12: the durable enqueue for a park deposit that FAILED. Distinct from onTtf because the
     // cause is distinct — nothing timed out here, the deposit was refused — and an event named for
     // the wrong cause is how this path stayed invisible.
+    // M12-P13 (review HIGH-1): returns whether the content is ACTUALLY queued. `false` means the
+    // queue dropped it (today: the content-derived dedupe key collided), and the caller must then not
+    // claim durability — nor commit the leaf that claim now authorises.
     #onParkFailed = null;
     /**
      * MSG-001-3b (2b): the live content-park deposit. The manager resolves the recipient + relay
@@ -370,6 +406,19 @@ export class SessionNodeManager {
      * result — the deposit itself and its logging are unchanged either way.
      */
     async #parkContent(agentName, sessionId, contentHashHex, content, structure1Cbor, structure2Cbor) {
+        // Fault injection FIRST, so it reproduces the real shape: the refusal happens at the same point
+        // the live hook refuses (before any deposit), with the same event and the same `cause`.
+        if (this.#parkFaultRemaining > 0) {
+            this.#parkFaultRemaining -= 1;
+            this.#logger.warn("content.park.deposit.failed", {
+                sessionId,
+                contentHash: contentHashHex,
+                reason: "standing_receiver_unavailable",
+                cause: this.#parkFaultCause,
+                injected: true,
+            });
+            return { outcome: "refused", cause: this.#parkFaultCause };
+        }
         const hook = this.#contentParkHook;
         const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
         // M12-P12 (review F6): "no park target configured" is NOT a refused deposit. Content in a
@@ -377,7 +426,7 @@ export class SessionNodeManager {
         // be a lie that grows the DB forever — every boot and every agent start would retry a row whose
         // only possible outcome is no_persisted_relay_endpoint. Reported as unconfigured, not refused.
         if (!hook || !entry || !entry.relayPeerId || !entry.relayAddrs)
-            return "unconfigured";
+            return { outcome: "unconfigured" };
         try {
             const result = await hook({
                 // SEC-1: the hook must sign as the SENDING agent — it needs to know who that is.
@@ -404,9 +453,9 @@ export class SessionNodeManager {
                     reason: result.reason,
                     cause: result.cause,
                 });
-                return "refused";
+                return { outcome: "refused", cause: result.cause ?? result.reason };
             }
-            return "parked";
+            return { outcome: "parked" };
         }
         catch (err) {
             this.#logger.warn("content.park.deposit.failed", {
@@ -414,7 +463,7 @@ export class SessionNodeManager {
                 contentHash: contentHashHex,
                 error: err instanceof Error ? err.message : String(err),
             });
-            return "refused";
+            return { outcome: "refused", cause: err instanceof Error ? err.message : String(err) };
         }
     }
     // ─── Initialization ──────────────────────────────────────────────────────
@@ -2896,7 +2945,10 @@ export class SessionNodeManager {
     async sendContent(agentName, sessionId, content, contentHash, correlationId) {
         const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
         if (!entry) {
-            return { ok: false, reason: "session_node_unavailable", error: "no active session node for this session" };
+            // M12-P13: no node, so nothing was witnessed and nothing was queued — the caller must NOT
+            // commit a leaf for this. `durable` is a required field precisely so a new failure branch
+            // cannot be added without answering the question every caller now asks.
+            return { ok: false, reason: "session_node_unavailable", error: "no active session node for this session", durable: false };
         }
         // R1 (MSG-001-3b): witness the message-leaf HASH to the relay FIRST, INDEPENDENT of
         // direct delivery. The relay is the ordering authority (Structure 2): it assigns the
@@ -2970,6 +3022,14 @@ export class SessionNodeManager {
                 structure1_cbor: orderingS1,
                 structure2_cbor: orderingS2,
             });
+            // Injected dial failure — thrown from inside the try so it lands in exactly the catch the
+            // real connection_lost lands in, and the whole downstream path (untrack → park → durable
+            // enqueue) runs unmodified.
+            if (this.#sendFaultRemaining > 0) {
+                this.#sendFaultRemaining -= 1;
+                this.#logger.warn("content.send.fault.injected", { sessionId, contentHash: Buffer.from(contentHash).toString("hex") });
+                throw new Error("connection_lost: injected direct-send fault");
+            }
             stream.send(lp.encode.single(frame));
             try {
                 await stream.close();
@@ -2989,7 +3049,7 @@ export class SessionNodeManager {
             // agent sees the truth (the message IS in flight, just not direct), not a false negative.
             const hashHex = Buffer.from(contentHash).toString("hex");
             const attempt = await this.#parkContent(agentName, sessionId, hashHex, content, orderingS1, orderingS2);
-            if (attempt === "parked") {
+            if (attempt.outcome === "parked") {
                 return { ok: true, delivered: false, parked: true };
             }
             // M12-P12: the deposit was refused, and #untrackAwaitingAck above already dropped the
@@ -3000,17 +3060,43 @@ export class SessionNodeManager {
             // rebuilt. Only on a REFUSAL — a successful deposit must not be re-parked, and an
             // unconfigured session has no park target to retry against (F6).
             let durable = false;
-            if (attempt === "refused") {
+            if (attempt.outcome === "refused") {
                 try {
-                    this.#onParkFailed?.(agentName, sessionId, hashHex, content, orderingS1, orderingS2);
-                    durable = true;
-                    // F5: the successful enqueue must be visible. Without this the live run that has to
-                    // PROVE this fix has nothing to point at, and this log is the sender-side counterpart to
-                    // `session.content.held` on the receiver — the two together make the trace readable.
-                    this.#logger.info("content.park.deferred", {
-                        sessionId, contentHash: hashHex, agentName,
-                        selfOrdering: Boolean(orderingS1 && orderingS2),
-                    });
+                    // M12-P13 (review HIGH-1): `durable` is now OBSERVED from the enqueue, not asserted around
+                    // it. Two ways this used to lie, both of which now commit a chain leaf and so cannot be
+                    // allowed to: the queue's content-derived dedupe key collides and it silently drops the
+                    // copy, and the `?.` no-ops entirely when the composition root never wired the hook. An
+                    // absent hook is not a queue.
+                    if (this.#onParkFailed === null) {
+                        this.#logger.error("content.park.durable_enqueue.unwired", {
+                            sessionId, contentHash: hashHex, agentName,
+                            impact: "no durable queue is wired — the content is NOT retained and will NOT be retried",
+                        });
+                    }
+                    else {
+                        durable = this.#onParkFailed(agentName, sessionId, hashHex, content, orderingS1, orderingS2);
+                    }
+                    if (!durable) {
+                        if (this.#onParkFailed !== null) {
+                            this.#logger.error("content.park.durable_enqueue.dropped", {
+                                sessionId, contentHash: hashHex, agentName,
+                                impact: "the durable queue refused this copy (identical content already queued) — it is NOT separately retained",
+                            });
+                        }
+                    }
+                    else {
+                        // F5: the successful enqueue must be visible. Without this the live run that has to
+                        // PROVE this fix has nothing to point at, and this log is the sender-side counterpart to
+                        // `session.content.held` on the receiver — the two together make the trace readable.
+                        // M12-P13: `witnessed` rides along because the caller is about to commit a hash-chain
+                        // leaf on the strength of this. Without a relay ordering record the recipient recovers
+                        // in arrival order instead — the accepted degradation, but it must not be invisible.
+                        this.#logger.info("content.park.deferred", {
+                            sessionId, contentHash: hashHex, agentName,
+                            selfOrdering: Boolean(orderingS1 && orderingS2),
+                            witnessed: Boolean(orderingS1 && orderingS2),
+                        });
+                    }
                 }
                 catch (hookErr) {
                     // F3: enqueueAwaitingContent throws ON PURPOSE when the persist fails, because that is
@@ -3045,8 +3131,19 @@ export class SessionNodeManager {
                 ok: false,
                 reason: "session_stream_unavailable",
                 error: errMsg,
+                // M12-P13: the machine-readable half of the distinction below. M12-P12 shipped it in the
+                // guidance SENTENCE only, so the callers that have to ACT on it — commit the leaf for a
+                // queued message, never for a lost one — would have had to substring-match English. None
+                // did, and the sequence the relay had already witnessed was left as a permanent hole.
+                durable,
+                // M12-P13 (review MEDIUM-5): the specific standing-receiver state, carried rather than
+                // discarded. `reason` names where this surfaced; `cause` names what actually blocked it —
+                // the exact distinction M12-P12 added `standingReceiverAbsenceReason()` for, which then
+                // died inside #parkContent. An operator keying on `reason` alone is sent to the transport
+                // when the blocker is the receiver.
+                ...(attempt.cause !== undefined ? { cause: attempt.cause } : {}),
                 guidance: durable
-                    ? "Direct delivery failed and the relay refused the hand-off, so the message is queued and will be re-sent automatically when the relay link is back. No action needed."
+                    ? "Direct delivery failed and the relay refused the hand-off, so the message is queued and will be re-sent automatically when the relay link is back. Do not re-send it: an identical re-send is not separately queued."
                     : "Direct delivery failed and the message could NOT be queued for retry — it is lost. Send it again.",
             };
         }
@@ -3768,9 +3865,11 @@ export class SessionNodeManager {
     }
     /**
      * DOD-MSG-4: the relay's high-water canonical sequence for this session (largest witnessed leaf),
-     * or -1 if none. Exposed for the next sub-increment (catch-up-before-live — on reconnect B holds
-     * live arrivals until its tree reaches this, because it has more to recover than it has appended);
-     * NOT yet consumed by the gate. Also `recordWitnessedSequence` maintains it.
+     * or -1 if none. The relay is the ordering authority, so this is the outside view of how far the
+     * session has actually progressed — which is why it is the right input to a catch-up-before-live
+     * gate. Consumed by `sealReadiness` (M12-P14) for REPORTING only: the missing-leaf decision is made
+     * from `#witnessedSeq`, because this counts the relay's sequence space (which includes ctrl leaves)
+     * and the tree does not. Maintained by `recordWitnessedSequence`.
      */
     /**
      * DOD-COATTEND-1 (review F2): leaf sequences whose plaintext failed to reach the transcript and
@@ -3782,6 +3881,55 @@ export class SessionNodeManager {
     getHighWaterSeq(agentName, sessionId) {
         return this.#highWaterSeq.get(this.#k(agentName, sessionId)) ?? -1;
     }
+    /**
+     * M12-P14: is this side's chain COMPLETE enough to be sealed?
+     *
+     * A seal is a bilateral signature over the same conversation, so a side that is missing a leaf
+     * cannot produce a signable one — the counterparty compares frontiers and refuses with
+     * `leaf_count_mismatch`. That refusal is correct and it is also terminal: there is no backfill
+     * request in the protocol, so the only exit is a force-abandon, which yields NO notarized receipt.
+     * Measured 2026-08-05 on two sessions that died exactly this way (initiator 2 leaves, responder 3).
+     *
+     * The cheap prevention is to notice BEFORE asking. Two local signals already exist and, until now,
+     * nothing read either of them at close time:
+     *  - `#highWaterSeq` — the largest canonical sequence the RELAY has witnessed for this session.
+     *    The relay is the ordering authority, so a high-water above our own frontier is proof that a
+     *    leaf exists which we have not appended. (Its own doc comment called it "reserved … NOT yet
+     *    consumed by the gate" — this is that consumer.)
+     *  - `#heldContent` — content we HAVE received and verified but cannot append because it sits
+     *    behind a gap. Holding content and sealing anyway would seal a chain we know is short.
+     *
+     * Deliberately NOT a network call: it must work when the counterparty is unreachable, which is
+     * the whole situation a seal-interrupted exists for.
+     *
+     * KNOWN LIMIT, stated rather than hidden: both maps are in-memory and cleared on teardown, so
+     * after a daemon restart this returns ready for a session whose gap predates the restart — which
+     * is the shape of the 2026-08-05 incident itself. Closing that needs the mailbox drained (or the
+     * high-water persisted) before the check; tracked with M12-P14, not claimed here.
+     */
+    sealReadiness(agentName, sessionId) {
+        const key = this.#k(agentName, sessionId);
+        const treeSize = this.getSessionTree(agentName, sessionId).size();
+        const highWaterSeq = this.#highWaterSeq.get(key) ?? -1;
+        const heldCount = this.#heldContent.get(key)?.size ?? 0;
+        // Review HIGH-2: NOT `(highWaterSeq + 1) - treeSize`. That subtraction silently assumes the
+        // relay's sequence space and this tree's index space count the same things, and they do not:
+        // `relay-node.ts` increments seq_counter for EVERY accepted leaf including CTRL (0x02), while
+        // `appendSessionLeaf` is only ever called with "msg" — `submitSealLeaf` deliberately computes
+        // its root without mutating the durable tree. So one seal ctrl leaf offsets the two spaces
+        // permanently, and any msg witnessed afterwards would read as a missing leaf FOREVER. That is a
+        // false positive, and a false positive here is worse than the bug it guards: it makes a healthy
+        // session unsealable, leaving force-abandon (no receipt) as the only exit. `seal-upgrade.ts`
+        // already documents the same `leaf_count - 1` offset.
+        //
+        // `#witnessedSeq` answers the question directly instead of inferring it. It gains an entry when
+        // the relay witnesses a COUNTERPARTY msg leaf (ctrl leaves are excluded at the call site) and
+        // loses it the moment that leaf is appended. So its remaining size IS the count of leaves the
+        // ordering authority has committed and this tree has not — no arithmetic, no space mismatch,
+        // and it cannot go negative.
+        const missingLeaves = this.#witnessedSeq.get(key)?.size ?? 0;
+        return { ready: missingLeaves === 0 && heldCount === 0, treeSize, highWaterSeq, heldCount, missingLeaves };
+    }
     /** DOD-MSG-4 / DAEMON-004: append a verified message leaf and buffer it for cello_receive. */
     #appendVerifiedContent(agentName, sessionId, content, contentHashHex, senderPubkey, correlationId) {
         const { leafIndex } = this.appendSessionLeaf(agentName, sessionId, "msg", contentHashHex, correlationId);