@cello-protocol/daemon 0.0.19 → 0.0.21

package/dist/session-node-manager.js

@@ -104,6 +104,13 @@ export class SessionNodeManager {
     // prevents two concurrent ensure() calls from building two nodes for the same agent.
     #standingReceivers = new Map();
     #standingReceiverCreating = new Set();
+    // M8B F14: agents that SHOULD have a standing receiver — marked by
+    // ensureStandingReceiverForAgent (cello_start_agent / the inbound accept path) and
+    // unmarked by removeStandingReceiverForAgent (cello_stop_agent). Consulted by the
+    // teardown re-arm so a session-node teardown never re-arms an offline agent.
+    #agentsWantingReceiver = new Set();
+    // M8B F14: standing-receiver create retry schedule (see constructor opts).
+    #srRetryDelaysMs;
     // Agents whose removeStandingReceiverForAgent ran while an #ensureStandingReceiver for them was
     // in flight (parked on createNode/start, so the map had no entry to delete yet). The in-flight
     // ensure checks this after start() and tears the fresh node down instead of installing an SR for
@@ -121,6 +128,14 @@ export class SessionNodeManager {
     // DAEMON-004: per-session FIFO buffer of verified received content awaiting
     // cello_receive. Populated by ingestReceivedContent / the content stream handler.
     #receivedContent = new Map();
+    // F1-b: a terminal answer for a sealed session, set at seal teardown BEFORE the
+    // received-content buffer is evicted. A blocking cello_receive waiting when the seal
+    // fires returns this instead of hanging or 404ing; `unreadCount` tells the caller how
+    // many buffered messages were dropped (still durable — read via cello_get_transcript).
+    // This map is deliberately NOT cleared by #evictSessionCaches (it must outlive teardown);
+    // it holds one tiny entry per sealed session for the daemon's lifetime and is cleared on
+    // restart. Idempotent: a sealed session always answers "sealed" to a receive.
+    #sessionTerminal = new Map();
     // CELLO-M7-TRANSPORT-001: the directory-node multiaddrs serving as AutoNAT
     // probers (SI-002). Empty () => [] when the directory is in 'reconnecting'
     // state — AutoNAT cannot run and dialability stays the conservative default.
@@ -156,7 +171,10 @@ export class SessionNodeManager {
     // M7-UPGRADE-002: sessions for which B has already submitted its responder SEAL leaf (via
     // auto-ack OR cello_close_session). Idempotency guard — A's SEAL ctrl leaf may be delivered
     // more than once (and the relay echoes leaves), so auto-ack fires AT MOST ONCE per session.
-    #responderSealSubmitted = new Set();
+    // M8B FINDING-1: the value carries the first successful submit's reportedRootHex/sequenceNumber
+    // (null while the submit is still in flight), so a RETRY close can escalate to a unilateral
+    // seal with the original reported root instead of deadlocking on seal_pending_bilateral.
+    #responderSealSubmitted = new Map();
     // M7-SESSION-001 (M-1 PUSH): optional callback fired when a session changes
     // state, so the composition root can dispatch a session_state_changed
     // notification to live MCP clients. Injected via a setter AFTER construction
@@ -195,6 +213,7 @@ export class SessionNodeManager {
             this.#contentTtfMs = opts.contentTtfMs;
         }
         this.#autoNatProbers = opts.autoNatProbers ?? (() => []);
+        this.#srRetryDelaysMs = opts.standingReceiverRetryDelaysMs ?? [1_000, 5_000, 15_000];
     }
     /**
      * CELLO-M7-MSG-001: wire the durable-backstop side effects of the awaiting-ACK
@@ -967,6 +986,18 @@ export class SessionNodeManager {
      * Status written to SQLite.
      */
     async destroySessionNode(agentName, sessionId, reason) {
+        // F1-b: record the terminal answer BEFORE the caches are evicted (and before the
+        // early-return below), so a blocking cello_receive that was waiting when the seal fired
+        // returns "session_sealed" (with how many buffered messages it never read) instead of
+        // hanging to timeout or 404ing. Set even if the node was already retired — a late receive
+        // on a sealed session should always learn it is sealed. The receiver (the party that races
+        // the seal on cello_receive) is torn down through THIS path; the closer goes through
+        // retireSessionNode and is not blocking on receive.
+        if (reason === "sealed") {
+            const tkey = this.#k(agentName, sessionId);
+            const unreadCount = this.#receivedContent.get(tkey)?.length ?? 0;
+            this.#sessionTerminal.set(tkey, { type: "sealed", unreadCount });
+        }
         const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
         if (!entry)
             return;
@@ -1004,6 +1035,26 @@ export class SessionNodeManager {
             agentName: entry.agentName,
             reason,
         });
+        // M8B F14 (fix 1): the torn-down node has just released its port — on a fixed-port
+        // deployment this is the FIRST moment a previously-failed re-arm can succeed. Re-arm
+        // the standing receiver for an online agent that has none (async, never awaited).
+        this.#rearmAfterTeardown(agentName);
+    }
+    /**
+     * M8B F14: re-arm an online agent's standing receiver after a session-node teardown
+     * freed resources (notably the fixed port). No-op when the agent is offline, already
+     * has a receiver, or one is being created. The re-arm is a NEW async flow — it mints
+     * its own correlationId (via the ensure default) rather than inheriting the torn-down
+     * session's.
+     */
+    #rearmAfterTeardown(agentName) {
+        if (this.#shuttingDown)
+            return;
+        if (!this.#agentsWantingReceiver.has(agentName))
+            return;
+        if (this.#standingReceivers.has(agentName) || this.#standingReceiverCreating.has(agentName))
+            return;
+        void this.#ensureStandingReceiver(agentName);
     }
     /**
      * round-2 finding #5: retire a session's live libp2p node WITHOUT changing its
@@ -1038,10 +1089,19 @@ export class SessionNodeManager {
             agentName: entry.agentName,
             reason: "sealing",
         });
+        // M8B F14 (fix 1): same re-arm point as destroySessionNode — the retired node freed its port.
+        this.#rearmAfterTeardown(agentName);
     }
     /** Drop the in-memory tree + received-content caches for a torn-down session (DOD-LOOP-1: per (agent, session)). */
     #evictSessionCaches(agentName, sessionId) {
         const key = this.#k(agentName, sessionId);
+        // F1-c: dropping a NON-empty received-content buffer means deliverable plaintext the app
+        // never read live is being discarded (still durable in the transcript). Make that silent
+        // drop diagnosable — it fires on both the destroy (sealed) and retire (sealing) paths.
+        const unreadCount = this.#receivedContent.get(key)?.length ?? 0;
+        if (unreadCount > 0) {
+            this.#logger.info("session.receive.buffer.evicted", { sessionId, agentName, unreadCount });
+        }
         this.#trees.delete(key);
         this.#receivedContent.delete(key);
         // CELLO-M7-MSG-001: cancel any armed TTF timers so a torn-down session never
@@ -1318,6 +1378,10 @@ export class SessionNodeManager {
                 agentName,
                 reason: "interrupted",
             });
+            // M8B F14 (fix 1): the relay-detected interruption is the THIRD teardown path that
+            // frees the fixed port — it must re-arm too, or a session ending on a network blip
+            // leaves the agent deaf again (review finding on the F14 fix).
+            this.#rearmAfterTeardown(agentName);
         }
         this.#logger.warn("session.interrupted.detected", {
             sessionId,
@@ -1717,40 +1781,63 @@ export class SessionNodeManager {
         // two near-simultaneous triggers (e.g. B's own close racing A's delivered SEAL ctrl leaf) cannot
         // both submit. Cleared below on a relay submit failure so a genuine retry can proceed.
         if (this.#responderSealSubmitted.has(sealKey)) {
-            return { ok: false, reason: "responder_seal_already_submitted" };
-        }
-        this.#responderSealSubmitted.add(sealKey);
-        const finalRootHex = this.getSessionTreeRootHex(agentName, sessionId);
-        const sealPayload = encodeSealPayload({
-            session_id: entry.relaySessionIdBytes,
-            final_root: new Uint8Array(Buffer.from(finalRootHex, "hex")),
-            close_timestamp: Date.now(),
-            attestation: "PENDING",
-        });
-        // content_hash = SHA-256(0x02 || seal_payload) — the ctrl leaf kind byte is 0x02.
-        const contentHash = new Uint8Array(createHash("sha256").update(new Uint8Array([LEAF_KIND_CTRL])).update(sealPayload).digest());
-        const result = await entry.relayClient.submitLeaf(entry.node, entry.relaySessionIdBytes, contentHash, LEAF_KIND_CTRL);
-        if (!result.ok) {
-            // Clear the idempotency mark so a genuine retry (agent close / reconnect) can proceed (DB-001).
+            // M8B FINDING-1: carry the FIRST submit's reported root/sequence so a retry close can
+            // still escalate to a unilateral seal. A null value means that submit is still in
+            // flight — return the bare reason and let the caller fall back to the pending path.
+            const prior = this.#responderSealSubmitted.get(sealKey);
+            return prior
+                ? {
+                    ok: false,
+                    reason: "responder_seal_already_submitted",
+                    reportedRootHex: prior.reportedRootHex,
+                    sequenceNumber: prior.sequenceNumber,
+                }
+                : { ok: false, reason: "responder_seal_already_submitted" };
+        }
+        this.#responderSealSubmitted.set(sealKey, null);
+        // A throw anywhere before the mark is finalized would strand the null in-flight marker
+        // and lock every future close out of escalation (a FINDING-1-shaped deadlock via a
+        // different trigger) — clear the mark on any unexpected exception.
+        try {
+            const finalRootHex = this.getSessionTreeRootHex(agentName, sessionId);
+            const sealPayload = encodeSealPayload({
+                session_id: entry.relaySessionIdBytes,
+                final_root: new Uint8Array(Buffer.from(finalRootHex, "hex")),
+                close_timestamp: Date.now(),
+                attestation: "PENDING",
+            });
+            // content_hash = SHA-256(0x02 || seal_payload) — the ctrl leaf kind byte is 0x02.
+            const contentHash = new Uint8Array(createHash("sha256").update(new Uint8Array([LEAF_KIND_CTRL])).update(sealPayload).digest());
+            const result = await entry.relayClient.submitLeaf(entry.node, entry.relaySessionIdBytes, contentHash, LEAF_KIND_CTRL);
+            if (!result.ok) {
+                // Clear the idempotency mark so a genuine retry (agent close / reconnect) can proceed (DB-001).
+                this.#responderSealSubmitted.delete(sealKey);
+                this.#logger.warn("session.seal.leaf.submit.failed", { sessionId, reason: result.reason, correlationId });
+                return { ok: false, reason: result.reason };
+            }
+            // SESSION-002: the reported_root for a unilateral seal is the content-hash root the
+            // local tree WOULD have with this SEAL ctrl leaf appended — the same root the directory
+            // rebuilds from the relay's content-hash chain (the relay records the identical
+            // content_hash for this ctrl leaf). Computed without mutating the durable tree /
+            // message_count, so the bilateral + interrupted seal paths are unaffected.
+            const contentHashHex = Buffer.from(contentHash).toString("hex");
+            const reportedRootHex = this.getSessionTree(agentName, sessionId).rootWithAppendedHex(contentHashHex);
+            // M8B FINDING-1: durably associate the submit's escalation values with the idempotency
+            // mark, so any LATER close call can retrieve them via the already-submitted result.
+            this.#responderSealSubmitted.set(sealKey, { reportedRootHex, sequenceNumber: result.sequence_number });
+            this.#logger.info("session.seal.leaf.submitted", {
+                sessionId,
+                sequenceNumber: result.sequence_number,
+                correlationId,
+            });
+            // M7-UPGRADE-002: #responderSealSubmitted was set synchronously at the top of this method —
+            // the guard now blocks any second submit (auto-ack OR a redelivered counterparty SEAL ctrl leaf).
+            return { ok: true, sequenceNumber: result.sequence_number, reportedRootHex };
+        }
+        catch (err) {
             this.#responderSealSubmitted.delete(sealKey);
-            this.#logger.warn("session.seal.leaf.submit.failed", { sessionId, reason: result.reason, correlationId });
-            return { ok: false, reason: result.reason };
-        }
-        // SESSION-002: the reported_root for a unilateral seal is the content-hash root the
-        // local tree WOULD have with this SEAL ctrl leaf appended — the same root the directory
-        // rebuilds from the relay's content-hash chain (the relay records the identical
-        // content_hash for this ctrl leaf). Computed without mutating the durable tree /
-        // message_count, so the bilateral + interrupted seal paths are unaffected.
-        const contentHashHex = Buffer.from(contentHash).toString("hex");
-        const reportedRootHex = this.getSessionTree(agentName, sessionId).rootWithAppendedHex(contentHashHex);
-        this.#logger.info("session.seal.leaf.submitted", {
-            sessionId,
-            sequenceNumber: result.sequence_number,
-            correlationId,
-        });
-        // M7-UPGRADE-002: #responderSealSubmitted was set synchronously at the top of this method —
-        // the guard now blocks any second submit (auto-ack OR a redelivered counterparty SEAL ctrl leaf).
-        return { ok: true, sequenceNumber: result.sequence_number, reportedRootHex };
+            throw err;
+        }
     }
     /**
      * CELLO-M7-UPGRADE-001 (DOD-UP-1): readiness of a session for B to RATIFY a unilateral seal
@@ -2087,6 +2174,26 @@ export class SessionNodeManager {
             return null;
         return buf.shift() ?? null;
     }
+    /**
+     * F1-b: the terminal answer for a session that sealed while a blocking receive was (or could be)
+     * waiting. Idempotent — a sealed session always answers "sealed" to a receive. Null while active.
+     */
+    peekTerminalMarker(agentName, sessionId) {
+        return this.#sessionTerminal.get(this.#k(agentName, sessionId)) ?? null;
+    }
+    /**
+     * F1-b: the durable sealed root hex for a session (written by recordSealCertificate on the
+     * bilateral path), or null if not recorded. Lets cello_receive echo the sealed root in its
+     * terminal answer without threading it through destroySessionNode.
+     */
+    getSealedRootHex(agentName, sessionId) {
+        if (!this.#db)
+            return null;
+        const row = this.#db
+            .prepare("SELECT sealed_root_hex FROM sessions WHERE agent_name = ? AND session_id = ?")
+            .get(agentName, sessionId);
+        return row?.sealed_root_hex ?? null;
+    }
     // ─── CELLO-M7-MSG-001: delivery ACK / TTF tracking (send side) ──────────────
     /**
      * Arm awaiting-ACK tracking for a just-sent content frame (AC-001/AC-003). Records
@@ -2535,8 +2642,15 @@ export class SessionNodeManager {
      * DOD-LOOP-1: ensure the given agent has a standing receiver node (idempotent). Created when an
      * agent comes online (cello_start_agent) and replaced after it is handed off to a session. The
      * `#standingReceiverCreating` guard prevents two concurrent ensure() calls (e.g. the
-     * cello_start_agent hook racing a consume-site retry) from building two nodes for one agent. A
-     * create failure logs + leaves no entry; the next consume-site ensure() call retries on demand.
+     * cello_start_agent hook racing a consume-site retry) from building two nodes for one agent.
+     *
+     * M8B F14: a create failure no longer strands the agent deaf. Each ensure runs a BOUNDED
+     * retry loop (`standingReceiverRetryDelaysMs`, default 1s/5s/15s) — covering the fixed-port
+     * race where the consumed receiver still holds the port until its session node is torn down —
+     * and when every attempt fails, fires the alarm-worthy `session.standing_receiver.dead`
+     * (error level), distinct from the per-attempt `session.node.create.failed`. Re-arm is also
+     * kicked from destroySessionNode/retireSessionNode (the moment the port frees) and from the
+     * inbound accept path (ensure on demand), so one failure can never leave the agent deaf forever.
      */
     async #ensureStandingReceiver(agentName, correlationId = randomUUID()) {
         if (this.#standingReceivers.has(agentName) || this.#standingReceiverCreating.has(agentName))
@@ -2547,76 +2661,111 @@ export class SessionNodeManager {
         this.#standingReceiverRemoving.delete(agentName);
         this.#standingReceiverCreating.add(agentName);
         try {
-            const sessionId = `standing_receiver_${randomUUID()}`;
-            const gater = new SessionConnectionGater({
-                sessionId,
-                allowedPeerId: null, // open — counterparty unknown at creation time
-                logger: this.#logger,
-            });
-            let node;
-            try {
-                node = await this.#factory.createNode({ sessionId, connectionGater: gater, nodeType: "standing_receiver" });
-                await node.start();
-            }
-            catch (err) {
-                this.#logger.error("session.node.create.failed", {
-                    sessionId,
-                    agentName: `${STANDING_RECEIVER_AGENT_NAME}:${agentName}`,
-                    error: err instanceof Error ? err.message : String(err),
-                    correlationId,
-                });
-                return; // not ready — callers check getStandingReceiverReady / retry via ensure on demand
-            }
-            // M2: gracefulShutdown may have begun while this node was starting (ensure runs un-awaited).
-            // Don't install an orphan bound to a TCP port — stop it and bail.
-            if (this.#shuttingDown) {
-                try {
-                    await node.stop();
+            let lastError = "";
+            for (let attempt = 0; attempt <= this.#srRetryDelaysMs.length; attempt++) {
+                if (attempt > 0) {
+                    await new Promise((r) => setTimeout(r, this.#srRetryDelaysMs[attempt - 1]));
                 }
-                catch { /* best-effort */ }
-                return;
-            }
-            // L1: the agent may have gone offline (cello_stop_agent → removeStandingReceiverForAgent)
-            // while this ensure was parked on start(). Removal found no map entry to delete, so the
-            // tombstone is how we learn of it — tear the fresh node down rather than install an SR for
-            // an offline agent.
-            if (this.#standingReceiverRemoving.has(agentName)) {
-                this.#standingReceiverRemoving.delete(agentName);
-                try {
-                    await node.stop();
+                if (this.#shuttingDown)
+                    return;
+                // L1 tombstone: the agent went offline while we were creating / backing off.
+                if (this.#standingReceiverRemoving.has(agentName)) {
+                    this.#standingReceiverRemoving.delete(agentName);
+                    return;
                 }
-                catch { /* best-effort */ }
-                return;
+                const result = await this.#tryCreateStandingReceiver(agentName, correlationId);
+                if (result.outcome !== "failed")
+                    return; // installed, or cleanly aborted (shutdown/offline)
+                lastError = result.error;
             }
-            // CELLO-M7-TRANSPORT-001: wrap in a NodeAutoNatService so its dialability drives session-
-            // address advertisement and the transport.autonat.* events fire.
-            const autoNat = new NodeAutoNatService({
-                node,
-                logger: this.#logger,
-                nodeType: "standing_receiver",
-                probers: this.#autoNatProbers(),
+            // M8B F14 (fix 4): an agent that WANTS a receiver has none after every attempt — the
+            // deaf-agent state. Fail LOUD so it is alarm-visible instead of a quiet degradation.
+            this.#logger.error("session.standing_receiver.dead", {
+                agentName,
+                reason: lastError,
+                attempts: this.#srRetryDelaysMs.length + 1,
+                correlationId,
             });
-            autoNat.emitInitialResult();
-            this.#standingReceivers.set(agentName, { node, gater, autoNat });
-            this.#logger.info("session.node.created", {
+        }
+        finally {
+            this.#standingReceiverCreating.delete(agentName);
+        }
+    }
+    /** One standing-receiver create attempt (extracted for the M8B F14 retry loop). */
+    async #tryCreateStandingReceiver(agentName, correlationId) {
+        const sessionId = `standing_receiver_${randomUUID()}`;
+        const gater = new SessionConnectionGater({
+            sessionId,
+            allowedPeerId: null, // open — counterparty unknown at creation time
+            logger: this.#logger,
+        });
+        let node;
+        try {
+            node = await this.#factory.createNode({ sessionId, connectionGater: gater, nodeType: "standing_receiver" });
+            await node.start();
+        }
+        catch (err) {
+            const error = err instanceof Error ? err.message : String(err);
+            this.#logger.error("session.node.create.failed", {
                 sessionId,
                 agentName: `${STANDING_RECEIVER_AGENT_NAME}:${agentName}`,
-                sessionPeerId: node.getPeerId(),
+                error,
                 correlationId,
             });
+            return { outcome: "failed", error };
         }
-        finally {
-            this.#standingReceiverCreating.delete(agentName);
+        // M2: gracefulShutdown may have begun while this node was starting (ensure runs un-awaited).
+        // Don't install an orphan bound to a TCP port — stop it and bail.
+        if (this.#shuttingDown) {
+            try {
+                await node.stop();
+            }
+            catch { /* best-effort */ }
+            return { outcome: "aborted" };
+        }
+        // L1: the agent may have gone offline (cello_stop_agent → removeStandingReceiverForAgent)
+        // while this ensure was parked on start(). Removal found no map entry to delete, so the
+        // tombstone is how we learn of it — tear the fresh node down rather than install an SR for
+        // an offline agent.
+        if (this.#standingReceiverRemoving.has(agentName)) {
+            this.#standingReceiverRemoving.delete(agentName);
+            try {
+                await node.stop();
+            }
+            catch { /* best-effort */ }
+            return { outcome: "aborted" };
         }
+        // CELLO-M7-TRANSPORT-001: wrap in a NodeAutoNatService so its dialability drives session-
+        // address advertisement and the transport.autonat.* events fire.
+        const autoNat = new NodeAutoNatService({
+            node,
+            logger: this.#logger,
+            nodeType: "standing_receiver",
+            probers: this.#autoNatProbers(),
+        });
+        autoNat.emitInitialResult();
+        this.#standingReceivers.set(agentName, { node, gater, autoNat });
+        this.#logger.info("session.node.created", {
+            sessionId,
+            agentName: `${STANDING_RECEIVER_AGENT_NAME}:${agentName}`,
+            sessionPeerId: node.getPeerId(),
+            correlationId,
+        });
+        return { outcome: "installed" };
     }
     /**
      * DOD-LOOP-1: public hook for the composition root to create an agent's standing receiver when
      * the agent comes online (cello_start_agent), and to tear it down when it goes offline.
+     * M8B F14: also called from the inbound accept path (ensure on demand). Marks the agent as
+     * WANTING a receiver, which arms the teardown re-arm in destroySessionNode/retireSessionNode.
      */
     async ensureStandingReceiverForAgent(agentName) {
+        this.#agentsWantingReceiver.add(agentName);
         await this.#ensureStandingReceiver(agentName);
     }
     async removeStandingReceiverForAgent(agentName) {
+        // M8B F14: the agent no longer wants a receiver — disarm the teardown re-arm.
+        this.#agentsWantingReceiver.delete(agentName);
         const sr = this.#standingReceivers.get(agentName);
         if (!sr) {
             // L1: an #ensureStandingReceiver for this agent may be in flight (parked on start(), so no