@cello-protocol/daemon 0.0.122 → 0.0.124

package/dist/session-node-manager.js

@@ -229,6 +229,46 @@ export class SessionNodeManager {
     // still fires and the ACK still resolves — only the durable crash-backstop is skipped.
     #onAwaitingPersisted = null;
     #onAwaitingTtf = null;
+    // M12-P12 verification: force the next N park deposits to be REFUSED, so the failure this unit
+    // fixes can be produced on demand instead of waited for. The real failure is a race — the deposit
+    // is refused only in the seconds-long window while the sender's standing receiver rebuilds — and
+    // no CLI lever reaches that window: set-agent-offline leaves an open session's node serving, and
+    // the CLI refuses a send from an offline agent. Without this the fix ships unwatched.
+    // INERT unless the daemon is started with CELLO_FAULT_INJECTION=1; the IPC handler that sets it
+    // refuses outright otherwise, so a normal daemon cannot be talked into dropping messages.
+    #parkFaultRemaining = 0;
+    #parkFaultCause = "standing_receiver_creating";
+    // The incident needs BOTH halves: the direct dial has to fail (or the park path is never entered
+    // — measured, the counterparty's session node accepts the frame and reports delivered:true even
+    // with its agent away), and the park deposit that follows has to be refused. One without the
+    // other reproduces nothing.
+    #sendFaultRemaining = 0;
+    /** Arm the park-deposit fault. Returns the count now armed. */
+    injectParkFault(count, cause) {
+        this.#parkFaultRemaining = Math.max(0, count);
+        if (cause)
+            this.#parkFaultCause = cause;
+        return this.#parkFaultRemaining;
+    }
+    /** Arm the direct-send fault — makes the next N sends take the dial-failure path. */
+    injectSendFault(count) {
+        this.#sendFaultRemaining = Math.max(0, count);
+        return this.#sendFaultRemaining;
+    }
+    getSendFaultRemaining() {
+        return this.#sendFaultRemaining;
+    }
+    /** Remaining armed park faults — so a test can assert the fault was actually consumed. */
+    getParkFaultRemaining() {
+        return this.#parkFaultRemaining;
+    }
+    // M12-P12: the durable enqueue for a park deposit that FAILED. Distinct from onTtf because the
+    // cause is distinct — nothing timed out here, the deposit was refused — and an event named for
+    // the wrong cause is how this path stayed invisible.
+    // M12-P13 (review HIGH-1): returns whether the content is ACTUALLY queued. `false` means the
+    // queue dropped it (today: the content-derived dedupe key collided), and the caller must then not
+    // claim durability — nor commit the leaf that claim now authorises.
+    #onParkFailed = null;
     /**
      * MSG-001-3b (2b): the live content-park deposit. The manager resolves the recipient + relay
      * endpoint from the session entry and calls this when a send is NOT confirmed delivered
@@ -277,6 +317,7 @@ export class SessionNodeManager {
     setAwaitingAckHooks(hooks) {
         this.#onAwaitingPersisted = hooks.onPersisted ?? null;
         this.#onAwaitingTtf = hooks.onTtf ?? null;
+        this.#onParkFailed = hooks.onParkFailed ?? null;
     }
     /**
      * MSG-001-3b (2b): inject the live content-park deposit (seal + ContentParkClient.deposit).
@@ -365,10 +406,27 @@ export class SessionNodeManager {
      * result — the deposit itself and its logging are unchanged either way.
      */
     async #parkContent(agentName, sessionId, contentHashHex, content, structure1Cbor, structure2Cbor) {
+        // Fault injection FIRST, so it reproduces the real shape: the refusal happens at the same point
+        // the live hook refuses (before any deposit), with the same event and the same `cause`.
+        if (this.#parkFaultRemaining > 0) {
+            this.#parkFaultRemaining -= 1;
+            this.#logger.warn("content.park.deposit.failed", {
+                sessionId,
+                contentHash: contentHashHex,
+                reason: "standing_receiver_unavailable",
+                cause: this.#parkFaultCause,
+                injected: true,
+            });
+            return { outcome: "refused", cause: this.#parkFaultCause };
+        }
         const hook = this.#contentParkHook;
         const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
+        // M12-P12 (review F6): "no park target configured" is NOT a refused deposit. Content in a
+        // session with no relay was never recoverable through the park, so queuing it for re-park would
+        // be a lie that grows the DB forever — every boot and every agent start would retry a row whose
+        // only possible outcome is no_persisted_relay_endpoint. Reported as unconfigured, not refused.
         if (!hook || !entry || !entry.relayPeerId || !entry.relayAddrs)
-            return false;
+            return { outcome: "unconfigured" };
         try {
             const result = await hook({
                 // SEC-1: the hook must sign as the SENDING agent — it needs to know who that is.
@@ -393,10 +451,11 @@ export class SessionNodeManager {
                     sessionId,
                     contentHash: contentHashHex,
                     reason: result.reason,
+                    cause: result.cause,
                 });
-                return false;
+                return { outcome: "refused", cause: result.cause ?? result.reason };
             }
-            return true;
+            return { outcome: "parked" };
         }
         catch (err) {
             this.#logger.warn("content.park.deposit.failed", {
@@ -404,7 +463,7 @@ export class SessionNodeManager {
                 contentHash: contentHashHex,
                 error: err instanceof Error ? err.message : String(err),
             });
-            return false;
+            return { outcome: "refused", cause: err instanceof Error ? err.message : String(err) };
         }
     }
     // ─── Initialization ──────────────────────────────────────────────────────
@@ -2886,7 +2945,10 @@ export class SessionNodeManager {
     async sendContent(agentName, sessionId, content, contentHash, correlationId) {
         const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
         if (!entry) {
-            return { ok: false, reason: "session_node_unavailable", error: "no active session node for this session" };
+            // M12-P13: no node, so nothing was witnessed and nothing was queued — the caller must NOT
+            // commit a leaf for this. `durable` is a required field precisely so a new failure branch
+            // cannot be added without answering the question every caller now asks.
+            return { ok: false, reason: "session_node_unavailable", error: "no active session node for this session", durable: false };
         }
         // R1 (MSG-001-3b): witness the message-leaf HASH to the relay FIRST, INDEPENDENT of
         // direct delivery. The relay is the ordering authority (Structure 2): it assigns the
@@ -2960,6 +3022,14 @@ export class SessionNodeManager {
                 structure1_cbor: orderingS1,
                 structure2_cbor: orderingS2,
             });
+            // Injected dial failure — thrown from inside the try so it lands in exactly the catch the
+            // real connection_lost lands in, and the whole downstream path (untrack → park → durable
+            // enqueue) runs unmodified.
+            if (this.#sendFaultRemaining > 0) {
+                this.#sendFaultRemaining -= 1;
+                this.#logger.warn("content.send.fault.injected", { sessionId, contentHash: Buffer.from(contentHash).toString("hex") });
+                throw new Error("connection_lost: injected direct-send fault");
+            }
             stream.send(lp.encode.single(frame));
             try {
                 await stream.close();
@@ -2977,10 +3047,69 @@ export class SessionNodeManager {
             // DOD-LEAVEMSG-1: the deposit is now AWAITED (was fire-and-forget) so a genuine park success
             // can be reported as "dispatched to relay" instead of a raw stream failure — the operator/
             // agent sees the truth (the message IS in flight, just not direct), not a false negative.
-            const parked = await this.#parkContent(agentName, sessionId, Buffer.from(contentHash).toString("hex"), content, orderingS1, orderingS2);
-            if (parked) {
+            const hashHex = Buffer.from(contentHash).toString("hex");
+            const attempt = await this.#parkContent(agentName, sessionId, hashHex, content, orderingS1, orderingS2);
+            if (attempt.outcome === "parked") {
                 return { ok: true, delivered: false, parked: true };
             }
+            // M12-P12: the deposit was refused, and #untrackAwaitingAck above already dropped the
+            // in-memory entry — so without this, NOTHING holds the content and the TTF timer that would
+            // have enqueued it is cancelled. The recipient has already witnessed this sequence, so it
+            // holds every later message in the session behind the gap, forever, and tells no one.
+            // Enqueue durably instead; the drain hook re-parks it the moment the standing receiver is
+            // rebuilt. Only on a REFUSAL — a successful deposit must not be re-parked, and an
+            // unconfigured session has no park target to retry against (F6).
+            let durable = false;
+            if (attempt.outcome === "refused") {
+                try {
+                    // M12-P13 (review HIGH-1): `durable` is now OBSERVED from the enqueue, not asserted around
+                    // it. Two ways this used to lie, both of which now commit a chain leaf and so cannot be
+                    // allowed to: the queue's content-derived dedupe key collides and it silently drops the
+                    // copy, and the `?.` no-ops entirely when the composition root never wired the hook. An
+                    // absent hook is not a queue.
+                    if (this.#onParkFailed === null) {
+                        this.#logger.error("content.park.durable_enqueue.unwired", {
+                            sessionId, contentHash: hashHex, agentName,
+                            impact: "no durable queue is wired — the content is NOT retained and will NOT be retried",
+                        });
+                    }
+                    else {
+                        durable = this.#onParkFailed(agentName, sessionId, hashHex, content, orderingS1, orderingS2);
+                    }
+                    if (!durable) {
+                        if (this.#onParkFailed !== null) {
+                            this.#logger.error("content.park.durable_enqueue.dropped", {
+                                sessionId, contentHash: hashHex, agentName,
+                                impact: "the durable queue refused this copy (identical content already queued) — it is NOT separately retained",
+                            });
+                        }
+                    }
+                    else {
+                        // F5: the successful enqueue must be visible. Without this the live run that has to
+                        // PROVE this fix has nothing to point at, and this log is the sender-side counterpart to
+                        // `session.content.held` on the receiver — the two together make the trace readable.
+                        // M12-P13: `witnessed` rides along because the caller is about to commit a hash-chain
+                        // leaf on the strength of this. Without a relay ordering record the recipient recovers
+                        // in arrival order instead — the accepted degradation, but it must not be invisible.
+                        this.#logger.info("content.park.deferred", {
+                            sessionId, contentHash: hashHex, agentName,
+                            selfOrdering: Boolean(orderingS1 && orderingS2),
+                            witnessed: Boolean(orderingS1 && orderingS2),
+                        });
+                    }
+                }
+                catch (hookErr) {
+                    // F3: enqueueAwaitingContent throws ON PURPOSE when the persist fails, because that is
+                    // data loss. Swallowing it into the same response the durable case returns would tell the
+                    // operator "it will retry" about a message that is simply gone. Named for its own cause,
+                    // and the response says so below.
+                    this.#logger.error("content.park.durable_enqueue.failed", {
+                        sessionId, contentHash: hashHex, agentName,
+                        impact: "content is NOT durable and will NOT be retried — the message is lost",
+                        error: hookErr instanceof Error ? hookErr.message : String(hookErr),
+                    });
+                }
+            }
             // error.message extracted — never [object Object]. libp2p/cross-package errors are not
             // always `instanceof Error` in this realm, so fall back to a message property / JSON.
             const errMsg = err instanceof Error
@@ -2995,7 +3124,28 @@ export class SessionNodeManager {
                             return String(err);
                         }
                     })();
-            return { ok: false, reason: "session_stream_unavailable", error: errMsg };
+            // F3: the two failures are NOT interchangeable to the caller. `reason` is a contract string
+            // and stays put; `guidance` carries the difference, because "we are retrying this" and "this
+            // message is gone, send it again" demand opposite actions from the operator.
+            return {
+                ok: false,
+                reason: "session_stream_unavailable",
+                error: errMsg,
+                // M12-P13: the machine-readable half of the distinction below. M12-P12 shipped it in the
+                // guidance SENTENCE only, so the callers that have to ACT on it — commit the leaf for a
+                // queued message, never for a lost one — would have had to substring-match English. None
+                // did, and the sequence the relay had already witnessed was left as a permanent hole.
+                durable,
+                // M12-P13 (review MEDIUM-5): the specific standing-receiver state, carried rather than
+                // discarded. `reason` names where this surfaced; `cause` names what actually blocked it —
+                // the exact distinction M12-P12 added `standingReceiverAbsenceReason()` for, which then
+                // died inside #parkContent. An operator keying on `reason` alone is sent to the transport
+                // when the blocker is the receiver.
+                ...(attempt.cause !== undefined ? { cause: attempt.cause } : {}),
+                guidance: durable
+                    ? "Direct delivery failed and the relay refused the hand-off, so the message is queued and will be re-sent automatically when the relay link is back. Do not re-send it: an identical re-send is not separately queued."
+                    : "Direct delivery failed and the message could NOT be queued for retry — it is lost. Send it again.",
+            };
         }
     }
     /**
@@ -3966,7 +4116,10 @@ export class SessionNodeManager {
             this.#awaitingAck.delete(ackKey);
         this.#logger.debug("content.delivery.ttf_expired", { sessionId, contentHash: hashHex });
         try {
-            this.#onAwaitingTtf?.(agentName, sessionId, hashHex, entry.content);
+            // M12-P12 (review pass 2): the ordering record travels on THIS path too. It is in hand — the
+            // very next statement hands it to #parkContent — and a TTF row written without it re-parks in
+            // arrival order, which is the divergent-leaf-index failure the durable columns exist to stop.
+            this.#onAwaitingTtf?.(agentName, sessionId, hashHex, entry.content, entry.structure1Cbor, entry.structure2Cbor);
         }
         catch (err) {
             this.#logger.error("content.park.backstop.failed", {