@cello-protocol/daemon 0.0.173 → 0.0.175

package/dist/session-node-manager.js

@@ -118,6 +118,13 @@ const REDIAL_COOLDOWN_MS = 15_000;
  * Two hours: comfortably longer than any churn worth attacking with, comfortably shorter than
  * "yesterday's conversation still blocks me".
  */
+/**
+ * DOD-M12B-RESERVATION-RETRY-1 — how many times to re-ask for a refused reservation before saying
+ * the agent is undialable and stopping. Bounded because a reservation is scarce and a fleet that
+ * retries forever is how a relay is exhausted. With the 5-minute base and doubling, five retries
+ * span about 2.5 hours.
+ */
+export const SR_RESERVATION_MAX_RETRIES = 5;
 export const CAP_INTERRUPTED_TTL_MS = Number(process.env["CELLO_CAP_INTERRUPTED_TTL_MS"]) || 2 * 60 * 60 * 1000;
 /**
  * DOD-CAP-SELF-HEAL-1 — what counts against a per-sender acceptance bound.
@@ -293,6 +300,15 @@ export class SessionNodeManager {
     #srReservationTimeoutMs;
     /** DOD-NAT-REACHABILITY-1: watchdog for a SILENTLY lost reservation. */
     #srWatchdogIntervalMs;
+    #srReservationRetryMs;
+    /**
+     * DOD-M12B-RESERVATION-RETRY-1: per-agent re-attempt state for a receiver the relay refused a
+     * reservation to. `attempts` counts RETRIES, not the creation attempt.
+     */
+    /** The reason the last reservation attempt was refused, per agent — captured at the rejection so
+     *  the retry and give-up can name a CAUSE instead of only their own exit point. */
+    #srLastRejectionReason = new Map();
+    #srReservationRetry = new Map();
     #reservationWatchdog = null;
     /** DOD-PARK-DRAIN-1: how often the backstop drain rides the watchdog grid — see #parkedDrainBackstopTick. */
     #parkedDrainBackstopMs;
@@ -540,6 +556,7 @@ export class SessionNodeManager {
         this.#srRetryDelaysMs = opts.standingReceiverRetryDelaysMs ?? [1_000, 5_000, 15_000];
         this.#srReservationTimeoutMs = opts.standingReceiverReservationTimeoutMs ?? 15_000;
         this.#srWatchdogIntervalMs = opts.standingReceiverWatchdogIntervalMs ?? 30_000;
+        this.#srReservationRetryMs = opts.standingReceiverReservationRetryMs ?? 5 * 60_000;
         this.#parkedDrainBackstopMs = opts.parkedDrainBackstopMs ?? 300_000;
         // REQUIRED, no fallback (INV-9, audit finding). This line used to read
         // `opts.securityGateway ?? new PassthroughGatewayClient()` — the identical shape as the defect
@@ -1944,6 +1961,31 @@ export class SessionNodeManager {
             return this.#standingReceivers.has(agentName);
         return this.#standingReceivers.size > 0;
     }
+    /**
+     * DOD-M12B-RESERVATION-RETRY-1 — whether a NAT'd peer can actually DIAL this agent.
+     *
+     * `standing_receiver_ready` answers "is there a receiver?", which is true for a plain TCP node
+     * that no relay would give a circuit reservation to. Behind NAT that node is reachable by nobody,
+     * and the difference was visible only in the log — where it was visible 481 times and nobody
+     * acted. `"retrying"` and `"unreachable"` are the states an operator can do something about.
+     *
+     *   reserved    — holds a circuit reservation; a NAT'd peer can dial it.
+     *   retrying    — no reservation yet, still re-asking on a backoff.
+     *   unreachable — no circuit reservation and the automatic re-attempts are spent, so only peers
+     *                 that can connect DIRECTLY will get in. It is not permanent: a directory
+     *                 reconnect carrying a DIFFERENT relay pool re-arms the budget, because a relay we
+     *                 have never tried is new information.
+     *   absent      — no receiver at all (the agent is not online).
+     */
+    getStandingReceiverReachability(agentName) {
+        const sr = this.#standingReceivers.get(agentName);
+        if (!sr)
+            return "absent";
+        if (sr.hasReservation && sr.relayPeerId !== undefined)
+            return "reserved";
+        const retry = this.#srReservationRetry.get(agentName);
+        return retry !== undefined && retry.attempts > SR_RESERVATION_MAX_RETRIES ? "unreachable" : "retrying";
+    }
     /** First ready standing receiver (any agent) — for agent-agnostic OUTBOUND use (gater-open). */
     #anyStandingReceiver() {
         for (const sr of this.#standingReceivers.values())
@@ -3127,6 +3169,8 @@ export class SessionNodeManager {
             }
         })), "standing_receivers", this.#standingReceivers.size);
         this.#standingReceivers.clear();
+        this.#srReservationRetry.clear();
+        this.#srLastRejectionReason.clear();
         // Release the SQLite handle so the DB file is no longer held open after shutdown
         // (review L5). Queries guard on `#db === null` and degrade to empty/null.
         if (this.#db) {
@@ -6794,7 +6838,21 @@ export class SessionNodeManager {
      * for a session handoff that (being unreachable) would never come.
      */
     setDirectoryRelayEndpoints(agentName, endpoints) {
+        // DOD-M12B-RESERVATION-RETRY-1: a DIFFERENT relay pool re-arms the retry budget; the same one
+        // does not. This fires on every signaling connect AND reconnect, so clearing unconditionally
+        // would reset the bound on a short grid and defeat the whole point of having one. But once the
+        // budget is spent, `getStandingReceiverReachability` reports `unreachable` for the rest of the
+        // online episode — including while this path is actively re-attempting against relays we have
+        // never tried. A new relay is new information; a repeat of the same list is not.
+        const previous = this.#directoryRelayEndpoints.get(agentName);
+        const poolChanged = previous === undefined ||
+            previous.length !== endpoints.length ||
+            endpoints.some((e, i) => e.relayPeerId !== previous[i]?.relayPeerId);
         this.#directoryRelayEndpoints.set(agentName, endpoints);
+        if (poolChanged) {
+            this.#srReservationRetry.delete(agentName);
+            this.#srLastRejectionReason.delete(agentName);
+        }
         if (endpoints.length === 0 || this.#shuttingDown)
             return;
         const sr = this.#standingReceivers.get(agentName);
@@ -6919,14 +6977,97 @@ export class SessionNodeManager {
      * already degraded and already loud (reservation.none / reservation.timeout);
      * rebuilding it on a timer would just thrash against relays we know are refusing.
      */
+    /**
+     * DOD-M12B-RESERVATION-RETRY-1 — ask again for a reservation the relay refused.
+     *
+     * The rebuild is the re-attempt: a circuit listener is fixed at node creation, so the only way to
+     * obtain a reservation is to build a new node asking for one.
+     */
+    #retryReservationIfDue(agentName) {
+        const now = Date.now();
+        const state = this.#srReservationRetry.get(agentName)
+            ?? { attempts: 0, nextAt: now + this.#srReservationRetryMs, correlationId: randomUUID() };
+        // The reason the LAST attempt was refused, captured where it is actually known.
+        const lastReason = this.#srLastRejectionReason.get(agentName);
+        if (lastReason !== undefined)
+            state.lastReason = lastReason;
+        if (state.attempts === 0 && !this.#srReservationRetry.has(agentName)) {
+            // First sighting — schedule, do not fire. The creation attempt just happened.
+            this.#srReservationRetry.set(agentName, state);
+            return;
+        }
+        if (now < state.nextAt)
+            return;
+        if (state.attempts >= SR_RESERVATION_MAX_RETRIES) {
+            if (state.attempts === SR_RESERVATION_MAX_RETRIES) {
+                state.attempts += 1; // mark as reported, so this fires exactly once
+                this.#srReservationRetry.set(agentName, state);
+                this.#logger.error("session.standing_receiver.reservation.gave_up", {
+                    agentName,
+                    attempts: SR_RESERVATION_MAX_RETRIES,
+                    correlationId: state.correlationId,
+                    // WHY, not just the consequence. Three different problems reach this one message and they
+                    // need three different responses: `relay_granted_no_reservation` is relay CAPACITY (and a
+                    // trustless-cello problem), `relay_unreachable` is the NETWORK, and
+                    // `reservation_did_not_complete_in_time` is LATENCY — and the only one of the three that
+                    // can pin a slot it never uses, so its appearance is also the signal that this retry
+                    // budget needs tightening.
+                    ...(state.lastReason !== undefined ? { lastRejectionReason: state.lastReason } : {}),
+                    // "No relay would grant" and "there was no relay to ask" are different facts and lead to
+                    // different places — the first at relay capacity, the second at this agent's directory
+                    // connection. Without this they are the same sentence.
+                    reservationsRequested: (this.#directoryRelayEndpoints.get(agentName)?.length ?? 0) > 0,
+                    impact: "no relay would grant this agent a circuit reservation, so anyone behind NAT cannot reach or dial it — inbound sessions will only arrive from peers that can connect directly, and everything else falls back to the relay's store-and-forward",
+                });
+            }
+            return;
+        }
+        state.attempts += 1;
+        // The FINAL attempt gets a fixed settle window rather than another doubled wait: at the top of
+        // the ladder that would be 80 minutes of silence after the last thing we did, which is a long
+        // time to tell an operator nothing. Every earlier attempt doubles, which is what keeps a fleet
+        // off a scarce relay.
+        state.nextAt = now + (state.attempts >= SR_RESERVATION_MAX_RETRIES
+            ? this.#srReservationRetryMs
+            : this.#srReservationRetryMs * 2 ** (state.attempts - 1));
+        this.#srReservationRetry.set(agentName, state);
+        this.#logger.warn("session.standing_receiver.reservation.retry", {
+            agentName,
+            attempt: state.attempts,
+            maxAttempts: SR_RESERVATION_MAX_RETRIES,
+            correlationId: state.correlationId,
+            ...(state.lastReason !== undefined ? { lastRejectionReason: state.lastReason } : {}),
+            impact: "this agent currently holds no circuit reservation, so a NAT'd peer cannot dial it",
+        });
+        void this.#rebuildStandingReceiver(agentName);
+    }
     #reservationWatchdogTick() {
         if (this.#shuttingDown)
             return;
         for (const [agentName, sr] of this.#standingReceivers) {
-            if (!sr.hasReservation || sr.relayPeerId === undefined)
-                continue; // never had one — not a LOSS
             if (!this.#agentsWantingReceiver.has(agentName))
                 continue; // agent went offline
+            // DOD-M12B-RESERVATION-RETRY-1 — NEVER HAD ONE IS NOT "NOTHING TO DO".
+            //
+            // This used to `continue` unconditionally, on the grounds that a receiver with no reservation
+            // is "already degraded and already loud". Measured over 17 days: `reservation.none` fired 481
+            // times and `relay.rejected` 2,215 — every one of the latter `relay_granted_no_reservation`,
+            // a relay out of slots completing the handshake and granting nothing. Nothing ever acted on
+            // the noise, and each of those receivers is a plain TCP node with no circuit address: behind
+            // NAT, dialable by NOBODY, for its whole life. That is the silent loss of inbound this file
+            // says three lines below it exists to kill.
+            //
+            // A relay out of slots at boot may have one minutes later, so re-attempt — on a BACKOFF and
+            // BOUNDED, never on this 30-second grid. A reservation is scarce: the relay holds it for its
+            // full TTL even after the client disconnects, and churning attempts across a fleet is how a
+            // relay is exhausted (`#startReceiverNode` records that hazard).
+            if (!sr.hasReservation || sr.relayPeerId === undefined) {
+                this.#retryReservationIfDue(agentName);
+                continue;
+            }
+            // It has one — any earlier retry budget, and the reason the last attempt failed, are stale.
+            this.#srReservationRetry.delete(agentName);
+            this.#srLastRejectionReason.delete(agentName);
             // Watch the CONNECTION to the relay, not the circuit address.
             //
             // Killing the relay does NOT make the /p2p-circuit address disappear: libp2p
@@ -7057,14 +7198,16 @@ export class SessionNodeManager {
             if (outcome === "started" && candidate.listenAddresses().some((a) => a.includes("/p2p-circuit"))) {
                 return candidate;
             }
+            const rejectionReason = outcome === "started"
+                ? "relay_granted_no_reservation"
+                : outcome === "failed"
+                    ? "relay_unreachable"
+                    : "reservation_did_not_complete_in_time";
+            this.#srLastRejectionReason.set(agentName, rejectionReason);
             this.#logger.warn("session.standing_receiver.relay.rejected", {
                 agentName,
                 circuitAddr,
-                reason: outcome === "started"
-                    ? "relay_granted_no_reservation"
-                    : outcome === "failed"
-                        ? "relay_unreachable"
-                        : "reservation_did_not_complete_in_time",
+                reason: rejectionReason,
                 ...(error !== "" ? { error } : {}),
                 correlationId,
             });
@@ -7146,8 +7289,27 @@ export class SessionNodeManager {
         });
         autoNat.emitInitialResult();
         const circuitAddrs = node.listenAddresses().filter((a) => a.includes("/p2p-circuit")).length;
-        // The relay we actually reserved with — the watchdog watches our connection to it.
-        const reservedRelayPeerId = circuitAddrs > 0 ? reservations.addrs[0]?.match(/\/p2p\/([^/]+)\/p2p-circuit/)?.[1] : undefined;
+        // FROM THE ADDRESS THE NODE ACTUALLY HOLDS, not from `reservations.addrs[0]`.
+        //
+        // `#startReceiverNode` tries candidates in order and returns the FIRST that actually grants —
+        // so when candidate 0 refuses (the measured `relay_granted_no_reservation` case) and candidate 1
+        // grants, reading candidate 0's address records a relay we are not connected to. The watchdog
+        // then evaluates `getConnections().some(c => c.peerId === relayPeerId)` against that wrong peer,
+        // finds it false on every tick forever, and rebuilds on the 30-second grid — churning the very
+        // reservations this unit exists to conserve. Dormant while the pool is size 1; the pool is
+        // designed to be larger.
+        // PREFER the held address, FALL BACK to the candidate — strictly better than either alone.
+        // The held address is authoritative about which relay actually granted, but it is libp2p's
+        // string, not ours: if a transport ever reports the circuit address without the relay's peer id
+        // in `/p2p/<id>/p2p-circuit` form, reading only it would yield UNDEFINED, and an undefined
+        // relayPeerId makes the watchdog treat a perfectly healthy reservation as absent and rebuild it.
+        // That would be a regression on the single-relay case that works today. The candidate string is
+        // ours and always carries the id, so it is the safe floor.
+        const heldCircuitAddr = node.listenAddresses().find((a) => a.includes("/p2p-circuit"));
+        const CIRCUIT_RELAY_ID = /\/p2p\/([^/]+)\/p2p-circuit/;
+        const reservedRelayPeerId = circuitAddrs > 0
+            ? (heldCircuitAddr?.match(CIRCUIT_RELAY_ID)?.[1] ?? reservations.addrs[0]?.match(CIRCUIT_RELAY_ID)?.[1])
+            : undefined;
         this.#standingReceivers.set(agentName, {
             node,
             gater,
@@ -7205,6 +7367,14 @@ export class SessionNodeManager {
         // gets a fresh set on its next connect — holding the old ones would keep a
         // retired agent's relay list alive for the daemon's lifetime.
         this.#directoryRelayEndpoints.delete(agentName);
+        // DOD-M12B-RESERVATION-RETRY-1: and the retry budget with them. A spent budget that survives an
+        // offline→online cycle is a LATCH: the new receiver gets no reservation, the watchdog finds
+        // `attempts` already past the cap, and returns having done nothing — no retry and not even a
+        // second give-up. The agent is undialable and the machinery is inert and mute until a daemon
+        // restart. Same reason the line above exists ("holding the old ones would keep a retired agent's
+        // relay list alive for the daemon's lifetime").
+        this.#srReservationRetry.delete(agentName);
+        this.#srLastRejectionReason.delete(agentName);
         const sr = this.#standingReceivers.get(agentName);
         if (!sr) {
             // L1: an #ensureStandingReceiver for this agent may be in flight (parked on start(), so no