@cello-protocol/daemon 0.0.175 → 0.0.177
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/close-session-handler.d.ts.map +1 -1
- package/dist/close-session-handler.js +17 -7
- package/dist/close-session-handler.js.map +1 -1
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +93 -3
- package/dist/daemon.js.map +1 -1
- package/dist/restart-seal-resolver.d.ts +24 -6
- package/dist/restart-seal-resolver.d.ts.map +1 -1
- package/dist/restart-seal-resolver.js +56 -3
- package/dist/restart-seal-resolver.js.map +1 -1
- package/dist/session-content-handlers.d.ts.map +1 -1
- package/dist/session-content-handlers.js +97 -2
- package/dist/session-content-handlers.js.map +1 -1
- package/dist/session-node-manager.d.ts +313 -7
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +1225 -62
- package/dist/session-node-manager.js.map +1 -1
- package/package.json +5 -5
|
@@ -26,7 +26,7 @@ import { TIER, normalizeTier, isKnownTierValue, tierBoundsFor, DEFAULT_TIER_BOUN
|
|
|
26
26
|
import { migrateCborBlobsToCanonical } from "./cbor-blob-migration.js";
|
|
27
27
|
import { ensureTrustSignalSchema } from "./trust-signal-store.js";
|
|
28
28
|
import { boundSettingKey, settableTierName, isValidSettingKey, awayTierSettingKey, AWAY_DEFAULT_KEY } from "./agent-settings-keys.js";
|
|
29
|
-
import { randomUUID, createHash } from "node:crypto";
|
|
29
|
+
import { randomUUID, createHash, randomBytes } from "node:crypto";
|
|
30
30
|
import * as lp from "it-length-prefixed";
|
|
31
31
|
import { decode } from "cbor-x";
|
|
32
32
|
import { encodeCbor } from "@cello-protocol/protocol-types";
|
|
@@ -173,6 +173,33 @@ export const ABUSE_MAX_UNKNOWN_SESSIONS_GLOBAL = 50;
|
|
|
173
173
|
* the leak the old destructive read was accidentally preventing.
|
|
174
174
|
*/
|
|
175
175
|
const RECEIVED_BUFFER_CAP = 32;
|
|
176
|
+
/**
|
|
177
|
+
* DOD-M12B-REVIVAL-BOUND-1 — how long an interrupted session stays revivable before it is closed.
|
|
178
|
+
*
|
|
179
|
+
* 24 hours. The bound exists because of Andre's 2026-08-18 tenet — *"leave nothing open that is no
|
|
180
|
+
* longer needed"* — and its value is set by the case it must not break: a laptop closed for the
|
|
181
|
+
* night. A window shorter than a night's sleep would abandon exactly the sessions case A/B exist to
|
|
182
|
+
* rescue, which is why this is not zero and not an hour.
|
|
183
|
+
*
|
|
184
|
+
* It is deliberately a plain constant and not a setting. A per-operator knob here is a knob that
|
|
185
|
+
* turns the guarantee off, and the guarantee is the security property, not a preference.
|
|
186
|
+
*/
|
|
187
|
+
export const REVIVAL_WINDOW_MS = 24 * 60 * 60 * 1000;
|
|
188
|
+
/**
|
|
189
|
+
* DOD-M12B-REVIVAL-BOUND-1 — how often the revival bound is applied.
|
|
190
|
+
*
|
|
191
|
+
* Hourly. The window is 24 hours, so the worst-case overshoot is ~4% of the bound, and the pass is
|
|
192
|
+
* one DB walk with no network in it. Boot-only was the first build and it is not a bound at all: a
|
|
193
|
+
* daemon left up for a week never applies it, and a long-lived daemon is the normal case.
|
|
194
|
+
*/
|
|
195
|
+
export const REVIVAL_BOUND_SWEEP_MS = 60 * 60 * 1000;
|
|
196
|
+
/** DOD-M12B-SESSION-SEED-1: per-relay deadline when a revived node asks for a circuit reservation.
|
|
197
|
+
* Three seconds — a relay that has a slot answers well inside it, and one that does not never
|
|
198
|
+
* answers at all (measured: 10,002ms and still waiting). */
|
|
199
|
+
export const REVIVE_RESERVATION_TIMEOUT_MS = 3_000;
|
|
200
|
+
/** How many relays a revival will ask before settling for a plain node. Two: the worst case is then
|
|
201
|
+
* ~6s to a usable session, against a measured "never" for the unbounded form. */
|
|
202
|
+
export const REVIVE_RESERVATION_CANDIDATES = 2;
|
|
176
203
|
export class SessionNodeManager {
|
|
177
204
|
#factory;
|
|
178
205
|
#logger;
|
|
@@ -287,6 +314,31 @@ export class SessionNodeManager {
|
|
|
287
314
|
// `hasReservation`: this receiver came up holding a /p2p-circuit address. The
|
|
288
315
|
// watchdog uses it to tell "lost its reservation" (must recover) apart from
|
|
289
316
|
// "never had one" (already degraded, and already loud) — see #reservationWatchdogTick.
|
|
317
|
+
/**
|
|
318
|
+
* DOD-M12B-SESSION-SEED-1 — session id → the transport seed its node identity derives from.
|
|
319
|
+
*
|
|
320
|
+
* **DELIBERATELY NOT ON `ActiveSessionEntry`.** That entry is deleted the instant a session is
|
|
321
|
+
* interrupted (`markInterruptedWithDetails` and `destroySessionNode` both do it), which is exactly
|
|
322
|
+
* the moment the seed becomes necessary — storing it there would destroy it precisely when the
|
|
323
|
+
* session needs to come back. It has to outlive the NODE without outliving the SESSION.
|
|
324
|
+
*
|
|
325
|
+
* Andre's 2026-08-18 tenet is the lifetime rule: *"it should be possible to revive that session on
|
|
326
|
+
* those peer IDs. But after that, those peer IDs and that peer connection needs to be shut down."*
|
|
327
|
+
* So this map is cleared in the same step that writes a terminal status (`#updateSessionStatus`),
|
|
328
|
+
* not on a later sweep — and the bytes are zeroed before the reference is dropped, because a seed
|
|
329
|
+
* that stays readable in the heap is exactly the "left open" the tenet forbids.
|
|
330
|
+
*
|
|
331
|
+
* It holds the counterparty's session peer id alongside the seed, and not for convenience: that
|
|
332
|
+
* value lives ONLY on `ActiveSessionEntry` and is destroyed with it on interruption, so without a
|
|
333
|
+
* copy here we could rebuild our own identity and still not know who to let back in. Both halves
|
|
334
|
+
* of the revival have the same lifetime, so they are destroyed in one step.
|
|
335
|
+
*
|
|
336
|
+
* In memory only, never persisted. A daemon restart genuinely destroys these identities, which is
|
|
337
|
+
* why restart is `RESTART-SEAL-1`'s case (resolve with a receipt) and not a revival case.
|
|
338
|
+
*/
|
|
339
|
+
#sessionSeeds = new Map();
|
|
340
|
+
/** DOD-M12B-SESSION-SEED-1: see setRetryDrainHook — fired when a session is revived. */
|
|
341
|
+
#retryDrainHook = null;
|
|
290
342
|
#standingReceivers = new Map();
|
|
291
343
|
#standingReceiverCreating = new Set();
|
|
292
344
|
// M8B F14: agents that SHOULD have a standing receiver — marked by
|
|
@@ -2176,6 +2228,9 @@ export class SessionNodeManager {
|
|
|
2176
2228
|
let node;
|
|
2177
2229
|
let gater;
|
|
2178
2230
|
let autoNat;
|
|
2231
|
+
// DOD-M12B-SESSION-SEED-1: whichever branch below runs, the session ends up owning a seed.
|
|
2232
|
+
// Promotion inherits the receiver's; a freshly-built node mints its own.
|
|
2233
|
+
let seed;
|
|
2179
2234
|
if (reuseStandingReceiver) {
|
|
2180
2235
|
const sr = this.#standingReceivers.get(agentName);
|
|
2181
2236
|
if (!sr) {
|
|
@@ -2189,7 +2244,7 @@ export class SessionNodeManager {
|
|
|
2189
2244
|
guidance: "The standing receiver node is initializing (completes within 200ms). Retry the session in a moment.",
|
|
2190
2245
|
};
|
|
2191
2246
|
}
|
|
2192
|
-
({ node, gater, autoNat } = sr);
|
|
2247
|
+
({ node, gater, autoNat, seed } = sr);
|
|
2193
2248
|
gater.setAllowedPeer(counterpartyPeerId);
|
|
2194
2249
|
// Hand this agent's standing receiver off to this session; a replacement is spun up below.
|
|
2195
2250
|
this.#standingReceivers.delete(agentName);
|
|
@@ -2201,7 +2256,8 @@ export class SessionNodeManager {
|
|
|
2201
2256
|
logger: this.#logger,
|
|
2202
2257
|
});
|
|
2203
2258
|
try {
|
|
2204
|
-
|
|
2259
|
+
seed = randomBytes(32);
|
|
2260
|
+
node = await this.#factory.createNode({ sessionId, connectionGater: gater, nodeType: "session", transportPrivateKey: seed });
|
|
2205
2261
|
await node.start();
|
|
2206
2262
|
}
|
|
2207
2263
|
catch (err) {
|
|
@@ -2262,7 +2318,7 @@ export class SessionNodeManager {
|
|
|
2262
2318
|
// Log observability event (session.node.created)
|
|
2263
2319
|
//
|
|
2264
2320
|
// `counterpartySessionPeerId` IS LOGGED because it is recorded here ONCE and never refreshed,
|
|
2265
|
-
// while a standing receiver is rebuilt with a fresh libp2p keypair on
|
|
2321
|
+
// while a standing receiver is rebuilt with a fresh libp2p keypair on a lost relay reservation
|
|
2266
2322
|
// and every lost reservation. If the peer rebuilds between advertising its endpoint and this
|
|
2267
2323
|
// handoff, we record an identity that no longer exists — and since `newStream` never dials, it
|
|
2268
2324
|
// only ever looks for an ALREADY-OPEN connection filed under exactly this string, so every send
|
|
@@ -2288,6 +2344,7 @@ export class SessionNodeManager {
|
|
|
2288
2344
|
counterpartySessionPeerId: counterpartyPeerId,
|
|
2289
2345
|
autoNat,
|
|
2290
2346
|
});
|
|
2347
|
+
this.#rememberSessionSeed(agentName, sessionId, seed, counterpartyPeerId, counterpartyPubkey);
|
|
2291
2348
|
// DAEMON-004: register the content stream handler so inbound content_frames
|
|
2292
2349
|
// are cross-checked, appended to the daemon-owned tree, and buffered.
|
|
2293
2350
|
await this.#registerContentHandler(agentName, sessionId, node, counterpartyPubkey);
|
|
@@ -2311,6 +2368,60 @@ export class SessionNodeManager {
|
|
|
2311
2368
|
* and leaves relayClient undefined — the session is NOT destroyed and the direct
|
|
2312
2369
|
* content path keeps working (the relay-park/recovery path is MSG-001-3b's domain).
|
|
2313
2370
|
*/
|
|
2371
|
+
/**
|
|
2372
|
+
* DOD-M12B-REVIVE-RELAY-1 — the relay witness leaf handler, shared by establishment and revival.
|
|
2373
|
+
*
|
|
2374
|
+
* Extracted because a REVIVED session must register the same handler. It was inline in
|
|
2375
|
+
* `#connectSessionRelay`, so revival — which never called that at all — had no live inbound path:
|
|
2376
|
+
* every message fell back to the five-minute mailbox poll, which is why a reconnected session took
|
|
2377
|
+
* three minutes to deliver what a fresh one delivers in seconds, and why doorbells stopped firing.
|
|
2378
|
+
*
|
|
2379
|
+
* A revived session that behaves differently from a fresh one is the defect. This is one of the
|
|
2380
|
+
* two halves of making them the same.
|
|
2381
|
+
*/
|
|
2382
|
+
#relayLeafHandler(agentName, sessionId, correlationId) {
|
|
2383
|
+
return (frame) => {
|
|
2384
|
+
// The counterparty's witnessed leaf arrived with its canonical sequence. The
|
|
2385
|
+
// plaintext is delivered separately over the direct content stream; this is the
|
|
2386
|
+
// ordering/witness signal. Full canonical-sequence reconciliation against the
|
|
2387
|
+
// local tree is MSG-001-3b (J-CONTENT).
|
|
2388
|
+
this.#logger.info("session.relay.leaf.delivered", {
|
|
2389
|
+
sessionId,
|
|
2390
|
+
sequenceNumber: frame.sequence_number,
|
|
2391
|
+
leafKind: frame.leaf_kind,
|
|
2392
|
+
correlationId,
|
|
2393
|
+
});
|
|
2394
|
+
// DOD-MSG-4 (strict in-order): record the relay-witnessed canonical sequence for the
|
|
2395
|
+
// counterparty's MSG leaves. The relay is the ordering authority; structure1_cbor =
|
|
2396
|
+
// [1, content_hash(32), sender_pubkey, session_id, last_seen_seq, ts]. The relay sequence
|
|
2397
|
+
// is 1-based and global per session; the daemon tree is 0-based — normalize with -1. Only
|
|
2398
|
+
// COUNTERPARTY leaves (the ones B will ingest); our own echoed leaf already lands via the
|
|
2399
|
+
// send path. The gate (ingestReceivedContent) reads this map to hold out-of-order arrivals.
|
|
2400
|
+
if (!frame.authored_by_us && frame.leaf_kind !== LEAF_KIND_CTRL) {
|
|
2401
|
+
try {
|
|
2402
|
+
const s1 = decode(frame.structure1_cbor);
|
|
2403
|
+
const contentHash = s1?.[1];
|
|
2404
|
+
if (contentHash instanceof Uint8Array && frame.sequence_number > 0) {
|
|
2405
|
+
this.recordWitnessedSequence(agentName, sessionId, Buffer.from(contentHash).toString("hex"), frame.sequence_number - 1);
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
catch (err) {
|
|
2409
|
+
this.#logger.warn("session.relay.leaf.witness.decode.failed", {
|
|
2410
|
+
sessionId,
|
|
2411
|
+
error: err instanceof Error ? err.message : String(err),
|
|
2412
|
+
correlationId,
|
|
2413
|
+
});
|
|
2414
|
+
}
|
|
2415
|
+
}
|
|
2416
|
+
// M7-UPGRADE-002: auto-acknowledge close. When the COUNTERPARTY's SEAL ctrl leaf (0x02)
|
|
2417
|
+
// arrives and B has verified the content, B's OWN node auto-co-signs the responder SEAL
|
|
2418
|
+
// leaf — no agent prompt — so the bilateral seal completes promptly instead of degrading
|
|
2419
|
+
// to unilateral on a slow/busy/crashed agent. Never auto-ack our OWN echoed ctrl leaf.
|
|
2420
|
+
if (frame.leaf_kind === LEAF_KIND_CTRL && !frame.authored_by_us) {
|
|
2421
|
+
this.#maybeAutoAcknowledgeSeal(agentName, sessionId, correlationId);
|
|
2422
|
+
}
|
|
2423
|
+
};
|
|
2424
|
+
}
|
|
2314
2425
|
async #connectSessionRelay(sessionId, node, agentName, relay, correlationId) {
|
|
2315
2426
|
try {
|
|
2316
2427
|
// The session node's gater admits only the counterparty; the relay witness is a
|
|
@@ -2344,47 +2455,7 @@ export class SessionNodeManager {
|
|
|
2344
2455
|
this.#relayClients.set(clientKey, client);
|
|
2345
2456
|
}
|
|
2346
2457
|
const sessionIdHexForRelay = Buffer.from(relay.sessionIdBytes).toString("hex");
|
|
2347
|
-
client.registerSession(sessionIdHexForRelay, node, (
|
|
2348
|
-
// The counterparty's witnessed leaf arrived with its canonical sequence. The
|
|
2349
|
-
// plaintext is delivered separately over the direct content stream; this is the
|
|
2350
|
-
// ordering/witness signal. Full canonical-sequence reconciliation against the
|
|
2351
|
-
// local tree is MSG-001-3b (J-CONTENT).
|
|
2352
|
-
this.#logger.info("session.relay.leaf.delivered", {
|
|
2353
|
-
sessionId,
|
|
2354
|
-
sequenceNumber: frame.sequence_number,
|
|
2355
|
-
leafKind: frame.leaf_kind,
|
|
2356
|
-
correlationId,
|
|
2357
|
-
});
|
|
2358
|
-
// DOD-MSG-4 (strict in-order): record the relay-witnessed canonical sequence for the
|
|
2359
|
-
// counterparty's MSG leaves. The relay is the ordering authority; structure1_cbor =
|
|
2360
|
-
// [1, content_hash(32), sender_pubkey, session_id, last_seen_seq, ts]. The relay sequence
|
|
2361
|
-
// is 1-based and global per session; the daemon tree is 0-based — normalize with -1. Only
|
|
2362
|
-
// COUNTERPARTY leaves (the ones B will ingest); our own echoed leaf already lands via the
|
|
2363
|
-
// send path. The gate (ingestReceivedContent) reads this map to hold out-of-order arrivals.
|
|
2364
|
-
if (!frame.authored_by_us && frame.leaf_kind !== LEAF_KIND_CTRL) {
|
|
2365
|
-
try {
|
|
2366
|
-
const s1 = decode(frame.structure1_cbor);
|
|
2367
|
-
const contentHash = s1?.[1];
|
|
2368
|
-
if (contentHash instanceof Uint8Array && frame.sequence_number > 0) {
|
|
2369
|
-
this.recordWitnessedSequence(agentName, sessionId, Buffer.from(contentHash).toString("hex"), frame.sequence_number - 1);
|
|
2370
|
-
}
|
|
2371
|
-
}
|
|
2372
|
-
catch (err) {
|
|
2373
|
-
this.#logger.warn("session.relay.leaf.witness.decode.failed", {
|
|
2374
|
-
sessionId,
|
|
2375
|
-
error: err instanceof Error ? err.message : String(err),
|
|
2376
|
-
correlationId,
|
|
2377
|
-
});
|
|
2378
|
-
}
|
|
2379
|
-
}
|
|
2380
|
-
// M7-UPGRADE-002: auto-acknowledge close. When the COUNTERPARTY's SEAL ctrl leaf (0x02)
|
|
2381
|
-
// arrives and B has verified the content, B's OWN node auto-co-signs the responder SEAL
|
|
2382
|
-
// leaf — no agent prompt — so the bilateral seal completes promptly instead of degrading
|
|
2383
|
-
// to unilateral on a slow/busy/crashed agent. Never auto-ack our OWN echoed ctrl leaf.
|
|
2384
|
-
if (frame.leaf_kind === LEAF_KIND_CTRL && !frame.authored_by_us) {
|
|
2385
|
-
this.#maybeAutoAcknowledgeSeal(agentName, sessionId, correlationId);
|
|
2386
|
-
}
|
|
2387
|
-
}, relay.assignment);
|
|
2458
|
+
client.registerSession(sessionIdHexForRelay, node, this.#relayLeafHandler(agentName, sessionId, correlationId), relay.assignment);
|
|
2388
2459
|
const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
|
|
2389
2460
|
if (entry) {
|
|
2390
2461
|
entry.relayClient = client;
|
|
@@ -2483,6 +2554,26 @@ export class SessionNodeManager {
|
|
|
2483
2554
|
node.onPeerConnect((peerId) => {
|
|
2484
2555
|
if (!isCounterparty(peerId))
|
|
2485
2556
|
return;
|
|
2557
|
+
/**
|
|
2558
|
+
* DOD-M12B-RESPONDER-ADDR-1 (review MEDIUM-4) — LEARN THE ADDRESS HERE, where it cannot race.
|
|
2559
|
+
*
|
|
2560
|
+
* The accept-time read was a race between two independent async chains: the responder accepts
|
|
2561
|
+
* off a signaling frame, while the initiator dials only after its own `createSessionNode`. If
|
|
2562
|
+
* accept looked before the dial landed it saw nothing, and the responder was back to holding
|
|
2563
|
+
* no address — the state that made every reply after an interruption park forever.
|
|
2564
|
+
*
|
|
2565
|
+
* This fires exactly when the counterparty connects: on both sides, on the first connection,
|
|
2566
|
+
* on every reconnect, and on a revived node too. It also REFRESHES, which the accept-time read
|
|
2567
|
+
* never did — a counterparty that rebuilds its receiver would otherwise leave us dialling a
|
|
2568
|
+
* dead address for the life of the session.
|
|
2569
|
+
*/
|
|
2570
|
+
const observed = node
|
|
2571
|
+
.getConnections()
|
|
2572
|
+
.filter((c) => c.peerId === peerId && typeof c.remoteAddr === "string")
|
|
2573
|
+
.map((c) => c.remoteAddr);
|
|
2574
|
+
if (observed.length > 0) {
|
|
2575
|
+
this.#counterpartyAddrs.set(key, [...new Set(observed)]);
|
|
2576
|
+
}
|
|
2486
2577
|
const prior = this.#sessionLiveness.get(key);
|
|
2487
2578
|
this.#sessionLiveness.set(key, "alive");
|
|
2488
2579
|
if (prior !== "alive") {
|
|
@@ -2736,7 +2827,7 @@ export class SessionNodeManager {
|
|
|
2736
2827
|
"Close an existing session before starting a new one.",
|
|
2737
2828
|
};
|
|
2738
2829
|
}
|
|
2739
|
-
const { node, gater, autoNat } = inboundSr;
|
|
2830
|
+
const { node, gater, autoNat, seed } = inboundSr;
|
|
2740
2831
|
// AC-015: update gater BEFORE retrieving multiaddr / returning to caller
|
|
2741
2832
|
gater.setAllowedPeer(initiatorPeerId);
|
|
2742
2833
|
const peerId = node.getPeerId();
|
|
@@ -2747,6 +2838,11 @@ export class SessionNodeManager {
|
|
|
2747
2838
|
// pointed at this initiator.
|
|
2748
2839
|
if (!this.#insertSessionRow(sessionId, agentName, counterpartyPubkey, "active")) {
|
|
2749
2840
|
this.#standingReceivers.delete(agentName);
|
|
2841
|
+
// DOD-M12B-SESSION-SEED-1 (review F8): this abort happens BEFORE `#rememberSessionSeed`, so
|
|
2842
|
+
// the identity is not being handed to a session — it is being discarded, and is zeroed like
|
|
2843
|
+
// any other discard. (The two PROMOTION sites deliberately do not zero: there the same bytes
|
|
2844
|
+
// become the session's.)
|
|
2845
|
+
inboundSr.seed.fill(0);
|
|
2750
2846
|
try {
|
|
2751
2847
|
await node.stop();
|
|
2752
2848
|
}
|
|
@@ -2792,9 +2888,55 @@ export class SessionNodeManager {
|
|
|
2792
2888
|
counterpartySessionPeerId: initiatorPeerId,
|
|
2793
2889
|
autoNat,
|
|
2794
2890
|
});
|
|
2891
|
+
this.#rememberSessionSeed(agentName, sessionId, seed, initiatorPeerId, counterpartyPubkey);
|
|
2795
2892
|
// DAEMON-004: register the content stream handler for the inbound session.
|
|
2796
2893
|
await this.#registerContentHandler(agentName, sessionId, node, counterpartyPubkey);
|
|
2797
2894
|
// M7-SESSION-003 AC-004: act on the inbound session node's peer events too.
|
|
2895
|
+
/**
|
|
2896
|
+
* DOD-M12B-RESPONDER-ADDR-1 — LEARN THE INITIATOR'S ADDRESS, because we will need it and this is
|
|
2897
|
+
* the only moment we have it.
|
|
2898
|
+
*
|
|
2899
|
+
* MEASURED LIVE 2026-08-18. After an interruption the responder's re-dial reported
|
|
2900
|
+
* `session.transport.redial.unavailable` — *"this side holds no address for the counterparty, so
|
|
2901
|
+
* every send parks until they re-establish"* — and every reply it tried to send failed. The
|
|
2902
|
+
* initiator can always come back because it kept the addresses it dialled; the responder dialled
|
|
2903
|
+
* nothing, so it kept nothing.
|
|
2904
|
+
*
|
|
2905
|
+
* In plain terms that meant: whoever ANSWERED a conversation could not restart it. Their replies
|
|
2906
|
+
* went nowhere until the other side spoke first.
|
|
2907
|
+
*
|
|
2908
|
+
* The live connection has known the address all along — the responder is holding it right now,
|
|
2909
|
+
* because the initiator just dialled in on it. `#counterpartyAddrs` is the same store the
|
|
2910
|
+
* initiator fills from its signed relay assignment, and `#evictSessionCaches` hands both to the
|
|
2911
|
+
* revival record on the way down, so this needs no separate lifetime.
|
|
2912
|
+
*/
|
|
2913
|
+
const inboundAddrs = node
|
|
2914
|
+
.getConnections()
|
|
2915
|
+
.filter((c) => c.peerId === initiatorPeerId && typeof c.remoteAddr === "string")
|
|
2916
|
+
.map((c) => c.remoteAddr);
|
|
2917
|
+
if (inboundAddrs.length > 0) {
|
|
2918
|
+
this.#counterpartyAddrs.set(this.#k(agentName, sessionId), [...new Set(inboundAddrs)]);
|
|
2919
|
+
this.#logger.info("session.counterparty.addr.learned", {
|
|
2920
|
+
agentName,
|
|
2921
|
+
sessionId,
|
|
2922
|
+
addrs: inboundAddrs.length,
|
|
2923
|
+
source: "inbound_connection",
|
|
2924
|
+
impact: "this side can now re-dial after an interruption instead of parking every reply",
|
|
2925
|
+
});
|
|
2926
|
+
}
|
|
2927
|
+
else {
|
|
2928
|
+
// NOT A WARNING. Review MEDIUM-4: accept runs off a signaling frame and the initiator dials
|
|
2929
|
+
// separately, so "no connection yet" is the ordinary in-flight case — warning on it puts a
|
|
2930
|
+
// signal on the normal path, which is how the one occurrence that matters gets buried. The
|
|
2931
|
+
// race-free capture is in `#wireSessionLiveness`'s onPeerConnect, which fires when the dial
|
|
2932
|
+
// actually lands; this read is only a fast path for when it already has.
|
|
2933
|
+
this.#logger.debug("session.counterparty.addr.deferred", {
|
|
2934
|
+
agentName,
|
|
2935
|
+
sessionId,
|
|
2936
|
+
initiatorPeerId,
|
|
2937
|
+
impact: "no connection observed yet; the address is captured when the counterparty connects",
|
|
2938
|
+
});
|
|
2939
|
+
}
|
|
2798
2940
|
this.#wireSessionLiveness(agentName, sessionId, node, counterpartyPubkey, correlationId, initiatorPeerId);
|
|
2799
2941
|
// M7 DOD-SPINE-6 / MSG-001-3b: the receiver also connects to the relay witness so
|
|
2800
2942
|
// the relay can deliver the initiator's witnessed leaves (leaf_deliver) to it.
|
|
@@ -3028,6 +3170,16 @@ export class SessionNodeManager {
|
|
|
3028
3170
|
this.#heldRestored.delete(key);
|
|
3029
3171
|
this.#heldReleased.delete(key);
|
|
3030
3172
|
this.#diverged.delete(key);
|
|
3173
|
+
// DOD-M12B-SESSION-SEED-1: HAND THEM TO THE REVIVAL RECORD BEFORE DROPPING THEM. This eviction
|
|
3174
|
+
// runs on every teardown, including the interruption a revival is meant to undo — so clearing
|
|
3175
|
+
// the addresses here is what left a revived session unable to dial anyone. The revival record
|
|
3176
|
+
// has exactly the right lifetime for them: it dies when the session reaches a terminal status.
|
|
3177
|
+
const survivingAddrs = this.#counterpartyAddrs.get(key);
|
|
3178
|
+
if (survivingAddrs && survivingAddrs.length > 0) {
|
|
3179
|
+
const identity = this.#sessionSeeds.get(key);
|
|
3180
|
+
if (identity)
|
|
3181
|
+
identity.counterpartyAddrs = [...survivingAddrs];
|
|
3182
|
+
}
|
|
3031
3183
|
this.#counterpartyAddrs.delete(key);
|
|
3032
3184
|
this.#redialNotBefore.delete(key);
|
|
3033
3185
|
this.#highWaterSeq.delete(key);
|
|
@@ -3150,6 +3302,13 @@ export class SessionNodeManager {
|
|
|
3150
3302
|
// plaintext must not survive shutdown in memory).
|
|
3151
3303
|
this.#trees.clear();
|
|
3152
3304
|
this.#receivedContent.clear();
|
|
3305
|
+
// DOD-M12B-SESSION-SEED-1 (review F5): transport identities are key material and belong in the
|
|
3306
|
+
// same sentence as the plaintext above. Shutdown marks every active row `interrupted` by direct
|
|
3307
|
+
// SQL, so no `#updateSessionStatus` destroy fires for them — without this, every live session's
|
|
3308
|
+
// seed survives the shutdown in memory for as long as the process lingers.
|
|
3309
|
+
for (const identity of this.#sessionSeeds.values())
|
|
3310
|
+
identity.seed.fill(0);
|
|
3311
|
+
this.#sessionSeeds.clear();
|
|
3153
3312
|
// Stop ALL per-agent standing receivers (DOD-LOOP-1). In PARALLEL and BOUNDED: this was a
|
|
3154
3313
|
// sequential await per agent with no deadline, so five agents meant five chances for one stuck
|
|
3155
3314
|
// libp2p teardown to hold the exit — and it sits between the operator being told the daemon is
|
|
@@ -3168,6 +3327,10 @@ export class SessionNodeManager {
|
|
|
3168
3327
|
});
|
|
3169
3328
|
}
|
|
3170
3329
|
})), "standing_receivers", this.#standingReceivers.size);
|
|
3330
|
+
// Same reason: a receiver's seed is the identity it has already advertised in
|
|
3331
|
+
// `session_offer_accept` for any session it is mid-handshake on.
|
|
3332
|
+
for (const sr of this.#standingReceivers.values())
|
|
3333
|
+
sr.seed.fill(0);
|
|
3171
3334
|
this.#standingReceivers.clear();
|
|
3172
3335
|
this.#srReservationRetry.clear();
|
|
3173
3336
|
this.#srLastRejectionReason.clear();
|
|
@@ -3207,9 +3370,38 @@ export class SessionNodeManager {
|
|
|
3207
3370
|
.all(status);
|
|
3208
3371
|
}
|
|
3209
3372
|
/**
|
|
3210
|
-
* DOD-M12B-RESTART-SEAL-1 —
|
|
3373
|
+
* DOD-M12B-RESTART-SEAL-1 / DOD-M12B-PENDING-RESOLVE-1 — sessions that need a receipt and have
|
|
3374
|
+
* nobody asking for one. TWO populations, one queue, each with its own safety argument.
|
|
3375
|
+
*
|
|
3376
|
+
* **(2) `seal_interrupted_pending` — a seal commitment nobody notarized.** Measured 2026-08-18 on
|
|
3377
|
+
* the live store: 28 sessions, aged 0.3 to 12.8 days, one of them 14 messages long, 26 holding
|
|
3378
|
+
* relay-witnessed seal leaves, and **not one with a sealed root**.
|
|
3211
3379
|
*
|
|
3212
|
-
*
|
|
3380
|
+
* **HALF OF THEM ARE NOT BILATERAL, and the first version of this comment claimed they were.**
|
|
3381
|
+
* Measured split: 14 initiator rows, each carrying the counterparty's signed leaf — and 14
|
|
3382
|
+
* responder rows with `counterparty_leaf = NULL`. A responder row is written by
|
|
3383
|
+
* `inbound-seal-request.ts` from an UNSIGNED `seal_interrupted_request` frame, before its ack is
|
|
3384
|
+
* even sent, so an ordinary send failure produces a one-sided pending row.
|
|
3385
|
+
*
|
|
3386
|
+
* So the licence is NOT "both parties signed". It is **somebody chose to end this**, on two
|
|
3387
|
+
* branches: an initiator row carries the counterparty's signature, and a responder row exists
|
|
3388
|
+
* because the counterparty sent a request to seal. And what makes the result VERIFIABLE is
|
|
3389
|
+
* neither — it is that the directory rebuilds the tree from relay-witnessed leaves and checks
|
|
3390
|
+
* their signatures, never consulting the commitment at all. The commitment is what makes it
|
|
3391
|
+
* legitimate to ASK. (`close-session-handler.ts` states this in full; the first draft of this
|
|
3392
|
+
* header contradicted it 200 lines away.) `PENDING-EXIT-1` built their exit and it works — but only when an operator runs
|
|
3393
|
+
* `cello_close_session` on that session by hand, having somehow deduced they should. Nothing
|
|
3394
|
+
* enumerated them, because both sweeps filtered `status = 'interrupted'`. An exit nobody is told
|
|
3395
|
+
* about is not an exit.
|
|
3396
|
+
*
|
|
3397
|
+
* `interrupted_by` is deliberately NOT consulted for that population. It answers "did WE cause
|
|
3398
|
+
* this, and may we therefore describe it" — and that is the wrong question once a seal was
|
|
3399
|
+
* requested or signed. **SI-001 is not weakened:** it forbids notarizing *"a conversation nobody
|
|
3400
|
+
* chose to end"*, and every row here was chosen to be ended by one side or the other. The only
|
|
3401
|
+
* thing missing is the request to notarize it.
|
|
3402
|
+
*
|
|
3403
|
+
* **(1) `interrupted` — sessions our own stop orphaned, and only those.** Here `interrupted_by` is
|
|
3404
|
+
* the whole safety argument. `'local'` means the boot sweep, the shutdown
|
|
3213
3405
|
* sweep, or the operator's own kill switch ended this session — nobody else did, and it cannot be
|
|
3214
3406
|
* resumed because the transport keypairs died with the process. Those are the ones the resolver
|
|
3215
3407
|
* may seal on its own.
|
|
@@ -3238,9 +3430,23 @@ export class SessionNodeManager {
|
|
|
3238
3430
|
// `restart_seal_gave_up_at IS NULL` — a session we have already exhausted. Without it a
|
|
3239
3431
|
// machine restarting ~6 times a day re-runs five ceremonies against a hopeless session on
|
|
3240
3432
|
// every boot, forever.
|
|
3241
|
-
`SELECT s.session_id AS session_id, s.message_count AS message_count, a.agent_name AS agent_name
|
|
3433
|
+
`SELECT s.session_id AS session_id, s.message_count AS message_count, a.agent_name AS agent_name,
|
|
3434
|
+
s.status AS status
|
|
3242
3435
|
FROM sessions s JOIN agents a ON a.agent_id = s.agent_id
|
|
3243
|
-
WHERE
|
|
3436
|
+
WHERE a.state != 'retired'
|
|
3437
|
+
AND (
|
|
3438
|
+
-- (1) OURS, and we can say so. SI-001 holds: an interrupted session with an
|
|
3439
|
+
-- unknown cause has no signatures behind it and must not be notarized.
|
|
3440
|
+
(s.status = 'interrupted' AND s.interrupted_by = 'local')
|
|
3441
|
+
-- (2) A seal commitment with nobody asking for it. review F5: the EXISTS is
|
|
3442
|
+
-- STRUCTURAL, not decoration. The header's licence is "a commitment was made", and
|
|
3443
|
+
-- a status check alone asserts that in prose while the query checks something
|
|
3444
|
+
-- else. Today status implies an artifact row by construction, which is exactly the
|
|
3445
|
+
-- kind of invariant that holds until someone adds a fourth writer.
|
|
3446
|
+
OR (s.status = 'seal_interrupted_pending'
|
|
3447
|
+
AND EXISTS (SELECT 1 FROM seal_interrupted_artifacts sa
|
|
3448
|
+
WHERE sa.agent_id = s.agent_id AND sa.session_id = s.session_id))
|
|
3449
|
+
)
|
|
3244
3450
|
AND s.message_count > 0
|
|
3245
3451
|
AND s.restart_seal_gave_up_at IS NULL
|
|
3246
3452
|
ORDER BY s.updated_at ASC`)
|
|
@@ -3249,8 +3455,216 @@ export class SessionNodeManager {
|
|
|
3249
3455
|
agentName: r.agent_name,
|
|
3250
3456
|
sessionId: r.session_id,
|
|
3251
3457
|
messageCount: r.message_count ?? 0,
|
|
3458
|
+
// Carried so a give-up can say something TRUE about this session: the two populations need
|
|
3459
|
+
// different words, and force-abandon is right for one and destructive for the other.
|
|
3460
|
+
status: r.status === "seal_interrupted_pending" ? "seal_interrupted_pending" : "interrupted",
|
|
3252
3461
|
}));
|
|
3253
3462
|
}
|
|
3463
|
+
/**
|
|
3464
|
+
* DOD-M12B-REVIVAL-BOUND-1 — interrupted sessions that can no longer be revived, and must close.
|
|
3465
|
+
*
|
|
3466
|
+
* Andre, 2026-08-18: *"after that, those peer IDs and that peer connection needs to be shut down.
|
|
3467
|
+
* It is an open connection that a malicious agent can farm for."* The tenet is **leave nothing
|
|
3468
|
+
* open that is no longer needed**, and the threat model is a daemon that has been reprogrammed —
|
|
3469
|
+
* so the guarantee has to hold on the side that is not the attacker.
|
|
3470
|
+
*
|
|
3471
|
+
* WHAT IS OPEN. `ingestReceivedContent` refuses `sealed`, `seal_interrupted_pending` and
|
|
3472
|
+
* `abandoned`, but deliberately ACCEPTS `interrupted` — that acceptance is the only reason
|
|
3473
|
+
* recovery can work. Nothing else ever leaves `interrupted`, so it means accepts FOREVER.
|
|
3474
|
+
*
|
|
3475
|
+
* **THIS IS THE BACKSTOP, NOT THE COMPLEMENT.** The seal path gets first refusal on every
|
|
3476
|
+
* local-cause session, because a session whose ending we can describe truthfully earns a
|
|
3477
|
+
* notarized receipt. But "the seal path owns it" is not the same as "the seal path will finish
|
|
3478
|
+
* it", and two populations fall through the gap between those:
|
|
3479
|
+
*
|
|
3480
|
+
* - **The resolver gave up.** `markRestartSealGaveUp` writes only `restart_seal_gave_up_at`;
|
|
3481
|
+
* the status stays `interrupted`, and `listRestartOrphanedSessions` then excludes the row by
|
|
3482
|
+
* `restart_seal_gave_up_at IS NULL` so it is never retried. `TERMINAL_SEAL_REFUSALS` has ten
|
|
3483
|
+
* entries and the measured figure is that 59% of seals that start never finish, so this is
|
|
3484
|
+
* the common case, not a corner.
|
|
3485
|
+
* - **Zero-message local sessions.** The resolver requires `message_count > 0` — a dead
|
|
3486
|
+
* handshake is not worth a ceremony. It is still an open write surface.
|
|
3487
|
+
*
|
|
3488
|
+
* Excluding those left them permanently interrupted and permanently writable, which is the exact
|
|
3489
|
+
* condition this line exists to end. And the population is about to become the majority: no row
|
|
3490
|
+
* has ever carried `interrupted_by = 'local'` yet, and from the next shutdown onward every
|
|
3491
|
+
* shutdown-orphaned session will. So the sweep takes a local-cause session once the seal path
|
|
3492
|
+
* has either declined it or exhausted it — never before.
|
|
3493
|
+
*
|
|
3494
|
+
* **THE CLOCK MUST BE ONE THE COUNTERPARTY CANNOT MOVE.** The obvious fallback for a row with no
|
|
3495
|
+
* `interrupted_at` is `updated_at` — and it is exactly wrong. `ingestReceivedContent` accepts
|
|
3496
|
+
* content into an `interrupted` session (that acceptance is this whole line's premise), and a
|
|
3497
|
+
* successful ingest runs `UPDATE sessions SET message_count = ?, updated_at = <now>`. So
|
|
3498
|
+
* `updated_at` is a clock the reprogrammed peer holds: one message every 24 hours and the session
|
|
3499
|
+
* never expires, forever. The fallback would have handed the attacker the off switch for the
|
|
3500
|
+
* control built to stop them.
|
|
3501
|
+
*
|
|
3502
|
+
* Instead the missing timestamps are STAMPED ONCE, by `#stampMissingInterruptedAt` immediately
|
|
3503
|
+
* before this query runs, and this query reads `interrupted_at` and nothing else. The stamp is
|
|
3504
|
+
* written under `WHERE interrupted_at IS NULL`, so it is monotone — set once, never moved, by us
|
|
3505
|
+
* and not by a peer. A legacy row therefore gets its full window starting from the first sweep
|
|
3506
|
+
* that sees it, which is later than the true interruption but is the only bound that is sound.
|
|
3507
|
+
*
|
|
3508
|
+
* Same retired-agent INNER JOIN as the sibling query: a retired agent's rows are kept for
|
|
3509
|
+
* accountability, are not resumable, and are not writable either.
|
|
3510
|
+
*
|
|
3511
|
+
* **THE TIME ARITHMETIC IS LOAD-BEARING, AND BOTH OBVIOUS FORMS OF IT ARE WRONG.** These two
|
|
3512
|
+
* columns do not hold the same kind of value:
|
|
3513
|
+
*
|
|
3514
|
+
* `interrupted_at` TEXT, an ISO-8601 string — `new Date(now).toISOString()`.
|
|
3515
|
+
* `updated_at` INTEGER, epoch milliseconds.
|
|
3516
|
+
*
|
|
3517
|
+
* There are FOUR writers of `status = 'interrupted'`, not three. The fourth is
|
|
3518
|
+
* `destroySessionNode` → `#updateSessionStatus(…, "interrupted", "local")`, which historically
|
|
3519
|
+
* wrote `interrupted_by` and **no timestamp at all** — and it is the path that produced the two
|
|
3520
|
+
* rows in Entry 41. It now stamps `interrupted_at` like the others, so NULL is a legacy state
|
|
3521
|
+
* rather than one production keeps creating.
|
|
3522
|
+
*
|
|
3523
|
+
* So a bare `interrupted_at <= ?` against a numeric bound is **always false** — the column has
|
|
3524
|
+
* TEXT affinity and the bound parameter has none, so SQLite applies TEXT affinity to the
|
|
3525
|
+
* parameter and compares them as STRINGS (`'2026-08-18T05:32:04.183Z' <= 1755000000000` → 0).
|
|
3526
|
+
* The query silently returns nothing forever and reads as "nothing has expired yet". And
|
|
3527
|
+
* `CAST(interrupted_at AS
|
|
3528
|
+
* INTEGER)` is worse than useless: SQLite casts by taking the leading digits, so
|
|
3529
|
+
* `'2026-08-18T05:32:04Z'` becomes **2026**, which is older than any epoch bound. That form
|
|
3530
|
+
* abandons every interrupted session on the next boot, immediately, whatever its age. It was
|
|
3531
|
+
* written, and `session-001`/`cello-list-sessions` failed on it in the gate.
|
|
3532
|
+
*
|
|
3533
|
+
* `strftime('%s', …) * 1000` parses the ISO string properly and returns NULL for anything it
|
|
3534
|
+
* cannot parse — so a malformed or differently-formatted value falls through the COALESCE to
|
|
3535
|
+
* `updated_at` rather than being read as the year 2026.
|
|
3536
|
+
*/
|
|
3537
|
+
listExpiredUnrevivableSessions(nowMs, windowMs) {
|
|
3538
|
+
if (!this.#db) {
|
|
3539
|
+
// ABSENT IS NOT FINE. An empty array here is indistinguishable from "the store is clean",
|
|
3540
|
+
// which is the state this line exists to end — so the one boot where the sweep could not
|
|
3541
|
+
// read the store must not look like the boots where it read it and found nothing.
|
|
3542
|
+
this.#logger.error("session.revival_bound.enumerate.failed", {
|
|
3543
|
+
error: "db not initialized",
|
|
3544
|
+
impact: "no interrupted session was checked against the revival window this boot; any that "
|
|
3545
|
+
+ "have expired are still accepting content",
|
|
3546
|
+
});
|
|
3547
|
+
return [];
|
|
3548
|
+
}
|
|
3549
|
+
const rows = this.#db
|
|
3550
|
+
.prepare(`SELECT s.session_id AS session_id, s.interrupted_by AS cause, a.agent_name AS agent_name
|
|
3551
|
+
FROM sessions s JOIN agents a ON a.agent_id = s.agent_id
|
|
3552
|
+
WHERE s.status = 'interrupted' AND a.state != 'retired'
|
|
3553
|
+
AND (COALESCE(s.interrupted_by, '') != 'local'
|
|
3554
|
+
OR s.restart_seal_gave_up_at IS NOT NULL
|
|
3555
|
+
OR s.message_count = 0)
|
|
3556
|
+
AND CAST(strftime('%s', s.interrupted_at) AS INTEGER) * 1000 <= ?
|
|
3557
|
+
ORDER BY CAST(strftime('%s', s.interrupted_at) AS INTEGER) * 1000 ASC`)
|
|
3558
|
+
.all(nowMs - windowMs);
|
|
3559
|
+
return rows.map((r) => ({ agentName: r.agent_name, sessionId: r.session_id, cause: r.cause }));
|
|
3560
|
+
}
|
|
3561
|
+
/**
|
|
3562
|
+
* DOD-M12B-REVIVAL-BOUND-1 — give every timestamp-less interrupted session a clock, once.
|
|
3563
|
+
*
|
|
3564
|
+
* A row with `interrupted_at IS NULL` has no bound that can be evaluated, and skipping such rows
|
|
3565
|
+
* would exempt the oldest sessions in the store from the control permanently — the same "open
|
|
3566
|
+
* forever" failure wearing a different NULL. The two rows measured in Entry 41 are exactly this
|
|
3567
|
+
* shape, written by a `destroySessionNode` path that set the cause and no timestamp.
|
|
3568
|
+
*
|
|
3569
|
+
* **`WHERE interrupted_at IS NULL` is the security property, not an optimisation.** It makes the
|
|
3570
|
+
* stamp write-once: this can run on every sweep forever and a row's clock still cannot be moved
|
|
3571
|
+
* after the first one. That is what disqualifies `updated_at`, which a peer moves with every
|
|
3572
|
+
* message it sends into the still-accepting session.
|
|
3573
|
+
*
|
|
3574
|
+
* The cost is honest and bounded: a legacy row's window starts at the first sweep that sees it
|
|
3575
|
+
* rather than at its true interruption, so it survives up to one window longer than it should.
|
|
3576
|
+
* A late close is recoverable; a clock the counterparty winds is not.
|
|
3577
|
+
*
|
|
3578
|
+
* @returns how many rows were stamped.
|
|
3579
|
+
*/
|
|
3580
|
+
#stampMissingInterruptedAt(nowMs) {
|
|
3581
|
+
if (!this.#db)
|
|
3582
|
+
return 0;
|
|
3583
|
+
try {
|
|
3584
|
+
const res = this.#db
|
|
3585
|
+
.prepare("UPDATE sessions SET interrupted_at = ? WHERE status = 'interrupted' AND interrupted_at IS NULL")
|
|
3586
|
+
.run(new Date(nowMs).toISOString());
|
|
3587
|
+
const stamped = Number(res?.changes ?? 0);
|
|
3588
|
+
if (stamped > 0) {
|
|
3589
|
+
this.#logger.info("session.revival_bound.clock.stamped", {
|
|
3590
|
+
stamped,
|
|
3591
|
+
impact: "these sessions had no interruption timestamp; their revival window starts now",
|
|
3592
|
+
});
|
|
3593
|
+
}
|
|
3594
|
+
return stamped;
|
|
3595
|
+
}
|
|
3596
|
+
catch (err) {
|
|
3597
|
+
this.#logger.error("session.revival_bound.clock.stamp.failed", {
|
|
3598
|
+
error: err instanceof Error ? err.message : String(err),
|
|
3599
|
+
impact: "sessions with no interruption timestamp cannot be evaluated and stay open",
|
|
3600
|
+
});
|
|
3601
|
+
return 0;
|
|
3602
|
+
}
|
|
3603
|
+
}
|
|
3604
|
+
/**
|
|
3605
|
+
* DOD-M12B-REVIVAL-BOUND-1 — close every session the revival window has expired.
|
|
3606
|
+
*
|
|
3607
|
+
* `abandonSession` is the right instrument and already exists: it flips the status FIRST and
|
|
3608
|
+
* synchronously, annexes held content so the operator does not lose mail that has nowhere to go,
|
|
3609
|
+
* and retires the node. It notarizes nothing, which is the point — we are closing a door, not
|
|
3610
|
+
* asserting how it came to be open.
|
|
3611
|
+
*
|
|
3612
|
+
* One session's failure must not strand the rest, so each is caught and logged; the sweep runs at
|
|
3613
|
+
* boot beside the restart-seal resolver and a throw there would take the daemon with it.
|
|
3614
|
+
*
|
|
3615
|
+
* @returns how many sessions actually flipped — not how many were attempted.
|
|
3616
|
+
*/
|
|
3617
|
+
async closeExpiredUnrevivableSessions(nowMs, windowMs) {
|
|
3618
|
+
// Stamp FIRST. A row with no clock cannot be evaluated by the query below, and this is the only
|
|
3619
|
+
// thing that gives it one. Running it before every sweep is safe because the write is scoped to
|
|
3620
|
+
// rows that have no timestamp yet.
|
|
3621
|
+
this.#stampMissingInterruptedAt(nowMs);
|
|
3622
|
+
const expired = this.listExpiredUnrevivableSessions(nowMs, windowMs);
|
|
3623
|
+
let closed = 0;
|
|
3624
|
+
for (const s of expired) {
|
|
3625
|
+
try {
|
|
3626
|
+
if (await this.abandonSession(s.agentName, s.sessionId)) {
|
|
3627
|
+
closed += 1;
|
|
3628
|
+
this.#logger.info("session.revival_bound.closed", {
|
|
3629
|
+
agentName: s.agentName,
|
|
3630
|
+
sessionId: s.sessionId,
|
|
3631
|
+
// The cause we could NOT establish is the reason this ends without a receipt — log it
|
|
3632
|
+
// so an operator asking "why no certificate?" gets the answer here.
|
|
3633
|
+
interruptedBy: s.cause ?? "unknown",
|
|
3634
|
+
windowMs,
|
|
3635
|
+
// NAME THE FORFEIT. Until this sweep ran, `cello_close_session` (without `force`) still
|
|
3636
|
+
// accepted this session — it takes `status IN ('active','interrupted')` — so the
|
|
3637
|
+
// operator could have come back days later and obtained a real seal. `abandoned` is in
|
|
3638
|
+
// TERMINAL_SEAL_REFUSALS, so after this they cannot. That is a deliberate trade (SI-001
|
|
3639
|
+
// forbids auto-sealing a session nobody chose to end) and it costs something real, so
|
|
3640
|
+
// it is stated rather than left implicit in a WHERE clause.
|
|
3641
|
+
forfeited: "a seal was still obtainable by hand until now; it is not after this",
|
|
3642
|
+
});
|
|
3643
|
+
}
|
|
3644
|
+
}
|
|
3645
|
+
catch (err) {
|
|
3646
|
+
this.#logger.warn("session.revival_bound.close.failed", {
|
|
3647
|
+
agentName: s.agentName,
|
|
3648
|
+
sessionId: s.sessionId,
|
|
3649
|
+
error: err instanceof Error ? err.message : String(err),
|
|
3650
|
+
});
|
|
3651
|
+
}
|
|
3652
|
+
}
|
|
3653
|
+
// UNCONDITIONAL. A sweep that found nothing and a sweep that never really ran must not produce
|
|
3654
|
+
// the same silence — this line is the only proof the control executed at all.
|
|
3655
|
+
this.#logger.info("session.revival_bound.sweep", { expired: expired.length, closed, windowMs });
|
|
3656
|
+
if (closed !== expired.length) {
|
|
3657
|
+
// Not arithmetic for the reader to do: a session the security control failed to close is one
|
|
3658
|
+
// that is still interrupted and still accepting content.
|
|
3659
|
+
this.#logger.warn("session.revival_bound.sweep.incomplete", {
|
|
3660
|
+
expired: expired.length,
|
|
3661
|
+
closed,
|
|
3662
|
+
failed: expired.length - closed,
|
|
3663
|
+
impact: "these sessions are still interrupted and still accept content from any peer that dials them",
|
|
3664
|
+
});
|
|
3665
|
+
}
|
|
3666
|
+
return closed;
|
|
3667
|
+
}
|
|
3254
3668
|
/**
|
|
3255
3669
|
* DOD-M12B-RESTART-SEAL-1 — record that automatic sealing has exhausted this session.
|
|
3256
3670
|
*
|
|
@@ -3562,7 +3976,18 @@ export class SessionNodeManager {
|
|
|
3562
3976
|
const result = this.#db
|
|
3563
3977
|
.prepare("UPDATE sessions SET status = 'seal_interrupted_pending', updated_at = ? WHERE agent_id = ? AND session_id = ? AND status IN ('active', 'interrupted')")
|
|
3564
3978
|
.run(now, this.#requireAgentId(opts.agentName), opts.sessionId);
|
|
3565
|
-
|
|
3979
|
+
const landed = Number(result.changes) > 0;
|
|
3980
|
+
if (landed) {
|
|
3981
|
+
// DOD-M12B-SESSION-SEED-1 (review F3): `seal_interrupted_pending` is NOT a state revival
|
|
3982
|
+
// exists for, and the first build's comment wrongly grouped it with `interrupted`.
|
|
3983
|
+
// `ingestReceivedContent` refuses it outright, and BOTH sweeps that could otherwise close a
|
|
3984
|
+
// session — `listRestartOrphanedSessions` and `listExpiredUnrevivableSessions` — filter
|
|
3985
|
+
// `status = 'interrupted'`, so a pending-seal session is unrevivable AND unswept. Keeping its
|
|
3986
|
+
// identity meant holding it until the process exited. Entry 42's own measurement is that 59%
|
|
3987
|
+
// of seals that start never finish, so that is the common path, not a corner.
|
|
3988
|
+
this.#destroySessionSeed(opts.agentName, opts.sessionId);
|
|
3989
|
+
}
|
|
3990
|
+
return landed;
|
|
3566
3991
|
}
|
|
3567
3992
|
/**
|
|
3568
3993
|
* M7-SESSION-001 (H-1): read back the persisted bilateral commitment artifacts
|
|
@@ -4112,9 +4537,12 @@ export class SessionNodeManager {
|
|
|
4112
4537
|
// like a protocol mystery for a night.
|
|
4113
4538
|
//
|
|
4114
4539
|
// `counterpartySessionPeerId` is the load-bearing field. It is recorded ONCE at session
|
|
4115
|
-
// establishment and never refreshed
|
|
4116
|
-
// on every signaling reconnect —
|
|
4117
|
-
//
|
|
4540
|
+
// establishment and never refreshed. (CORRECTED 2026-08-18: this used to say a standing
|
|
4541
|
+
// receiver is rebuilt "on every signaling reconnect" — it is not. `ensureStandingReceiverForAgent`
|
|
4542
|
+
// no-ops on a healthy receiver; the only rebuild triggers are a LOST RELAY RESERVATION and the
|
|
4543
|
+
// one-shot upgrade when relay endpoints first arrive.) If the two ever cross, every send goes
|
|
4544
|
+
// one-way forever and nothing says so. With this line that becomes a single grep instead of a
|
|
4545
|
+
// night.
|
|
4118
4546
|
this.#logger.warn("session.content.direct.send.failed", {
|
|
4119
4547
|
agentName,
|
|
4120
4548
|
sessionId,
|
|
@@ -6882,6 +7310,22 @@ export class SessionNodeManager {
|
|
|
6882
7310
|
const sr = this.#standingReceivers.get(agentName);
|
|
6883
7311
|
if (sr) {
|
|
6884
7312
|
this.#standingReceivers.delete(agentName);
|
|
7313
|
+
/**
|
|
7314
|
+
* DOD-M12B-SESSION-SEED-1 (review F8): drop it zeroed, like every other seed.
|
|
7315
|
+
*
|
|
7316
|
+
* (review F7, DECIDED AGAINST — deliberately NOT reusing this seed for the replacement.)
|
|
7317
|
+
* Reuse is attractive: this receiver's peer id may already be inside a `session_offer_accept`
|
|
7318
|
+
* the counterparty is acting on, and a rebuild in that window is the documented "we record
|
|
7319
|
+
* an identity that no longer exists… every send in this direction parks forever" defect. But
|
|
7320
|
+
* a preserved identity would have to be handed to the candidate loop in
|
|
7321
|
+
* `#startReceiverNode`, whose rejected candidates are stopped WITHOUT awaiting `start()` —
|
|
7322
|
+
* so two nodes could be briefly live on one advertised peer id, which is review F1, a HIGH,
|
|
7323
|
+
* and the reason each candidate now mints its own. Fixing F7 properly means bounding and
|
|
7324
|
+
* awaiting the loser's teardown first, and an unawaited stop is precisely what the current
|
|
7325
|
+
* code chose to avoid a stuck libp2p teardown blocking receiver creation. Filed as
|
|
7326
|
+
* follow-on work rather than trading a MEDIUM fix for a HIGH regression.
|
|
7327
|
+
*/
|
|
7328
|
+
sr.seed.fill(0);
|
|
6885
7329
|
try {
|
|
6886
7330
|
sr.autoNat.stop();
|
|
6887
7331
|
await sr.node.stop();
|
|
@@ -7166,11 +7610,25 @@ export class SessionNodeManager {
|
|
|
7166
7610
|
*/
|
|
7167
7611
|
async #startReceiverNode(agentName, sessionId, gater, candidateCircuitAddrs, correlationId) {
|
|
7168
7612
|
for (const circuitAddr of candidateCircuitAddrs) {
|
|
7613
|
+
// DOD-M12B-SESSION-SEED-1: A SEED PER CANDIDATE, NOT ONE FOR THE LOOP.
|
|
7614
|
+
//
|
|
7615
|
+
// A rejected candidate is stopped with an unawaited `void …then(() => candidate.stop())`
|
|
7616
|
+
// while its `start()` may still be in flight, so two candidate nodes can briefly be live at
|
|
7617
|
+
// once. Sharing one seed would give both the SAME peer id — and the loser would then be a
|
|
7618
|
+
// second live node under the identity we advertise in `session_offer_accept`, sharing this
|
|
7619
|
+
// gater (so it admits dials) with no content handler registered. Inbound arriving there goes
|
|
7620
|
+
// nowhere, and it is an open endpoint under our advertised id: the "connection a malicious
|
|
7621
|
+
// agent can farm for" the tenet names. Before seeds existed the loser had its own random key
|
|
7622
|
+
// and was harmless; introducing a shared seed is what would have made it dangerous.
|
|
7623
|
+
//
|
|
7624
|
+
// Nothing reads the seed before the winner is installed, so per-candidate costs nothing.
|
|
7625
|
+
const candidateSeed = randomBytes(32);
|
|
7169
7626
|
const candidate = await this.#factory.createNode({
|
|
7170
7627
|
sessionId,
|
|
7171
7628
|
connectionGater: gater,
|
|
7172
7629
|
nodeType: "standing_receiver",
|
|
7173
7630
|
circuitRelayListenAddrs: [circuitAddr],
|
|
7631
|
+
transportPrivateKey: candidateSeed,
|
|
7174
7632
|
});
|
|
7175
7633
|
let timer;
|
|
7176
7634
|
const timedOut = Symbol("reservation_timeout");
|
|
@@ -7196,7 +7654,7 @@ export class SessionNodeManager {
|
|
|
7196
7654
|
// completes the handshake and simply grants nothing, leaving a node that looks
|
|
7197
7655
|
// started and is reachable by nobody.
|
|
7198
7656
|
if (outcome === "started" && candidate.listenAddresses().some((a) => a.includes("/p2p-circuit"))) {
|
|
7199
|
-
return candidate;
|
|
7657
|
+
return { node: candidate, seed: candidateSeed };
|
|
7200
7658
|
}
|
|
7201
7659
|
const rejectionReason = outcome === "started"
|
|
7202
7660
|
? "relay_granted_no_reservation"
|
|
@@ -7217,13 +7675,15 @@ export class SessionNodeManager {
|
|
|
7217
7675
|
.then(() => candidate.stop())
|
|
7218
7676
|
.catch(() => { });
|
|
7219
7677
|
}
|
|
7678
|
+
const plainSeed = randomBytes(32);
|
|
7220
7679
|
const plain = await this.#factory.createNode({
|
|
7221
7680
|
sessionId,
|
|
7222
7681
|
connectionGater: gater,
|
|
7223
7682
|
nodeType: "standing_receiver",
|
|
7683
|
+
transportPrivateKey: plainSeed,
|
|
7224
7684
|
});
|
|
7225
7685
|
await plain.start();
|
|
7226
|
-
return plain;
|
|
7686
|
+
return { node: plain, seed: plainSeed };
|
|
7227
7687
|
}
|
|
7228
7688
|
/** One standing-receiver create attempt (extracted for the M8B F14 retry loop). */
|
|
7229
7689
|
async #tryCreateStandingReceiver(agentName, correlationId) {
|
|
@@ -7242,8 +7702,21 @@ export class SessionNodeManager {
|
|
|
7242
7702
|
gater.setAllowedOutboundPeer(relayPeerId);
|
|
7243
7703
|
}
|
|
7244
7704
|
let node;
|
|
7705
|
+
/**
|
|
7706
|
+
* DOD-M12B-SESSION-SEED-1 — the transport identity of the receiver that actually survived.
|
|
7707
|
+
*
|
|
7708
|
+
* Minted per CANDIDATE inside `#startReceiverNode` and returned with the winner, not minted
|
|
7709
|
+
* here: a rejected candidate is stopped without awaiting its `start()`, so two candidates can
|
|
7710
|
+
* be briefly live, and one shared seed would put both on the same advertised peer id.
|
|
7711
|
+
*
|
|
7712
|
+
* FRESH EVERY TIME, which is the privacy property rather than an implementation detail. A
|
|
7713
|
+
* receiver serves at most one session (it is promoted into the session at handoff and replaced),
|
|
7714
|
+
* so no identifier is ever shared between two sessions and the 2026-04-11 rationale —
|
|
7715
|
+
* unlinkability of an agent's sessions to a passive observer — survives intact.
|
|
7716
|
+
*/
|
|
7717
|
+
let seed;
|
|
7245
7718
|
try {
|
|
7246
|
-
node = await this.#startReceiverNode(agentName, sessionId, gater, reservations.addrs, correlationId);
|
|
7719
|
+
({ node, seed } = await this.#startReceiverNode(agentName, sessionId, gater, reservations.addrs, correlationId));
|
|
7247
7720
|
}
|
|
7248
7721
|
catch (err) {
|
|
7249
7722
|
// extractErrorMessage, NOT String(err): the transport throws structured
|
|
@@ -7314,6 +7787,7 @@ export class SessionNodeManager {
|
|
|
7314
7787
|
node,
|
|
7315
7788
|
gater,
|
|
7316
7789
|
autoNat,
|
|
7790
|
+
seed,
|
|
7317
7791
|
hasReservation: circuitAddrs > 0,
|
|
7318
7792
|
...(reservedRelayPeerId !== undefined ? { relayPeerId: reservedRelayPeerId } : {}),
|
|
7319
7793
|
});
|
|
@@ -7355,6 +7829,627 @@ export class SessionNodeManager {
|
|
|
7355
7829
|
* M8B F14: also called from the inbound accept path (ensure on demand). Marks the agent as
|
|
7356
7830
|
* WANTING a receiver, which arms the teardown re-arm in destroySessionNode/retireSessionNode.
|
|
7357
7831
|
*/
|
|
7832
|
+
/**
|
|
7833
|
+
* DOD-M12B-SESSION-SEED-1 test seam: the seed the agent's current standing receiver holds.
|
|
7834
|
+
*
|
|
7835
|
+
* The property under test — "the receiver built behind a promoted one never reuses its seed" — is
|
|
7836
|
+
* about an identity that by design never leaves the process, so there is no observable surface for
|
|
7837
|
+
* it short of a live two-node dial. Reading it here is the narrowest way to pin it.
|
|
7838
|
+
*/
|
|
7839
|
+
/** DOD-M12B-SESSION-SEED-1: record the identity this session must be able to return at. */
|
|
7840
|
+
#rememberSessionSeed(agentName, sessionId, seed, counterpartyPeerId, counterpartyPubkey) {
|
|
7841
|
+
const key = this.#k(agentName, sessionId);
|
|
7842
|
+
// Defensive: unreachable today because `insertSessionRow` PK-conflicts on a repeat, but an
|
|
7843
|
+
// overwrite that dropped a live seed un-zeroed would leave the one copy we are responsible for
|
|
7844
|
+
// in the heap with nothing tracking it.
|
|
7845
|
+
this.#sessionSeeds.get(key)?.seed.fill(0);
|
|
7846
|
+
// `counterpartyAddrs` starts empty: at creation the signed assignment has not necessarily
|
|
7847
|
+
// arrived yet. It is filled by `#evictSessionCaches` on the way down, which is the last moment
|
|
7848
|
+
// the live addresses exist.
|
|
7849
|
+
this.#sessionSeeds.set(key, { seed, counterpartyPeerId, counterpartyPubkey, counterpartyAddrs: [] });
|
|
7850
|
+
}
|
|
7851
|
+
/**
|
|
7852
|
+
* DOD-M12B-SESSION-SEED-1 — destroy a session's transport identity.
|
|
7853
|
+
*
|
|
7854
|
+
* Called from `#updateSessionStatus` on a terminal status, in the SAME step that writes it, so
|
|
7855
|
+
* there is no window in which a session is closed on paper and still revivable in memory.
|
|
7856
|
+
*
|
|
7857
|
+
* **WHAT THE ZERO-FILL DOES AND DOES NOT DO** — checked against the derivation, not assumed.
|
|
7858
|
+
* `createNode` hands the buffer to `generateKeyPairFromSeed`, and `@libp2p/crypto` COPIES it
|
|
7859
|
+
* (`uint8arrayConcat([seed, publicKeyRaw])`, then `Uint8Array.from`). Two consequences:
|
|
7860
|
+
* - zeroing after the node has started is SAFE — the running node holds its own copy;
|
|
7861
|
+
* - it does NOT erase the key from the heap. An identical usable copy is the first 32 bytes of
|
|
7862
|
+
* `privateKey.raw` on the node object until that node is dropped.
|
|
7863
|
+
* So this removes OUR long-lived copy — the one that would otherwise sit in a map for the life of
|
|
7864
|
+
* the process, decoupled from any node — and that is worth doing. It is not a heap scrub, and
|
|
7865
|
+
* the DoD already says the bound rather than secrecy is the control.
|
|
7866
|
+
*/
|
|
7867
|
+
#destroySessionSeed(agentName, sessionId) {
|
|
7868
|
+
const key = this.#k(agentName, sessionId);
|
|
7869
|
+
const identity = this.#sessionSeeds.get(key);
|
|
7870
|
+
if (identity === undefined)
|
|
7871
|
+
return;
|
|
7872
|
+
identity.seed.fill(0);
|
|
7873
|
+
this.#sessionSeeds.delete(key);
|
|
7874
|
+
this.#logger.debug("session.seed.destroyed", { agentName, sessionId });
|
|
7875
|
+
}
|
|
7876
|
+
/**
|
|
7877
|
+
* DOD-M12B-SESSION-SEED-1 — build a revived session node that is REACHABLE, without ever hanging.
|
|
7878
|
+
*
|
|
7879
|
+
* MEASURED 2026-08-18, live, three ways:
|
|
7880
|
+
* - handed 2 relay addrs at once, no deadline: `start()` never completes (10,002ms and counting)
|
|
7881
|
+
* - handed none: `start()` in 1ms, but NOBODY can dial the node —
|
|
7882
|
+
* the counterparty's re-dial fails
|
|
7883
|
+
* `counterparty_dial_failed` and every message in
|
|
7884
|
+
* both directions has to go the relay park route
|
|
7885
|
+
* - this: one candidate at a time, each raced against its
|
|
7886
|
+
* own deadline, plain node as the floor
|
|
7887
|
+
*
|
|
7888
|
+
* The middle option is what shipped for one test run and it made the session half-dead: revived,
|
|
7889
|
+
* `active`, and unreachable. The first is what shipped before that and it hung. Neither is a
|
|
7890
|
+
* choice between "fast" and "reliable" — the per-candidate race is how `#startReceiverNode` has
|
|
7891
|
+
* always done it, and it is the shape that works in production every day.
|
|
7892
|
+
*
|
|
7893
|
+
* A FAILED CANDIDATE IS TORN DOWN AT SETTLEMENT. The first version awaited `stop()` immediately
|
|
7894
|
+
* and claimed that made seed reuse safe; it did not — `libp2p.stop()` returns at once unless the
|
|
7895
|
+
* node is `'started'`, and during the timeout window it is `'starting'` (review HIGH-3, verified
|
|
7896
|
+
* against libp2p 3.3.2). The teardown is now chained onto the candidate's OWN start promise, so it
|
|
7897
|
+
* runs whenever that settles, however late.
|
|
7898
|
+
*
|
|
7899
|
+
* A BRIEF OVERLAP IS THEREFORE POSSIBLE and is stated rather than denied: a candidate that grants
|
|
7900
|
+
* at 4s comes up on this session's peer id and is stopped immediately after. What is guaranteed is
|
|
7901
|
+
* that it dies, not that it never lives. The receiver path avoids even that by minting a seed per
|
|
7902
|
+
* candidate; here the identity is fixed, which is the whole point of a revival, so that option
|
|
7903
|
+
* does not exist.
|
|
7904
|
+
*
|
|
7905
|
+
* The floor is a plain node: a session that is usable over the relay park route beats no session.
|
|
7906
|
+
*/
|
|
7907
|
+
async #buildRevivedNode(sessionId, gater, seed, candidateAddrs, agentName) {
|
|
7908
|
+
for (const circuitAddr of candidateAddrs.slice(0, REVIVE_RESERVATION_CANDIDATES)) {
|
|
7909
|
+
const candidate = await this.#factory.createNode({
|
|
7910
|
+
sessionId,
|
|
7911
|
+
connectionGater: gater,
|
|
7912
|
+
nodeType: "session",
|
|
7913
|
+
inboundReachable: true,
|
|
7914
|
+
transportPrivateKey: seed,
|
|
7915
|
+
circuitRelayListenAddrs: [circuitAddr],
|
|
7916
|
+
});
|
|
7917
|
+
// KEEP THE START PROMISE. Review HIGH-3: `libp2p.stop()` opens with
|
|
7918
|
+
// `if (this.status !== 'started') return`, and during the whole timeout window the status is
|
|
7919
|
+
// `'starting'` — so awaiting `stop()` on a timed-out candidate stopped nothing and waited for
|
|
7920
|
+
// nothing. The abandoned `start()` stayed in flight, and if the relay answered late the node
|
|
7921
|
+
// went live holding THIS SESSION'S peer id, sharing the gater (so it admits the counterparty)
|
|
7922
|
+
// with no content handler registered, and with no reference left to stop it. Verified against
|
|
7923
|
+
// libp2p 3.3.2 rather than assumed.
|
|
7924
|
+
const startP = candidate.start();
|
|
7925
|
+
let startError;
|
|
7926
|
+
const started = await Promise.race([
|
|
7927
|
+
startP.then(() => true),
|
|
7928
|
+
new Promise((res) => setTimeout(() => res(false), REVIVE_RESERVATION_TIMEOUT_MS).unref?.()),
|
|
7929
|
+
]).catch((err) => { startError = err; return false; });
|
|
7930
|
+
if (started && candidate.listenAddresses().some((a) => a.includes("/p2p-circuit"))) {
|
|
7931
|
+
this.#logger.info("session.revive.reservation.granted", { agentName, sessionId });
|
|
7932
|
+
return candidate;
|
|
7933
|
+
}
|
|
7934
|
+
// Started but granted nothing, or never started. Either way this node is not the one.
|
|
7935
|
+
//
|
|
7936
|
+
// Review MEDIUM-5: name WHICH of the three causes this was, the way `#startReceiverNode` does.
|
|
7937
|
+
// "declined" alone stood for a relay that is full, a relay that is unreachable, and a relay
|
|
7938
|
+
// that is merely slow — three different problems with three different responses, and the
|
|
7939
|
+
// thrown error was discarded entirely.
|
|
7940
|
+
const declineReason = started
|
|
7941
|
+
? "relay_granted_no_reservation"
|
|
7942
|
+
: startError !== undefined
|
|
7943
|
+
? "relay_unreachable"
|
|
7944
|
+
: "reservation_did_not_complete_in_time";
|
|
7945
|
+
const isLast = circuitAddr === candidateAddrs.slice(0, REVIVE_RESERVATION_CANDIDATES).at(-1);
|
|
7946
|
+
this.#logger.warn("session.revive.reservation.declined", {
|
|
7947
|
+
agentName,
|
|
7948
|
+
sessionId,
|
|
7949
|
+
circuitAddr,
|
|
7950
|
+
reason: declineReason,
|
|
7951
|
+
...(startError !== undefined ? { error: extractErrorMessage(startError) } : {}),
|
|
7952
|
+
impact: isLast
|
|
7953
|
+
? "no relay granted; the session comes up reachable only via the relay park route"
|
|
7954
|
+
: "trying the next relay",
|
|
7955
|
+
});
|
|
7956
|
+
// Teardown at SETTLEMENT, not now: a `stop()` issued while the node is still starting is a
|
|
7957
|
+
// no-op (see above), so the only way to guarantee this node dies is to wait for its own start
|
|
7958
|
+
// to finish first. Not awaited, so a hung start cannot hold the revival up — the point is that
|
|
7959
|
+
// the teardown eventually happens, not that it happens before the next candidate.
|
|
7960
|
+
void startP.then(() => candidate.stop().catch(() => { }), () => { });
|
|
7961
|
+
}
|
|
7962
|
+
// THE FLOOR. No reservation, so the counterparty cannot dial us directly — but their messages
|
|
7963
|
+
// park at the relay and drain, which is how every message in the 2026-08-18 test arrived. A
|
|
7964
|
+
// session usable one way beats a session that never comes back.
|
|
7965
|
+
const plain = await this.#factory.createNode({
|
|
7966
|
+
sessionId,
|
|
7967
|
+
connectionGater: gater,
|
|
7968
|
+
nodeType: "session",
|
|
7969
|
+
inboundReachable: true,
|
|
7970
|
+
transportPrivateKey: seed,
|
|
7971
|
+
});
|
|
7972
|
+
await plain.start();
|
|
7973
|
+
if (candidateAddrs.length > 0) {
|
|
7974
|
+
this.#logger.warn("session.revive.reservation.none", {
|
|
7975
|
+
agentName,
|
|
7976
|
+
sessionId,
|
|
7977
|
+
candidates: candidateAddrs.length,
|
|
7978
|
+
impact: "the revived session holds no circuit address — the counterparty cannot dial it, so "
|
|
7979
|
+
+ "delivery in both directions depends on relay store-and-forward until it is rebuilt",
|
|
7980
|
+
});
|
|
7981
|
+
}
|
|
7982
|
+
return plain;
|
|
7983
|
+
}
|
|
7984
|
+
/**
|
|
7985
|
+
* DOD-M12B-REVIVE-RELAY-1 — reconnect the session's relay WITNESS, which revival never did.
|
|
7986
|
+
*
|
|
7987
|
+
* THE FIRST-PRINCIPLES DEFECT, and the one that explains every symptom chased separately before
|
|
7988
|
+
* it. Establishment does five things: build the node, register the content handler, wire liveness,
|
|
7989
|
+
* **connect the relay**, and dial the counterparty. Revival did the first three. A revived session
|
|
7990
|
+
* was therefore not a session — it looked live, reported `active`, and had no live inbound path at
|
|
7991
|
+
* all.
|
|
7992
|
+
*
|
|
7993
|
+
* MEASURED 2026-08-18 with two real agents: a message on a reconnected session took **three
|
|
7994
|
+
* minutes**, against seconds on a fresh one, because only the five-minute mailbox backstop ever
|
|
7995
|
+
* found it. Doorbells stopped firing for the same reason — the relay stream is what rings them.
|
|
7996
|
+
* And `#parkContent` refuses without `entry.relayClient`, so sends could not park either.
|
|
7997
|
+
*
|
|
7998
|
+
* NO ASSIGNMENT IS PRESENTED, and that is by design rather than omission: `RelayConnectParams`
|
|
7999
|
+
* documents the reconnect mode itself — *"absent … on the restart/persisted reconnect path (the
|
|
8000
|
+
* relay already recorded the session at first establishment) — the client then just reconnects
|
|
8001
|
+
* without re-recording."* A revival is exactly that path.
|
|
8002
|
+
*
|
|
8003
|
+
* Best-effort and non-fatal: a session that comes back without its witness is still better than
|
|
8004
|
+
* one that does not come back, and the failure is named rather than silent.
|
|
8005
|
+
*/
|
|
8006
|
+
async #reconnectRevivedSessionRelay(agentName, sessionId, node, gater, correlationId, ep) {
|
|
8007
|
+
if (!ep) {
|
|
8008
|
+
this.#logger.warn("session.revive.relay.absent", {
|
|
8009
|
+
agentName,
|
|
8010
|
+
sessionId,
|
|
8011
|
+
impact: "no relay is recorded for this session, so it comes back with no live inbound path — "
|
|
8012
|
+
+ "messages arrive only on the periodic mailbox poll, and a failed send cannot park and is "
|
|
8013
|
+
+ "reported lost",
|
|
8014
|
+
});
|
|
8015
|
+
return;
|
|
8016
|
+
}
|
|
8017
|
+
try {
|
|
8018
|
+
// The gater admits only the counterparty inbound; the relay is a third peer and must be
|
|
8019
|
+
// permitted OUTBOUND or our own gate refuses the dial (INV-5 keeps inbound counterparty-only).
|
|
8020
|
+
gater.setAllowedOutboundPeer(ep.relayPeerId);
|
|
8021
|
+
const clientKey = `${agentName}::${ep.relayPeerId}`;
|
|
8022
|
+
let client = this.#relayClients.get(clientKey);
|
|
8023
|
+
if (!client) {
|
|
8024
|
+
if (!this.#relayReceiptStore && this.#db)
|
|
8025
|
+
this.#relayReceiptStore = new RelayReceiptStore(this.#db, this.#logger);
|
|
8026
|
+
if (!this.#sealLeafStore && this.#db)
|
|
8027
|
+
this.#sealLeafStore = new SessionSealLeafStore(this.#db, this.#logger);
|
|
8028
|
+
client = this.#detachedRelayClientBuilder?.(agentName, ep.relayPeerId, [...ep.relayAddrs], {
|
|
8029
|
+
receiptStore: this.#relayReceiptStore ?? undefined,
|
|
8030
|
+
sealLeafStore: this.#sealLeafStore ?? undefined,
|
|
8031
|
+
});
|
|
8032
|
+
if (!client) {
|
|
8033
|
+
this.#logger.warn("session.revive.relay.builder_absent", {
|
|
8034
|
+
agentName,
|
|
8035
|
+
sessionId,
|
|
8036
|
+
impact: "no relay client could be built, so this revived session has no live inbound path",
|
|
8037
|
+
});
|
|
8038
|
+
return;
|
|
8039
|
+
}
|
|
8040
|
+
this.#relayClients.set(clientKey, client);
|
|
8041
|
+
}
|
|
8042
|
+
client.registerSession(sessionId, node, this.#relayLeafHandler(agentName, sessionId, correlationId));
|
|
8043
|
+
const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
|
|
8044
|
+
if (entry) {
|
|
8045
|
+
entry.relayClient = client;
|
|
8046
|
+
entry.relaySessionIdBytes = Uint8Array.from(Buffer.from(sessionId, "hex"));
|
|
8047
|
+
entry.relayClientKey = clientKey;
|
|
8048
|
+
}
|
|
8049
|
+
/**
|
|
8050
|
+
* THE STEP THIS METHOD IS NAMED AFTER, and the first version did not take it (review HIGH-2).
|
|
8051
|
+
*
|
|
8052
|
+
* `registerSession` files a handler in a Map. It opens nothing — no dial, no auth, no reader
|
|
8053
|
+
* loop. `#connectSessionRelay` ends with exactly this line and the reconnect ended without it,
|
|
8054
|
+
* so a revived session registered a handler on a client whose stream was `null` and then
|
|
8055
|
+
* logged that its live inbound path was back. It was not: the counterparty's leaves queued at
|
|
8056
|
+
* the relay, no doorbell fired, and delivery fell back to the five-minute mailbox poll — the
|
|
8057
|
+
* three-minutes-versus-seconds symptom the whole unit exists to remove.
|
|
8058
|
+
*
|
|
8059
|
+
* Worse, the client is usually BRAND NEW here: `markInterruptedWithDetails` closes and drops
|
|
8060
|
+
* the client for the last session on a relay, so a single-session agent always lands in the
|
|
8061
|
+
* build branch above with a fresh, unconnected client.
|
|
8062
|
+
*
|
|
8063
|
+
* `#ensureConnected` is idempotent, so this also repairs the cached-but-dead-stream case.
|
|
8064
|
+
*/
|
|
8065
|
+
await client.connect(node);
|
|
8066
|
+
this.#logger.info("session.revive.relay.connected", {
|
|
8067
|
+
agentName,
|
|
8068
|
+
sessionId,
|
|
8069
|
+
relayPeerId: ep.relayPeerId,
|
|
8070
|
+
impact: "the revived session has its live inbound path back — messages arrive promptly "
|
|
8071
|
+
+ "instead of waiting for the periodic mailbox poll",
|
|
8072
|
+
});
|
|
8073
|
+
}
|
|
8074
|
+
catch (err) {
|
|
8075
|
+
this.#logger.warn("session.revive.relay.failed", {
|
|
8076
|
+
agentName,
|
|
8077
|
+
sessionId,
|
|
8078
|
+
error: err instanceof Error ? err.message : String(err),
|
|
8079
|
+
impact: "the session is back but without its witness — delivery falls back to the periodic poll",
|
|
8080
|
+
});
|
|
8081
|
+
}
|
|
8082
|
+
}
|
|
8083
|
+
/**
|
|
8084
|
+
* DOD-M12B-SESSION-SEED-1 — bring an interrupted session back on the peer id it already has.
|
|
8085
|
+
*
|
|
8086
|
+
* THE DEFECT THIS CLOSES. `markInterruptedWithDetails` and `destroySessionNode` stop the node and
|
|
8087
|
+
* delete it from `#activeNodes`, and until now **nothing anywhere recreated one**. A laptop-close
|
|
8088
|
+
* session stayed stuck even though both processes were alive and both keypairs were still in
|
|
8089
|
+
* memory — the trace on 2026-08-17 found no missing transport capability, just a missing edge.
|
|
8090
|
+
*
|
|
8091
|
+
* TWO THINGS HAVE TO HAPPEN, and doing only one leaves the session exactly as stuck:
|
|
8092
|
+
* 1. the NODE comes back, at the same peer id, or the counterparty can never dial us again;
|
|
8093
|
+
* 2. the STATUS comes back to `active`, or every send still refuses with `session_not_active`.
|
|
8094
|
+
*
|
|
8095
|
+
* **DEMAND-DRIVEN ONLY.** Nothing calls this on a timer. That is the `REDIAL-1` discipline and it
|
|
8096
|
+
* is also Andre's tenet — a background rebuilder would hold a dialable endpoint open for a session
|
|
8097
|
+
* nobody is using, which is the "open connection a malicious agent can farm for" in as many words.
|
|
8098
|
+
*
|
|
8099
|
+
* **TERMINAL IS TERMINAL.** A sealed or abandoned session had its seed zeroed in the same step
|
|
8100
|
+
* that wrote its status, so there is nothing to come back on. This refuses by name rather than
|
|
8101
|
+
* minting a fresh identity — a revival that quietly mints would hand one session a second peer id
|
|
8102
|
+
* and break the invariant while appearing to work.
|
|
8103
|
+
*
|
|
8104
|
+
* Idempotent: a session that already has a live node returns ok without building a second one.
|
|
8105
|
+
*/
|
|
8106
|
+
async reviveSessionNode(agentName, sessionId) {
|
|
8107
|
+
const key = this.#k(agentName, sessionId);
|
|
8108
|
+
const live = this.#activeNodes.get(key);
|
|
8109
|
+
if (live)
|
|
8110
|
+
return { ok: true, peerId: live.node.getPeerId() };
|
|
8111
|
+
const record = this.getSessionRecord(agentName, sessionId);
|
|
8112
|
+
if (!record)
|
|
8113
|
+
return { ok: false, reason: "session_not_found" };
|
|
8114
|
+
if (record.status === "sealed" || record.status === "abandoned" || record.status === "seal_interrupted_pending") {
|
|
8115
|
+
return {
|
|
8116
|
+
ok: false,
|
|
8117
|
+
reason: "session_terminal",
|
|
8118
|
+
guidance: `Session is '${record.status}'. A session that has ended cannot be revived; start a new one.`,
|
|
8119
|
+
};
|
|
8120
|
+
}
|
|
8121
|
+
/**
|
|
8122
|
+
* THE CAP APPLIES TO A REVIVAL TOO (review: parity gap). Establishment refuses at
|
|
8123
|
+
* `MAX_SESSION_NODES` because each node is a real libp2p instance with listeners, connections
|
|
8124
|
+
* and a relay reservation. A revival builds exactly the same thing, so letting it past the cap
|
|
8125
|
+
* would let a daemon walk over the limit one reconnect at a time — and the limit exists to stop
|
|
8126
|
+
* a machine being taken down by its own session count.
|
|
8127
|
+
*
|
|
8128
|
+
* Refused by name, so the caller can say something true: this is a local resource limit, not a
|
|
8129
|
+
* problem with the session or the counterparty.
|
|
8130
|
+
*/
|
|
8131
|
+
if (this.#activeNodes.size >= MAX_SESSION_NODES) {
|
|
8132
|
+
this.#logger.warn("session.revive.cap.reached", {
|
|
8133
|
+
agentName,
|
|
8134
|
+
sessionId,
|
|
8135
|
+
activeCount: this.#activeNodes.size,
|
|
8136
|
+
maxCount: MAX_SESSION_NODES,
|
|
8137
|
+
impact: "this session stays interrupted until another session ends and frees a node slot",
|
|
8138
|
+
});
|
|
8139
|
+
return {
|
|
8140
|
+
ok: false,
|
|
8141
|
+
reason: "session_node_cap_reached",
|
|
8142
|
+
guidance: `This daemon already holds ${MAX_SESSION_NODES} active session nodes, so this session ` +
|
|
8143
|
+
"cannot be brought back yet. Close a session you have finished with and try again.",
|
|
8144
|
+
};
|
|
8145
|
+
}
|
|
8146
|
+
const identity = this.#sessionSeeds.get(key);
|
|
8147
|
+
if (identity === undefined) {
|
|
8148
|
+
// The honest case: the daemon restarted, so the keypair is genuinely gone. That is
|
|
8149
|
+
// RESTART-SEAL-1's territory (resolve with a receipt), not a revival — and saying so is the
|
|
8150
|
+
// difference between an operator waiting for a reconnect that cannot happen and one closing
|
|
8151
|
+
// the session.
|
|
8152
|
+
return {
|
|
8153
|
+
ok: false,
|
|
8154
|
+
reason: "session_identity_lost",
|
|
8155
|
+
guidance: "This session's transport identity did not survive a daemon restart, so it cannot be " +
|
|
8156
|
+
"revived. It will be sealed automatically, or you can close it now to get its receipt.",
|
|
8157
|
+
};
|
|
8158
|
+
}
|
|
8159
|
+
const gater = new SessionConnectionGater({
|
|
8160
|
+
sessionId,
|
|
8161
|
+
allowedPeerId: identity.counterpartyPeerId,
|
|
8162
|
+
logger: this.#logger,
|
|
8163
|
+
});
|
|
8164
|
+
// The relay peers must be allowed OUTBOUND before the node starts, or the reservation the line
|
|
8165
|
+
// below depends on is refused by our own gater — the same ordering the receiver builder uses.
|
|
8166
|
+
const reservations = this.#reservationCircuitAddrs(agentName);
|
|
8167
|
+
for (const relayPeerId of reservations.relayPeerIds)
|
|
8168
|
+
gater.setAllowedOutboundPeer(relayPeerId);
|
|
8169
|
+
let node;
|
|
8170
|
+
const t0 = Date.now();
|
|
8171
|
+
this.#logger.info("session.revive.node.building", {
|
|
8172
|
+
agentName,
|
|
8173
|
+
sessionId,
|
|
8174
|
+
circuitAddrs: reservations.addrs.length,
|
|
8175
|
+
relayPeerIds: reservations.relayPeerIds.length,
|
|
8176
|
+
});
|
|
8177
|
+
try {
|
|
8178
|
+
node = await this.#buildRevivedNode(sessionId, gater, identity.seed, reservations.addrs, agentName);
|
|
8179
|
+
this.#logger.info("session.revive.node.started", {
|
|
8180
|
+
agentName,
|
|
8181
|
+
sessionId,
|
|
8182
|
+
startMs: Date.now() - t0,
|
|
8183
|
+
listenAddrs: node.listenAddresses().length,
|
|
8184
|
+
circuitListen: node.listenAddresses().filter((a) => a.includes("/p2p-circuit")).length,
|
|
8185
|
+
});
|
|
8186
|
+
}
|
|
8187
|
+
catch (err) {
|
|
8188
|
+
this.#logger.error("session.revive.node.failed", {
|
|
8189
|
+
agentName,
|
|
8190
|
+
sessionId,
|
|
8191
|
+
error: err instanceof Error ? err.message : String(err),
|
|
8192
|
+
impact: "the session stays interrupted; the next send will attempt this again",
|
|
8193
|
+
});
|
|
8194
|
+
return { ok: false, reason: "session_node_creation_failed" };
|
|
8195
|
+
}
|
|
8196
|
+
const autoNat = new NodeAutoNatService({
|
|
8197
|
+
node,
|
|
8198
|
+
logger: this.#logger,
|
|
8199
|
+
nodeType: "session",
|
|
8200
|
+
probers: this.#autoNatProbers(),
|
|
8201
|
+
});
|
|
8202
|
+
autoNat.emitInitialResult();
|
|
8203
|
+
// DOD-M12B-SESSION-SEED-1: give the re-dial its addresses back BEFORE the session goes active,
|
|
8204
|
+
// so the first send after a revival has somewhere to go. Without this the send fails instantly
|
|
8205
|
+
// on a connection that was never made, and — measured live — is lost rather than parked.
|
|
8206
|
+
if (identity.counterpartyAddrs.length > 0) {
|
|
8207
|
+
this.#counterpartyAddrs.set(key, [...identity.counterpartyAddrs]);
|
|
8208
|
+
}
|
|
8209
|
+
const correlationId = randomUUID();
|
|
8210
|
+
/**
|
|
8211
|
+
* DOD-M12B-REVIVE-PARK-1 — RESTORE THE RELAY, or a revived session cannot park and every failed
|
|
8212
|
+
* send is declared lost.
|
|
8213
|
+
*
|
|
8214
|
+
* This is the defect behind five identical live failures on 2026-08-18. `#parkContent` opens
|
|
8215
|
+
* with `if (!hook || !entry || !entry.relayPeerId || !entry.relayAddrs) return "unconfigured"`,
|
|
8216
|
+
* and a revived entry carried none of it — so the park was skipped and the send fell through to
|
|
8217
|
+
* *"could NOT be queued for retry — it is lost. Send it again."* The relay was recorded on the
|
|
8218
|
+
* session row the whole time, and store-and-forward would have delivered the message: the
|
|
8219
|
+
* counterparty's own sends park through it successfully in the same minute.
|
|
8220
|
+
*
|
|
8221
|
+
* What it cost the operator: their reply was accepted, discarded, and they were told to retype
|
|
8222
|
+
* it — which is how a transcript gets duplicates of a message that was never lost in the first
|
|
8223
|
+
* place.
|
|
8224
|
+
*
|
|
8225
|
+
* Read from the row rather than carried in the revival record on purpose: the row is where the
|
|
8226
|
+
* relay assignment is durable, and it is the same source `getPersistedRelayEndpoint` already
|
|
8227
|
+
* serves the startup flush from — a path that exists precisely because in-memory entries are
|
|
8228
|
+
* gone by then, which is exactly the situation a revival is in.
|
|
8229
|
+
*/
|
|
8230
|
+
// ONE lookup, and ONE event for the absent case (review LOW-8). This used to read the endpoint
|
|
8231
|
+
// here and again inside the relay reconnect, and both logged `session.revive.relay.absent` with
|
|
8232
|
+
// different `impact` text — one event name standing for two meanings, fired twice for a single
|
|
8233
|
+
// condition. `#reconnectRevivedSessionRelay` takes it as a parameter now.
|
|
8234
|
+
const persistedRelay = this.getPersistedRelayEndpoint(agentName, sessionId);
|
|
8235
|
+
this.#activeNodes.set(key, {
|
|
8236
|
+
node,
|
|
8237
|
+
agentName,
|
|
8238
|
+
sessionId,
|
|
8239
|
+
counterpartyPubkey: identity.counterpartyPubkey,
|
|
8240
|
+
gater,
|
|
8241
|
+
correlationId,
|
|
8242
|
+
counterpartySessionPeerId: identity.counterpartyPeerId,
|
|
8243
|
+
autoNat,
|
|
8244
|
+
...(persistedRelay
|
|
8245
|
+
? { relayPeerId: persistedRelay.relayPeerId, relayAddrs: persistedRelay.relayAddrs }
|
|
8246
|
+
: {}),
|
|
8247
|
+
});
|
|
8248
|
+
await this.#registerContentHandler(agentName, sessionId, node, identity.counterpartyPubkey);
|
|
8249
|
+
/**
|
|
8250
|
+
* review HIGH-2 — REWIRE LIVENESS, or this session can never be interrupted again.
|
|
8251
|
+
*
|
|
8252
|
+
* Both creation paths call this; the first build of the revival did not. Without it the revived
|
|
8253
|
+
* session is pinned `active`: a later disconnect fires no transition, no `session_state_changed`
|
|
8254
|
+
* reaches the MCP client, and the receive surface renders unknown liveness as healthy-and-quiet.
|
|
8255
|
+
* So the SECOND laptop close would leave the operator staring at a session that reports fine and
|
|
8256
|
+
* is dead — this milestone's founding defect, one revival later, and with no status change left
|
|
8257
|
+
* to trigger the next revival either.
|
|
8258
|
+
*/
|
|
8259
|
+
this.#wireSessionLiveness(agentName, sessionId, node, identity.counterpartyPubkey, correlationId, identity.counterpartyPeerId);
|
|
8260
|
+
// DOD-M12B-REVIVE-RELAY-1: the step revival skipped. Establishment connects the relay witness
|
|
8261
|
+
// here; without it the session comes back with no live inbound path at all.
|
|
8262
|
+
await this.#reconnectRevivedSessionRelay(agentName, sessionId, node, gater, correlationId, persistedRelay);
|
|
8263
|
+
// THE REVERSE EDGE. A transport event took this session out of `active` and nothing has ever
|
|
8264
|
+
// put one back. Written after the node is live and its handler registered, so the row never
|
|
8265
|
+
// claims `active` for a session that cannot yet receive.
|
|
8266
|
+
//
|
|
8267
|
+
// review MEDIUM-3: the result is CHECKED. `#updateSessionStatus` returns false when the write
|
|
8268
|
+
// matched no row or the DB errored — and reporting revival ok on a row that still says
|
|
8269
|
+
// `interrupted` leaves a live, talking session where REVIVAL-BOUND-1's sweep can seal or abandon
|
|
8270
|
+
// it. Failing here means tearing the node back down rather than running in that split state.
|
|
8271
|
+
if (!this.#updateSessionStatus(agentName, sessionId, "active")) {
|
|
8272
|
+
this.#activeNodes.delete(key);
|
|
8273
|
+
try {
|
|
8274
|
+
await node.stop();
|
|
8275
|
+
}
|
|
8276
|
+
catch { /* best-effort: the status write already failed and is logged with its cause */ }
|
|
8277
|
+
return {
|
|
8278
|
+
ok: false,
|
|
8279
|
+
reason: "session_status_write_failed",
|
|
8280
|
+
guidance: "The session node was rebuilt but its status could not be written, so it was torn back " +
|
|
8281
|
+
"down rather than left live under an interrupted row. The daemon logged the cause.",
|
|
8282
|
+
};
|
|
8283
|
+
}
|
|
8284
|
+
// The messages that failed while this session was down were queued on a promise of "retried on
|
|
8285
|
+
// reconnect". This is that reconnect — fire it before anyone is told the session is back.
|
|
8286
|
+
if (this.#retryDrainHook !== null) {
|
|
8287
|
+
try {
|
|
8288
|
+
this.#retryDrainHook(agentName, sessionId);
|
|
8289
|
+
}
|
|
8290
|
+
catch (err) {
|
|
8291
|
+
this.#logger.warn("session.revive.retry_drain.failed", {
|
|
8292
|
+
agentName,
|
|
8293
|
+
sessionId,
|
|
8294
|
+
error: err instanceof Error ? err.message : String(err),
|
|
8295
|
+
impact: "messages queued while this session was down are still queued",
|
|
8296
|
+
});
|
|
8297
|
+
}
|
|
8298
|
+
}
|
|
8299
|
+
const peerId = node.getPeerId();
|
|
8300
|
+
this.#logger.info("session.revived", {
|
|
8301
|
+
agentName,
|
|
8302
|
+
sessionId,
|
|
8303
|
+
peerId,
|
|
8304
|
+
// The whole claim of this line, in the log: the id did not change, so the counterparty's
|
|
8305
|
+
// stored dial target is still correct and they do not need to be told anything.
|
|
8306
|
+
identityPreserved: true,
|
|
8307
|
+
});
|
|
8308
|
+
return { ok: true, peerId };
|
|
8309
|
+
}
|
|
8310
|
+
/**
|
|
8311
|
+
* DOD-M12B-SESSION-SEED-1 — the DEMAND edge: a send on an interrupted session revives it.
|
|
8312
|
+
*
|
|
8313
|
+
* One of TWO production callers of `reviveSessionNode` — `reviveIfNeededForRead` is the other —
|
|
8314
|
+
* and both are deliberately demand paths rather than timers. The `REDIAL-1` discipline and Andre's tenet say the same thing from two
|
|
8315
|
+
* directions: nothing may re-open on its own, because a background rebuilder would hold a dialable
|
|
8316
|
+
* endpoint open for a session nobody is using — the *"open connection a malicious agent can farm
|
|
8317
|
+
* for"*. The operator sending is the demand; there is no other trigger.
|
|
8318
|
+
*
|
|
8319
|
+
* A no-op for the normal case. An `active` session with a live node returns immediately without
|
|
8320
|
+
* touching it — this sits on the hot path of every send, and replacing a healthy node would be
|
|
8321
|
+
* churn that changes the peer id for no reason.
|
|
8322
|
+
*/
|
|
8323
|
+
async reviveIfNeededForSend(agentName, sessionId) {
|
|
8324
|
+
const record = this.getSessionRecord(agentName, sessionId);
|
|
8325
|
+
if (!record)
|
|
8326
|
+
return { ok: false, reason: "session_not_found" };
|
|
8327
|
+
// The overwhelmingly common case: nothing to do, and no node was disturbed to find that out.
|
|
8328
|
+
if (record.status === "active" && this.#activeNodes.has(this.#k(agentName, sessionId)))
|
|
8329
|
+
return { ok: true };
|
|
8330
|
+
const revived = await this.reviveSessionNode(agentName, sessionId);
|
|
8331
|
+
if (!revived.ok) {
|
|
8332
|
+
this.#logger.info("session.revive.declined", {
|
|
8333
|
+
agentName,
|
|
8334
|
+
sessionId,
|
|
8335
|
+
previousStatus: record.status,
|
|
8336
|
+
trigger: "send",
|
|
8337
|
+
reason: revived.reason,
|
|
8338
|
+
});
|
|
8339
|
+
return revived;
|
|
8340
|
+
}
|
|
8341
|
+
this.#logger.info("session.revived.on_demand", {
|
|
8342
|
+
agentName,
|
|
8343
|
+
sessionId,
|
|
8344
|
+
previousStatus: record.status,
|
|
8345
|
+
trigger: "send",
|
|
8346
|
+
});
|
|
8347
|
+
return { ok: true };
|
|
8348
|
+
}
|
|
8349
|
+
/**
|
|
8350
|
+
* DOD-M12B-SESSION-SEED-1 (case B) — the INBOUND half of the demand edge.
|
|
8351
|
+
*
|
|
8352
|
+
* `reviveIfNeededForSend` covers the operator waking first. Case B's triggers are symmetric — a
|
|
8353
|
+
* wifi hop, a relay restart, a directory node cycling — so half the time the COUNTERPARTY wakes
|
|
8354
|
+
* first. They send; we have no node yet, because revival is demand-driven and we have demanded
|
|
8355
|
+
* nothing. Their content parks at the relay, which is the backstop working as designed.
|
|
8356
|
+
*
|
|
8357
|
+
* Then the operator comes back and READS, and until now that told them nothing: the receive
|
|
8358
|
+
* handler reads the transcript and never gates on status, so it happily reports what is already
|
|
8359
|
+
* stored while messages sit parked, waiting for a node that will not exist until the operator
|
|
8360
|
+
* happens to SEND. An operator who only reads was stuck forever with a surface that looked fine.
|
|
8361
|
+
*
|
|
8362
|
+
* **WHY A READ MAY TRIGGER THIS AND AN INBOUND DIAL MAY NOT.** Andre's tenet is about what a
|
|
8363
|
+
* REMOTE party can cause: *"an open connection that a malicious agent can farm for."* Reviving
|
|
8364
|
+
* because a peer dialled us would hand that lever straight to the peer — a stranger could keep our
|
|
8365
|
+
* endpoints open indefinitely by poking dead sessions. A read is the OPERATOR asking, on their own
|
|
8366
|
+
* machine, for their own session: the same class of demand as a send, and the class the tenet
|
|
8367
|
+
* allows. That distinction is the whole reason this is a separate entry point rather than a
|
|
8368
|
+
* revival triggered from the inbound handler.
|
|
8369
|
+
*/
|
|
8370
|
+
async reviveIfNeededForRead(agentName, sessionId) {
|
|
8371
|
+
const record = this.getSessionRecord(agentName, sessionId);
|
|
8372
|
+
if (!record)
|
|
8373
|
+
return { ok: false, reason: "session_not_found" };
|
|
8374
|
+
if (record.status === "active" && this.#activeNodes.has(this.#k(agentName, sessionId)))
|
|
8375
|
+
return { ok: true };
|
|
8376
|
+
// Reading the transcript of an ended session is normal and must keep working — the CALLER does
|
|
8377
|
+
// not treat this refusal as an error, it just reads what is stored. What must not happen is the
|
|
8378
|
+
// read bringing the session back: the receipt is issued and the identity is gone.
|
|
8379
|
+
const revived = await this.reviveSessionNode(agentName, sessionId);
|
|
8380
|
+
if (!revived.ok) {
|
|
8381
|
+
// review MEDIUM-4: the absence of a success line was the only signal that a session could not
|
|
8382
|
+
// come back. `session_identity_lost` is the one an operator most needs, and it was generated
|
|
8383
|
+
// and destroyed one stack frame later with nothing written down.
|
|
8384
|
+
this.#logger.info("session.revive.declined", {
|
|
8385
|
+
agentName,
|
|
8386
|
+
sessionId,
|
|
8387
|
+
previousStatus: record.status,
|
|
8388
|
+
trigger: "read",
|
|
8389
|
+
reason: revived.reason,
|
|
8390
|
+
});
|
|
8391
|
+
return revived;
|
|
8392
|
+
}
|
|
8393
|
+
this.#logger.info("session.revived.on_demand", {
|
|
8394
|
+
agentName,
|
|
8395
|
+
sessionId,
|
|
8396
|
+
previousStatus: record.status,
|
|
8397
|
+
trigger: "read",
|
|
8398
|
+
});
|
|
8399
|
+
// Fetch what is waiting NOW. Review MEDIUM-5 corrected the claim this used to make: the drain
|
|
8400
|
+
// runs off the AGENT's standing receiver, not the session node, and the 5-minute backstop would
|
|
8401
|
+
// have delivered this content anyway. So this is an accelerator, not a rescue — worth having,
|
|
8402
|
+
// and worth describing accurately. (The send path deliberately does not fire one: the same
|
|
8403
|
+
// backstop covers it, at a cost of at most one interval.)
|
|
8404
|
+
this.#fireParkedDrain(agentName, "session_revived");
|
|
8405
|
+
return { ok: true };
|
|
8406
|
+
}
|
|
8407
|
+
/** DOD-M12B-REVIVE-PARK-1 test seam: the relay the live entry will park to. Not otherwise
|
|
8408
|
+
* observable — `#activeNodes` is private and the park's own refusal is silent about which of its
|
|
8409
|
+
* four preconditions was missing. */
|
|
8410
|
+
getSessionRelayForTest(agentName, sessionId) {
|
|
8411
|
+
const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
|
|
8412
|
+
if (!entry)
|
|
8413
|
+
return null;
|
|
8414
|
+
return {
|
|
8415
|
+
...(entry.relayPeerId !== undefined ? { relayPeerId: entry.relayPeerId } : {}),
|
|
8416
|
+
...(entry.relayAddrs !== undefined ? { relayAddrs: entry.relayAddrs } : {}),
|
|
8417
|
+
};
|
|
8418
|
+
}
|
|
8419
|
+
/** DOD-M12B-SESSION-SEED-1 test seams: the counterparty addresses a re-dial depends on. Not
|
|
8420
|
+
* otherwise observable — they are set from a signed relay assignment that a fixture cannot mint. */
|
|
8421
|
+
setCounterpartyAddrsForTest(agentName, sessionId, addrs) {
|
|
8422
|
+
this.#counterpartyAddrs.set(this.#k(agentName, sessionId), [...addrs]);
|
|
8423
|
+
}
|
|
8424
|
+
getCounterpartyAddrsForTest(agentName, sessionId) {
|
|
8425
|
+
return this.#counterpartyAddrs.get(this.#k(agentName, sessionId)) ?? [];
|
|
8426
|
+
}
|
|
8427
|
+
/** DOD-M12B-SESSION-SEED-1 test seam: drop a seed WITHOUT zeroing or a status change — what a
|
|
8428
|
+
* process restart does to it. The refusal that follows is the one an operator most needs named. */
|
|
8429
|
+
forgetSessionSeedForTest(agentName, sessionId) {
|
|
8430
|
+
this.#sessionSeeds.delete(this.#k(agentName, sessionId));
|
|
8431
|
+
}
|
|
8432
|
+
/**
|
|
8433
|
+
* DOD-M12B-SESSION-SEED-1 — drain the direct-resend queue when a session comes back.
|
|
8434
|
+
*
|
|
8435
|
+
* `retryQueue.drainSession` had NO production caller. The send path enqueues into it on a failed
|
|
8436
|
+
* delivery and tells the operator the message will be "retried on reconnect", and nothing ever
|
|
8437
|
+
* reconnected it — the row sat there until the session went terminal and was reaped. Measured
|
|
8438
|
+
* live 2026-08-18: two of the operator's messages went in, and the response told them both were
|
|
8439
|
+
* lost and to send again, which is how a transcript gets duplicates.
|
|
8440
|
+
*
|
|
8441
|
+
* A revival IS the reconnect that sentence promised. This is the hook that makes it true.
|
|
8442
|
+
*/
|
|
8443
|
+
setRetryDrainHook(fn) {
|
|
8444
|
+
this.#retryDrainHook = fn;
|
|
8445
|
+
}
|
|
8446
|
+
/** DOD-M12B-SESSION-SEED-1 test seam: does this session still hold a revivable identity? */
|
|
8447
|
+
hasSessionSeedForTest(agentName, sessionId) {
|
|
8448
|
+
return this.#sessionSeeds.has(this.#k(agentName, sessionId));
|
|
8449
|
+
}
|
|
8450
|
+
getStandingReceiverSeedForTest(agentName) {
|
|
8451
|
+
return this.#standingReceivers.get(agentName)?.seed;
|
|
8452
|
+
}
|
|
7358
8453
|
async ensureStandingReceiverForAgent(agentName) {
|
|
7359
8454
|
this.#agentsWantingReceiver.add(agentName);
|
|
7360
8455
|
this.#startReservationWatchdog();
|
|
@@ -7385,7 +8480,11 @@ export class SessionNodeManager {
|
|
|
7385
8480
|
this.#standingReceiverRemoving.add(agentName);
|
|
7386
8481
|
return;
|
|
7387
8482
|
}
|
|
8483
|
+
const removed = this.#standingReceivers.get(agentName);
|
|
7388
8484
|
this.#standingReceivers.delete(agentName);
|
|
8485
|
+
// DOD-M12B-SESSION-SEED-1 (review F8): the operator took this agent offline — its advertised
|
|
8486
|
+
// identity is not needed any more, and the tenet's rule is that nothing unneeded stays live.
|
|
8487
|
+
removed?.seed.fill(0);
|
|
7389
8488
|
// Best-effort teardown, but NOT silent: a standing receiver that failed to stop keeps a libp2p
|
|
7390
8489
|
// node live on the network. For a removal/retire (a revocation-class action) that must be visible,
|
|
7391
8490
|
// so the caller and the operator can see the leak rather than trust a false "torn down". autoNat is
|
|
@@ -7428,6 +8527,34 @@ export class SessionNodeManager {
|
|
|
7428
8527
|
/** CC-5/F21: count of RECEIVED messages on a session — the "did the counterparty ever speak"
|
|
7429
8528
|
* signal the dead-half-open reaper uses (message_count also counts our own auto-"Dispatched." ack,
|
|
7430
8529
|
* so it is NOT a reliable half-open discriminator). Mirrors #getReceivedBytesTotal. */
|
|
8530
|
+
/**
|
|
8531
|
+
* DOD-M12B-REAP-HELD-1 — did the counterparty EVER establish? Counted from every place their
|
|
8532
|
+
* messages can be, not just the one.
|
|
8533
|
+
*
|
|
8534
|
+
* OBSERVED LIVE 2026-08-18: the half-open reaper abandoned session `d28db475…` — twenty leaves in
|
|
8535
|
+
* the chain and sixteen more frames verified and held, ten of them from the counterparty — while
|
|
8536
|
+
* the restart-seal resolver was actively trying to notarize it. The receipt was forfeited.
|
|
8537
|
+
*
|
|
8538
|
+
* `countReceivedMessages` asks the TRANSCRIPT, and **held content never reaches the transcript**;
|
|
8539
|
+
* it sits in `held_content` until it can join the chain. So the very condition that holds content
|
|
8540
|
+
* — an interrupted session — is the condition that makes the counterparty's messages invisible,
|
|
8541
|
+
* and a fully-established conversation reads identically to an offer nobody ever answered.
|
|
8542
|
+
*
|
|
8543
|
+
* `origin = 'received'` is load-bearing. Our OWN held frames prove nothing about them, and
|
|
8544
|
+
* counting those would make every session we ever spoke into un-reapable — which is exactly the
|
|
8545
|
+
* clutter the reaper exists to clear. D18 also depends on the zero case staying zero: reaping only
|
|
8546
|
+
* genuinely 0-received ghosts is what stops a stranger whose first handshakes died from being
|
|
8547
|
+
* locked out by the acceptance bound forever.
|
|
8548
|
+
*/
|
|
8549
|
+
countEstablishedReceived(agentName, sessionId) {
|
|
8550
|
+
if (!this.#db)
|
|
8551
|
+
return 0;
|
|
8552
|
+
const agentId = this.#requireAgentId(agentName);
|
|
8553
|
+
const held = this.#db
|
|
8554
|
+
.prepare("SELECT COUNT(*) AS n FROM held_content WHERE agent_id = ? AND session_id = ? AND origin = 'received'")
|
|
8555
|
+
.get(agentId, sessionId);
|
|
8556
|
+
return this.countReceivedMessages(agentName, sessionId) + (held?.n ?? 0);
|
|
8557
|
+
}
|
|
7431
8558
|
countReceivedMessages(agentName, sessionId) {
|
|
7432
8559
|
if (!this.#db)
|
|
7433
8560
|
return 0;
|
|
@@ -7522,6 +8649,26 @@ export class SessionNodeManager {
|
|
|
7522
8649
|
interruptedBy) {
|
|
7523
8650
|
if (!this.#db)
|
|
7524
8651
|
return false;
|
|
8652
|
+
/**
|
|
8653
|
+
* DOD-M12B-SESSION-SEED-1 (review F2) — the identity dies on terminal INTENT, not on a
|
|
8654
|
+
* successful UPDATE.
|
|
8655
|
+
*
|
|
8656
|
+
* The first build destroyed the seed only after the write landed. `#requireAgentId` THROWS for
|
|
8657
|
+
* a retired agent, so every terminal write for a revoked agent's sessions fell into the catch
|
|
8658
|
+
* and kept its transport identity for the life of the process — an identity whose agent has
|
|
8659
|
+
* just been revoked in the directory, held with nothing reporting it, and REVIVAL-BOUND-1's
|
|
8660
|
+
* sweep excludes retired agents so nothing else closed it either. The same held for a
|
|
8661
|
+
* `session.status.write.missed` and for any DB error.
|
|
8662
|
+
*
|
|
8663
|
+
* Coupling a security teardown to a database write is backwards: the write can fail, and the
|
|
8664
|
+
* failure is exactly when we least want a live key lying around. So it runs FIRST and
|
|
8665
|
+
* unconditionally, and if the write then fails the session is one we can no longer revive —
|
|
8666
|
+
* which is the safe direction, and is reported loudly below rather than inferred from the
|
|
8667
|
+
* absence of a debug line.
|
|
8668
|
+
*/
|
|
8669
|
+
if (status === "sealed" || status === "abandoned") {
|
|
8670
|
+
this.#destroySessionSeed(agentName, sessionId);
|
|
8671
|
+
}
|
|
7525
8672
|
// THE TERMINAL GUARD LIVES HERE, not in one wrapper, because there are three writers of
|
|
7526
8673
|
// "sealed": markSealed, destroySessionNode, and retireSession on the witnessed-submit path.
|
|
7527
8674
|
// Guarding only the wrapper asserts the invariant in a test while two other paths still break
|
|
@@ -7557,10 +8704,22 @@ export class SessionNodeManager {
|
|
|
7557
8704
|
const now = Date.now();
|
|
7558
8705
|
try {
|
|
7559
8706
|
const res = this.#db
|
|
7560
|
-
.prepare(
|
|
7561
|
-
|
|
7562
|
-
|
|
7563
|
-
|
|
8707
|
+
.prepare(
|
|
8708
|
+
// DOD-M12B-REVIVAL-BOUND-1: this is the FOURTH writer of `status = 'interrupted'`, and
|
|
8709
|
+
// until now the only one that wrote no `interrupted_at`. That is where Entry 41's two
|
|
8710
|
+
// timestamp-less rows came from, and a row with no timestamp has no revival bound that
|
|
8711
|
+
// can be evaluated. `COALESCE` matches the three sibling producers: the FIRST
|
|
8712
|
+
// interruption is the clock, so re-entering the status cannot push the deadline out.
|
|
8713
|
+
status === "interrupted"
|
|
8714
|
+
? (interruptedBy === undefined
|
|
8715
|
+
? "UPDATE sessions SET status = ?, updated_at = ?, interrupted_at = COALESCE(interrupted_at, ?) WHERE agent_id = ? AND session_id = ?"
|
|
8716
|
+
: "UPDATE sessions SET status = ?, updated_at = ?, interrupted_at = COALESCE(interrupted_at, ?), interrupted_by = 'local' WHERE agent_id = ? AND session_id = ?")
|
|
8717
|
+
: (interruptedBy === undefined
|
|
8718
|
+
? "UPDATE sessions SET status = ?, updated_at = ? WHERE agent_id = ? AND session_id = ?"
|
|
8719
|
+
: "UPDATE sessions SET status = ?, updated_at = ?, interrupted_by = 'local' WHERE agent_id = ? AND session_id = ?"))
|
|
8720
|
+
.run(...(status === "interrupted"
|
|
8721
|
+
? [status, now, new Date(now).toISOString(), this.#requireAgentId(agentName), sessionId]
|
|
8722
|
+
: [status, now, this.#requireAgentId(agentName), sessionId]));
|
|
7564
8723
|
// "Did not throw" is NOT "landed". An UPDATE whose WHERE matches no row — a wrong agent_id, a
|
|
7565
8724
|
// session_id with no row — succeeds silently and changes nothing. Reporting that as a written
|
|
7566
8725
|
// status flip is what let a disposition hook delete a live session's content, so the row count
|
|
@@ -7571,7 +8730,11 @@ export class SessionNodeManager {
|
|
|
7571
8730
|
sessionId,
|
|
7572
8731
|
status,
|
|
7573
8732
|
agentName,
|
|
7574
|
-
impact:
|
|
8733
|
+
impact: (status === "sealed" || status === "abandoned")
|
|
8734
|
+
? "no session row matched — the status was NOT changed and no disposition was run, AND "
|
|
8735
|
+
+ "this session's transport identity has already been destroyed, so it can no longer "
|
|
8736
|
+
+ "be revived even though its row still says it is open"
|
|
8737
|
+
: "no session row matched — the status was NOT changed and no disposition was run",
|
|
7575
8738
|
});
|
|
7576
8739
|
return false;
|
|
7577
8740
|
}
|