@cello-protocol/daemon 0.0.176 → 0.0.178
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/close-session-handler.d.ts.map +1 -1
- package/dist/close-session-handler.js +17 -7
- package/dist/close-session-handler.js.map +1 -1
- package/dist/daemon.js +40 -6
- package/dist/daemon.js.map +1 -1
- package/dist/document-layer.d.ts +5 -0
- package/dist/document-layer.d.ts.map +1 -1
- package/dist/document-layer.js +38 -13
- package/dist/document-layer.js.map +1 -1
- package/dist/document-reconcile-scheduler.d.ts +62 -23
- package/dist/document-reconcile-scheduler.d.ts.map +1 -1
- package/dist/document-reconcile-scheduler.js +103 -27
- package/dist/document-reconcile-scheduler.js.map +1 -1
- package/dist/restart-seal-resolver.d.ts +24 -6
- package/dist/restart-seal-resolver.d.ts.map +1 -1
- package/dist/restart-seal-resolver.js +56 -3
- package/dist/restart-seal-resolver.js.map +1 -1
- package/dist/session-content-handlers.d.ts.map +1 -1
- package/dist/session-content-handlers.js +45 -31
- package/dist/session-content-handlers.js.map +1 -1
- package/dist/session-node-manager.d.ts +107 -6
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +647 -59
- package/dist/session-node-manager.js.map +1 -1
- package/package.json +5 -5
|
@@ -193,6 +193,24 @@ export const REVIVAL_WINDOW_MS = 24 * 60 * 60 * 1000;
|
|
|
193
193
|
* daemon left up for a week never applies it, and a long-lived daemon is the normal case.
|
|
194
194
|
*/
|
|
195
195
|
export const REVIVAL_BOUND_SWEEP_MS = 60 * 60 * 1000;
|
|
196
|
+
/** DOD-M12B-SESSION-SEED-1: per-relay deadline when a revived node asks for a circuit reservation.
|
|
197
|
+
* Three seconds — a relay that has a slot answers well inside it, and one that does not never
|
|
198
|
+
* answers at all (measured: 10,002ms and still waiting). */
|
|
199
|
+
export const REVIVE_RESERVATION_TIMEOUT_MS = 3_000;
|
|
200
|
+
/** How many relays a revival will ask before settling for a plain node. Two: the worst case is then
|
|
201
|
+
* ~6s to a usable session, against a measured "never" for the unbounded form. */
|
|
202
|
+
export const REVIVE_RESERVATION_CANDIDATES = 2;
|
|
203
|
+
/**
|
|
204
|
+
* DOD-M12B-LEAF-TRIGGERS-FETCH-1 — how long the DIRECT path gets before we go and fetch content the
|
|
205
|
+
* relay has already told us about.
|
|
206
|
+
*
|
|
207
|
+
* Two seconds. The witness leaf and the plaintext are separate deliveries, and on a healthy session
|
|
208
|
+
* the direct content normally lands within milliseconds of the leaf — so fetching the instant a leaf
|
|
209
|
+
* arrives would put a relay round trip on the hot path of every message in every session, which is a
|
|
210
|
+
* self-inflicted load problem. Waiting forever is what cost 102 seconds. Two seconds is far above
|
|
211
|
+
* the healthy direct latency and far below anything a person would notice.
|
|
212
|
+
*/
|
|
213
|
+
export const LEAF_FETCH_GRACE_MS = 2_000;
|
|
196
214
|
export class SessionNodeManager {
|
|
197
215
|
#factory;
|
|
198
216
|
#logger;
|
|
@@ -330,6 +348,17 @@ export class SessionNodeManager {
|
|
|
330
348
|
* why restart is `RESTART-SEAL-1`'s case (resolve with a receipt) and not a revival case.
|
|
331
349
|
*/
|
|
332
350
|
#sessionSeeds = new Map();
|
|
351
|
+
/** DOD-M12B-SESSION-SEED-1: see setRetryDrainHook — fired when a session is revived. */
|
|
352
|
+
#retryDrainHook = null;
|
|
353
|
+
/** DOD-M12B-LEAF-TRIGGERS-FETCH-1: content hashes this session has actually resolved — ingested,
|
|
354
|
+
* held, or authored by us. A witnessed leaf whose hash is in here needs no fetch. */
|
|
355
|
+
#resolvedContent = new Map();
|
|
356
|
+
/** In-flight grace timers, keyed session+hash, so a redelivered leaf does not schedule a second
|
|
357
|
+
* fetch for the same content — a slow relay must not be turned into a storm against itself. */
|
|
358
|
+
#leafFetchTimers = new Map();
|
|
359
|
+
/** Test seam: collapse the grace window so a test does not have to wait two real seconds. The
|
|
360
|
+
* window itself is covered by its own case. */
|
|
361
|
+
#leafFetchGraceMs = LEAF_FETCH_GRACE_MS;
|
|
333
362
|
#standingReceivers = new Map();
|
|
334
363
|
#standingReceiverCreating = new Set();
|
|
335
364
|
// M8B F14: agents that SHOULD have a standing receiver — marked by
|
|
@@ -2359,6 +2388,60 @@ export class SessionNodeManager {
|
|
|
2359
2388
|
* and leaves relayClient undefined — the session is NOT destroyed and the direct
|
|
2360
2389
|
* content path keeps working (the relay-park/recovery path is MSG-001-3b's domain).
|
|
2361
2390
|
*/
|
|
2391
|
+
/**
|
|
2392
|
+
* DOD-M12B-REVIVE-RELAY-1 — the relay witness leaf handler, shared by establishment and revival.
|
|
2393
|
+
*
|
|
2394
|
+
* Extracted because a REVIVED session must register the same handler. It was inline in
|
|
2395
|
+
* `#connectSessionRelay`, so revival — which never called that at all — had no live inbound path:
|
|
2396
|
+
* every message fell back to the five-minute mailbox poll, which is why a reconnected session took
|
|
2397
|
+
* three minutes to deliver what a fresh one delivers in seconds, and why doorbells stopped firing.
|
|
2398
|
+
*
|
|
2399
|
+
* A revived session that behaves differently from a fresh one is the defect. This is one of the
|
|
2400
|
+
* two halves of making them the same.
|
|
2401
|
+
*/
|
|
2402
|
+
#relayLeafHandler(agentName, sessionId, correlationId) {
|
|
2403
|
+
return (frame) => {
|
|
2404
|
+
// The counterparty's witnessed leaf arrived with its canonical sequence. The
|
|
2405
|
+
// plaintext is delivered separately over the direct content stream; this is the
|
|
2406
|
+
// ordering/witness signal. Full canonical-sequence reconciliation against the
|
|
2407
|
+
// local tree is MSG-001-3b (J-CONTENT).
|
|
2408
|
+
this.#logger.info("session.relay.leaf.delivered", {
|
|
2409
|
+
sessionId,
|
|
2410
|
+
sequenceNumber: frame.sequence_number,
|
|
2411
|
+
leafKind: frame.leaf_kind,
|
|
2412
|
+
correlationId,
|
|
2413
|
+
});
|
|
2414
|
+
// DOD-MSG-4 (strict in-order): record the relay-witnessed canonical sequence for the
|
|
2415
|
+
// counterparty's MSG leaves. The relay is the ordering authority; structure1_cbor =
|
|
2416
|
+
// [1, content_hash(32), sender_pubkey, session_id, last_seen_seq, ts]. The relay sequence
|
|
2417
|
+
// is 1-based and global per session; the daemon tree is 0-based — normalize with -1. Only
|
|
2418
|
+
// COUNTERPARTY leaves (the ones B will ingest); our own echoed leaf already lands via the
|
|
2419
|
+
// send path. The gate (ingestReceivedContent) reads this map to hold out-of-order arrivals.
|
|
2420
|
+
if (!frame.authored_by_us && frame.leaf_kind !== LEAF_KIND_CTRL) {
|
|
2421
|
+
try {
|
|
2422
|
+
const s1 = decode(frame.structure1_cbor);
|
|
2423
|
+
const contentHash = s1?.[1];
|
|
2424
|
+
if (contentHash instanceof Uint8Array && frame.sequence_number > 0) {
|
|
2425
|
+
this.recordWitnessedSequence(agentName, sessionId, Buffer.from(contentHash).toString("hex"), frame.sequence_number - 1);
|
|
2426
|
+
}
|
|
2427
|
+
}
|
|
2428
|
+
catch (err) {
|
|
2429
|
+
this.#logger.warn("session.relay.leaf.witness.decode.failed", {
|
|
2430
|
+
sessionId,
|
|
2431
|
+
error: err instanceof Error ? err.message : String(err),
|
|
2432
|
+
correlationId,
|
|
2433
|
+
});
|
|
2434
|
+
}
|
|
2435
|
+
}
|
|
2436
|
+
// M7-UPGRADE-002: auto-acknowledge close. When the COUNTERPARTY's SEAL ctrl leaf (0x02)
|
|
2437
|
+
// arrives and B has verified the content, B's OWN node auto-co-signs the responder SEAL
|
|
2438
|
+
// leaf — no agent prompt — so the bilateral seal completes promptly instead of degrading
|
|
2439
|
+
// to unilateral on a slow/busy/crashed agent. Never auto-ack our OWN echoed ctrl leaf.
|
|
2440
|
+
if (frame.leaf_kind === LEAF_KIND_CTRL && !frame.authored_by_us) {
|
|
2441
|
+
this.#maybeAutoAcknowledgeSeal(agentName, sessionId, correlationId);
|
|
2442
|
+
}
|
|
2443
|
+
};
|
|
2444
|
+
}
|
|
2362
2445
|
async #connectSessionRelay(sessionId, node, agentName, relay, correlationId) {
|
|
2363
2446
|
try {
|
|
2364
2447
|
// The session node's gater admits only the counterparty; the relay witness is a
|
|
@@ -2392,47 +2475,7 @@ export class SessionNodeManager {
|
|
|
2392
2475
|
this.#relayClients.set(clientKey, client);
|
|
2393
2476
|
}
|
|
2394
2477
|
const sessionIdHexForRelay = Buffer.from(relay.sessionIdBytes).toString("hex");
|
|
2395
|
-
client.registerSession(sessionIdHexForRelay, node, (
|
|
2396
|
-
// The counterparty's witnessed leaf arrived with its canonical sequence. The
|
|
2397
|
-
// plaintext is delivered separately over the direct content stream; this is the
|
|
2398
|
-
// ordering/witness signal. Full canonical-sequence reconciliation against the
|
|
2399
|
-
// local tree is MSG-001-3b (J-CONTENT).
|
|
2400
|
-
this.#logger.info("session.relay.leaf.delivered", {
|
|
2401
|
-
sessionId,
|
|
2402
|
-
sequenceNumber: frame.sequence_number,
|
|
2403
|
-
leafKind: frame.leaf_kind,
|
|
2404
|
-
correlationId,
|
|
2405
|
-
});
|
|
2406
|
-
// DOD-MSG-4 (strict in-order): record the relay-witnessed canonical sequence for the
|
|
2407
|
-
// counterparty's MSG leaves. The relay is the ordering authority; structure1_cbor =
|
|
2408
|
-
// [1, content_hash(32), sender_pubkey, session_id, last_seen_seq, ts]. The relay sequence
|
|
2409
|
-
// is 1-based and global per session; the daemon tree is 0-based — normalize with -1. Only
|
|
2410
|
-
// COUNTERPARTY leaves (the ones B will ingest); our own echoed leaf already lands via the
|
|
2411
|
-
// send path. The gate (ingestReceivedContent) reads this map to hold out-of-order arrivals.
|
|
2412
|
-
if (!frame.authored_by_us && frame.leaf_kind !== LEAF_KIND_CTRL) {
|
|
2413
|
-
try {
|
|
2414
|
-
const s1 = decode(frame.structure1_cbor);
|
|
2415
|
-
const contentHash = s1?.[1];
|
|
2416
|
-
if (contentHash instanceof Uint8Array && frame.sequence_number > 0) {
|
|
2417
|
-
this.recordWitnessedSequence(agentName, sessionId, Buffer.from(contentHash).toString("hex"), frame.sequence_number - 1);
|
|
2418
|
-
}
|
|
2419
|
-
}
|
|
2420
|
-
catch (err) {
|
|
2421
|
-
this.#logger.warn("session.relay.leaf.witness.decode.failed", {
|
|
2422
|
-
sessionId,
|
|
2423
|
-
error: err instanceof Error ? err.message : String(err),
|
|
2424
|
-
correlationId,
|
|
2425
|
-
});
|
|
2426
|
-
}
|
|
2427
|
-
}
|
|
2428
|
-
// M7-UPGRADE-002: auto-acknowledge close. When the COUNTERPARTY's SEAL ctrl leaf (0x02)
|
|
2429
|
-
// arrives and B has verified the content, B's OWN node auto-co-signs the responder SEAL
|
|
2430
|
-
// leaf — no agent prompt — so the bilateral seal completes promptly instead of degrading
|
|
2431
|
-
// to unilateral on a slow/busy/crashed agent. Never auto-ack our OWN echoed ctrl leaf.
|
|
2432
|
-
if (frame.leaf_kind === LEAF_KIND_CTRL && !frame.authored_by_us) {
|
|
2433
|
-
this.#maybeAutoAcknowledgeSeal(agentName, sessionId, correlationId);
|
|
2434
|
-
}
|
|
2435
|
-
}, relay.assignment);
|
|
2478
|
+
client.registerSession(sessionIdHexForRelay, node, this.#relayLeafHandler(agentName, sessionId, correlationId), relay.assignment);
|
|
2436
2479
|
const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
|
|
2437
2480
|
if (entry) {
|
|
2438
2481
|
entry.relayClient = client;
|
|
@@ -2531,6 +2574,26 @@ export class SessionNodeManager {
|
|
|
2531
2574
|
node.onPeerConnect((peerId) => {
|
|
2532
2575
|
if (!isCounterparty(peerId))
|
|
2533
2576
|
return;
|
|
2577
|
+
/**
|
|
2578
|
+
* DOD-M12B-RESPONDER-ADDR-1 (review MEDIUM-4) — LEARN THE ADDRESS HERE, where it cannot race.
|
|
2579
|
+
*
|
|
2580
|
+
* The accept-time read was a race between two independent async chains: the responder accepts
|
|
2581
|
+
* off a signaling frame, while the initiator dials only after its own `createSessionNode`. If
|
|
2582
|
+
* accept looked before the dial landed it saw nothing, and the responder was back to holding
|
|
2583
|
+
* no address — the state that made every reply after an interruption park forever.
|
|
2584
|
+
*
|
|
2585
|
+
* This fires exactly when the counterparty connects: on both sides, on the first connection,
|
|
2586
|
+
* on every reconnect, and on a revived node too. It also REFRESHES, which the accept-time read
|
|
2587
|
+
* never did — a counterparty that rebuilds its receiver would otherwise leave us dialling a
|
|
2588
|
+
* dead address for the life of the session.
|
|
2589
|
+
*/
|
|
2590
|
+
const observed = node
|
|
2591
|
+
.getConnections()
|
|
2592
|
+
.filter((c) => c.peerId === peerId && typeof c.remoteAddr === "string")
|
|
2593
|
+
.map((c) => c.remoteAddr);
|
|
2594
|
+
if (observed.length > 0) {
|
|
2595
|
+
this.#counterpartyAddrs.set(key, [...new Set(observed)]);
|
|
2596
|
+
}
|
|
2534
2597
|
const prior = this.#sessionLiveness.get(key);
|
|
2535
2598
|
this.#sessionLiveness.set(key, "alive");
|
|
2536
2599
|
if (prior !== "alive") {
|
|
@@ -2849,6 +2912,51 @@ export class SessionNodeManager {
|
|
|
2849
2912
|
// DAEMON-004: register the content stream handler for the inbound session.
|
|
2850
2913
|
await this.#registerContentHandler(agentName, sessionId, node, counterpartyPubkey);
|
|
2851
2914
|
// M7-SESSION-003 AC-004: act on the inbound session node's peer events too.
|
|
2915
|
+
/**
|
|
2916
|
+
* DOD-M12B-RESPONDER-ADDR-1 — LEARN THE INITIATOR'S ADDRESS, because we will need it and this is
|
|
2917
|
+
* the only moment we have it.
|
|
2918
|
+
*
|
|
2919
|
+
* MEASURED LIVE 2026-08-18. After an interruption the responder's re-dial reported
|
|
2920
|
+
* `session.transport.redial.unavailable` — *"this side holds no address for the counterparty, so
|
|
2921
|
+
* every send parks until they re-establish"* — and every reply it tried to send failed. The
|
|
2922
|
+
* initiator can always come back because it kept the addresses it dialled; the responder dialled
|
|
2923
|
+
* nothing, so it kept nothing.
|
|
2924
|
+
*
|
|
2925
|
+
* In plain terms that meant: whoever ANSWERED a conversation could not restart it. Their replies
|
|
2926
|
+
* went nowhere until the other side spoke first.
|
|
2927
|
+
*
|
|
2928
|
+
* The live connection has known the address all along — the responder is holding it right now,
|
|
2929
|
+
* because the initiator just dialled in on it. `#counterpartyAddrs` is the same store the
|
|
2930
|
+
* initiator fills from its signed relay assignment, and `#evictSessionCaches` hands both to the
|
|
2931
|
+
* revival record on the way down, so this needs no separate lifetime.
|
|
2932
|
+
*/
|
|
2933
|
+
const inboundAddrs = node
|
|
2934
|
+
.getConnections()
|
|
2935
|
+
.filter((c) => c.peerId === initiatorPeerId && typeof c.remoteAddr === "string")
|
|
2936
|
+
.map((c) => c.remoteAddr);
|
|
2937
|
+
if (inboundAddrs.length > 0) {
|
|
2938
|
+
this.#counterpartyAddrs.set(this.#k(agentName, sessionId), [...new Set(inboundAddrs)]);
|
|
2939
|
+
this.#logger.info("session.counterparty.addr.learned", {
|
|
2940
|
+
agentName,
|
|
2941
|
+
sessionId,
|
|
2942
|
+
addrs: inboundAddrs.length,
|
|
2943
|
+
source: "inbound_connection",
|
|
2944
|
+
impact: "this side can now re-dial after an interruption instead of parking every reply",
|
|
2945
|
+
});
|
|
2946
|
+
}
|
|
2947
|
+
else {
|
|
2948
|
+
// NOT A WARNING. Review MEDIUM-4: accept runs off a signaling frame and the initiator dials
|
|
2949
|
+
// separately, so "no connection yet" is the ordinary in-flight case — warning on it puts a
|
|
2950
|
+
// signal on the normal path, which is how the one occurrence that matters gets buried. The
|
|
2951
|
+
// race-free capture is in `#wireSessionLiveness`'s onPeerConnect, which fires when the dial
|
|
2952
|
+
// actually lands; this read is only a fast path for when it already has.
|
|
2953
|
+
this.#logger.debug("session.counterparty.addr.deferred", {
|
|
2954
|
+
agentName,
|
|
2955
|
+
sessionId,
|
|
2956
|
+
initiatorPeerId,
|
|
2957
|
+
impact: "no connection observed yet; the address is captured when the counterparty connects",
|
|
2958
|
+
});
|
|
2959
|
+
}
|
|
2852
2960
|
this.#wireSessionLiveness(agentName, sessionId, node, counterpartyPubkey, correlationId, initiatorPeerId);
|
|
2853
2961
|
// M7 DOD-SPINE-6 / MSG-001-3b: the receiver also connects to the relay witness so
|
|
2854
2962
|
// the relay can deliver the initiator's witnessed leaves (leaf_deliver) to it.
|
|
@@ -3082,6 +3190,16 @@ export class SessionNodeManager {
|
|
|
3082
3190
|
this.#heldRestored.delete(key);
|
|
3083
3191
|
this.#heldReleased.delete(key);
|
|
3084
3192
|
this.#diverged.delete(key);
|
|
3193
|
+
// DOD-M12B-SESSION-SEED-1: HAND THEM TO THE REVIVAL RECORD BEFORE DROPPING THEM. This eviction
|
|
3194
|
+
// runs on every teardown, including the interruption a revival is meant to undo — so clearing
|
|
3195
|
+
// the addresses here is what left a revived session unable to dial anyone. The revival record
|
|
3196
|
+
// has exactly the right lifetime for them: it dies when the session reaches a terminal status.
|
|
3197
|
+
const survivingAddrs = this.#counterpartyAddrs.get(key);
|
|
3198
|
+
if (survivingAddrs && survivingAddrs.length > 0) {
|
|
3199
|
+
const identity = this.#sessionSeeds.get(key);
|
|
3200
|
+
if (identity)
|
|
3201
|
+
identity.counterpartyAddrs = [...survivingAddrs];
|
|
3202
|
+
}
|
|
3085
3203
|
this.#counterpartyAddrs.delete(key);
|
|
3086
3204
|
this.#redialNotBefore.delete(key);
|
|
3087
3205
|
this.#highWaterSeq.delete(key);
|
|
@@ -3272,9 +3390,38 @@ export class SessionNodeManager {
|
|
|
3272
3390
|
.all(status);
|
|
3273
3391
|
}
|
|
3274
3392
|
/**
|
|
3275
|
-
* DOD-M12B-RESTART-SEAL-1 —
|
|
3276
|
-
*
|
|
3277
|
-
*
|
|
3393
|
+
* DOD-M12B-RESTART-SEAL-1 / DOD-M12B-PENDING-RESOLVE-1 — sessions that need a receipt and have
|
|
3394
|
+
* nobody asking for one. TWO populations, one queue, each with its own safety argument.
|
|
3395
|
+
*
|
|
3396
|
+
* **(2) `seal_interrupted_pending` — a seal commitment nobody notarized.** Measured 2026-08-18 on
|
|
3397
|
+
* the live store: 28 sessions, aged 0.3 to 12.8 days, one of them 14 messages long, 26 holding
|
|
3398
|
+
* relay-witnessed seal leaves, and **not one with a sealed root**.
|
|
3399
|
+
*
|
|
3400
|
+
* **HALF OF THEM ARE NOT BILATERAL, and the first version of this comment claimed they were.**
|
|
3401
|
+
* Measured split: 14 initiator rows, each carrying the counterparty's signed leaf — and 14
|
|
3402
|
+
* responder rows with `counterparty_leaf = NULL`. A responder row is written by
|
|
3403
|
+
* `inbound-seal-request.ts` from an UNSIGNED `seal_interrupted_request` frame, before its ack is
|
|
3404
|
+
* even sent, so an ordinary send failure produces a one-sided pending row.
|
|
3405
|
+
*
|
|
3406
|
+
* So the licence is NOT "both parties signed". It is **somebody chose to end this**, on two
|
|
3407
|
+
* branches: an initiator row carries the counterparty's signature, and a responder row exists
|
|
3408
|
+
* because the counterparty sent a request to seal. And what makes the result VERIFIABLE is
|
|
3409
|
+
* neither — it is that the directory rebuilds the tree from relay-witnessed leaves and checks
|
|
3410
|
+
* their signatures, never consulting the commitment at all. The commitment is what makes it
|
|
3411
|
+
* legitimate to ASK. (`close-session-handler.ts` states this in full; the first draft of this
|
|
3412
|
+
* header contradicted it 200 lines away.) `PENDING-EXIT-1` built their exit and it works — but only when an operator runs
|
|
3413
|
+
* `cello_close_session` on that session by hand, having somehow deduced they should. Nothing
|
|
3414
|
+
* enumerated them, because both sweeps filtered `status = 'interrupted'`. An exit nobody is told
|
|
3415
|
+
* about is not an exit.
|
|
3416
|
+
*
|
|
3417
|
+
* `interrupted_by` is deliberately NOT consulted for that population. It answers "did WE cause
|
|
3418
|
+
* this, and may we therefore describe it" — and that is the wrong question once a seal was
|
|
3419
|
+
* requested or signed. **SI-001 is not weakened:** it forbids notarizing *"a conversation nobody
|
|
3420
|
+
* chose to end"*, and every row here was chosen to be ended by one side or the other. The only
|
|
3421
|
+
* thing missing is the request to notarize it.
|
|
3422
|
+
*
|
|
3423
|
+
* **(1) `interrupted` — sessions our own stop orphaned, and only those.** Here `interrupted_by` is
|
|
3424
|
+
* the whole safety argument. `'local'` means the boot sweep, the shutdown
|
|
3278
3425
|
* sweep, or the operator's own kill switch ended this session — nobody else did, and it cannot be
|
|
3279
3426
|
* resumed because the transport keypairs died with the process. Those are the ones the resolver
|
|
3280
3427
|
* may seal on its own.
|
|
@@ -3303,9 +3450,23 @@ export class SessionNodeManager {
|
|
|
3303
3450
|
// `restart_seal_gave_up_at IS NULL` — a session we have already exhausted. Without it a
|
|
3304
3451
|
// machine restarting ~6 times a day re-runs five ceremonies against a hopeless session on
|
|
3305
3452
|
// every boot, forever.
|
|
3306
|
-
`SELECT s.session_id AS session_id, s.message_count AS message_count, a.agent_name AS agent_name
|
|
3453
|
+
`SELECT s.session_id AS session_id, s.message_count AS message_count, a.agent_name AS agent_name,
|
|
3454
|
+
s.status AS status
|
|
3307
3455
|
FROM sessions s JOIN agents a ON a.agent_id = s.agent_id
|
|
3308
|
-
WHERE
|
|
3456
|
+
WHERE a.state != 'retired'
|
|
3457
|
+
AND (
|
|
3458
|
+
-- (1) OURS, and we can say so. SI-001 holds: an interrupted session with an
|
|
3459
|
+
-- unknown cause has no signatures behind it and must not be notarized.
|
|
3460
|
+
(s.status = 'interrupted' AND s.interrupted_by = 'local')
|
|
3461
|
+
-- (2) A seal commitment with nobody asking for it. review F5: the EXISTS is
|
|
3462
|
+
-- STRUCTURAL, not decoration. The header's licence is "a commitment was made", and
|
|
3463
|
+
-- a status check alone asserts that in prose while the query checks something
|
|
3464
|
+
-- else. Today status implies an artifact row by construction, which is exactly the
|
|
3465
|
+
-- kind of invariant that holds until someone adds a fourth writer.
|
|
3466
|
+
OR (s.status = 'seal_interrupted_pending'
|
|
3467
|
+
AND EXISTS (SELECT 1 FROM seal_interrupted_artifacts sa
|
|
3468
|
+
WHERE sa.agent_id = s.agent_id AND sa.session_id = s.session_id))
|
|
3469
|
+
)
|
|
3309
3470
|
AND s.message_count > 0
|
|
3310
3471
|
AND s.restart_seal_gave_up_at IS NULL
|
|
3311
3472
|
ORDER BY s.updated_at ASC`)
|
|
@@ -3314,6 +3475,9 @@ export class SessionNodeManager {
|
|
|
3314
3475
|
agentName: r.agent_name,
|
|
3315
3476
|
sessionId: r.session_id,
|
|
3316
3477
|
messageCount: r.message_count ?? 0,
|
|
3478
|
+
// Carried so a give-up can say something TRUE about this session: the two populations need
|
|
3479
|
+
// different words, and force-abandon is right for one and destructive for the other.
|
|
3480
|
+
status: r.status === "seal_interrupted_pending" ? "seal_interrupted_pending" : "interrupted",
|
|
3317
3481
|
}));
|
|
3318
3482
|
}
|
|
3319
3483
|
/**
|
|
@@ -5286,6 +5450,73 @@ export class SessionNodeManager {
|
|
|
5286
5450
|
const hw = this.#highWaterSeq.get(key) ?? -1;
|
|
5287
5451
|
if (sequenceNumber > hw)
|
|
5288
5452
|
this.#highWaterSeq.set(key, sequenceNumber);
|
|
5453
|
+
/**
|
|
5454
|
+
* DOD-M12B-LEAF-TRIGGERS-FETCH-1 — A LEAF WE CANNOT READ IS A FETCH ORDER.
|
|
5455
|
+
*
|
|
5456
|
+
* MEASURED LIVE 2026-08-18: the relay delivered this leaf one second after the counterparty
|
|
5457
|
+
* sent. We had the hash and the sequence, the bytes were parked at that same relay, and the
|
|
5458
|
+
* plaintext arrived 102 seconds later on a background sweep. Nothing connected the two facts —
|
|
5459
|
+
* this method recorded the sequence and stopped.
|
|
5460
|
+
*
|
|
5461
|
+
* The witness leaf and the plaintext are separate deliveries: the leaf comes over the relay, the
|
|
5462
|
+
* bytes over the direct content stream. After an interruption the two session nodes have no
|
|
5463
|
+
* direct connection, so the bytes go to the park instead and only a timer ever finds them.
|
|
5464
|
+
*
|
|
5465
|
+
* The grace window is what keeps this off the hot path. On a healthy session the direct content
|
|
5466
|
+
* lands within milliseconds of its leaf, so fetching immediately would mean a relay round trip
|
|
5467
|
+
* for every message in every session. We give the direct path its two seconds first.
|
|
5468
|
+
*/
|
|
5469
|
+
this.#scheduleLeafFetchIfUnresolved(agentName, sessionId, contentHashHex);
|
|
5470
|
+
}
|
|
5471
|
+
/** DOD-M12B-LEAF-TRIGGERS-FETCH-1: this content is here — no fetch is owed for it, and any
|
|
5472
|
+
* pending one is cancelled. Called wherever content actually lands. */
|
|
5473
|
+
#markContentResolved(agentName, sessionId, contentHashHex) {
|
|
5474
|
+
const key = this.#k(agentName, sessionId);
|
|
5475
|
+
let set = this.#resolvedContent.get(key);
|
|
5476
|
+
if (!set) {
|
|
5477
|
+
set = new Set();
|
|
5478
|
+
this.#resolvedContent.set(key, set);
|
|
5479
|
+
}
|
|
5480
|
+
set.add(contentHashHex);
|
|
5481
|
+
const timerKey = `${key}::${contentHashHex}`;
|
|
5482
|
+
const t = this.#leafFetchTimers.get(timerKey);
|
|
5483
|
+
if (t !== undefined) {
|
|
5484
|
+
clearTimeout(t);
|
|
5485
|
+
this.#leafFetchTimers.delete(timerKey);
|
|
5486
|
+
}
|
|
5487
|
+
}
|
|
5488
|
+
#scheduleLeafFetchIfUnresolved(agentName, sessionId, contentHashHex) {
|
|
5489
|
+
const key = this.#k(agentName, sessionId);
|
|
5490
|
+
if (this.#resolvedContent.get(key)?.has(contentHashHex))
|
|
5491
|
+
return;
|
|
5492
|
+
const timerKey = `${key}::${contentHashHex}`;
|
|
5493
|
+
// ONE fetch per content hash. The relay redelivers, and a redelivery carries the same sequence —
|
|
5494
|
+
// scheduling per redelivery turns a slow relay into a storm against itself.
|
|
5495
|
+
if (this.#leafFetchTimers.has(timerKey))
|
|
5496
|
+
return;
|
|
5497
|
+
const timer = setTimeout(() => {
|
|
5498
|
+
this.#leafFetchTimers.delete(timerKey);
|
|
5499
|
+
if (this.#resolvedContent.get(key)?.has(contentHashHex))
|
|
5500
|
+
return; // the direct path won
|
|
5501
|
+
if (this.#shuttingDown)
|
|
5502
|
+
return;
|
|
5503
|
+
this.#logger.info("session.content.leaf_unresolved.fetch", {
|
|
5504
|
+
agentName,
|
|
5505
|
+
sessionId,
|
|
5506
|
+
contentHash: contentHashHex,
|
|
5507
|
+
graceMs: this.#leafFetchGraceMs,
|
|
5508
|
+
impact: "the relay told us this message exists and its plaintext never arrived directly — "
|
|
5509
|
+
+ "fetching it now instead of waiting for the periodic sweep",
|
|
5510
|
+
});
|
|
5511
|
+
this.#fireParkedDrain(agentName, "witnessed_leaf_unresolved");
|
|
5512
|
+
}, this.#leafFetchGraceMs);
|
|
5513
|
+
timer.unref?.();
|
|
5514
|
+
this.#leafFetchTimers.set(timerKey, timer);
|
|
5515
|
+
}
|
|
5516
|
+
/** DOD-M12B-LEAF-TRIGGERS-FETCH-1 test seams. */
|
|
5517
|
+
setLeafFetchGraceMsForTest(ms) { this.#leafFetchGraceMs = ms; }
|
|
5518
|
+
markContentPresentForTest(agentName, sessionId, contentHashHex) {
|
|
5519
|
+
this.#markContentResolved(agentName, sessionId, contentHashHex);
|
|
5289
5520
|
}
|
|
5290
5521
|
/**
|
|
5291
5522
|
* DOD-MSG-4: the relay's high-water canonical sequence for this session (largest witnessed leaf),
|
|
@@ -5625,6 +5856,11 @@ export class SessionNodeManager {
|
|
|
5625
5856
|
// transcript. A later replay of the same hash is still caught by the dedup leaf-scan, which is
|
|
5626
5857
|
// independent of the witness map.
|
|
5627
5858
|
this.#witnessedSeq.get(recvKey)?.delete(contentHashHex);
|
|
5859
|
+
// DOD-M12B-LEAF-TRIGGERS-FETCH-1: the bytes are here, so cancel any fetch the witness leaf
|
|
5860
|
+
// scheduled. On a healthy session this is the branch that runs — the direct path beats the
|
|
5861
|
+
// grace window and the relay is never asked, which is what keeps a fetch off the hot path of
|
|
5862
|
+
// every message.
|
|
5863
|
+
this.#markContentResolved(agentName, sessionId, contentHashHex);
|
|
5628
5864
|
let buf = this.#receivedContent.get(recvKey);
|
|
5629
5865
|
if (!buf) {
|
|
5630
5866
|
buf = [];
|
|
@@ -7699,7 +7935,10 @@ export class SessionNodeManager {
|
|
|
7699
7935
|
// overwrite that dropped a live seed un-zeroed would leave the one copy we are responsible for
|
|
7700
7936
|
// in the heap with nothing tracking it.
|
|
7701
7937
|
this.#sessionSeeds.get(key)?.seed.fill(0);
|
|
7702
|
-
|
|
7938
|
+
// `counterpartyAddrs` starts empty: at creation the signed assignment has not necessarily
|
|
7939
|
+
// arrived yet. It is filled by `#evictSessionCaches` on the way down, which is the last moment
|
|
7940
|
+
// the live addresses exist.
|
|
7941
|
+
this.#sessionSeeds.set(key, { seed, counterpartyPeerId, counterpartyPubkey, counterpartyAddrs: [] });
|
|
7703
7942
|
}
|
|
7704
7943
|
/**
|
|
7705
7944
|
* DOD-M12B-SESSION-SEED-1 — destroy a session's transport identity.
|
|
@@ -7726,6 +7965,213 @@ export class SessionNodeManager {
|
|
|
7726
7965
|
this.#sessionSeeds.delete(key);
|
|
7727
7966
|
this.#logger.debug("session.seed.destroyed", { agentName, sessionId });
|
|
7728
7967
|
}
|
|
7968
|
+
/**
|
|
7969
|
+
* DOD-M12B-SESSION-SEED-1 — build a revived session node that is REACHABLE, without ever hanging.
|
|
7970
|
+
*
|
|
7971
|
+
* MEASURED 2026-08-18, live, three ways:
|
|
7972
|
+
* - handed 2 relay addrs at once, no deadline: `start()` never completes (10,002ms and counting)
|
|
7973
|
+
* - handed none: `start()` in 1ms, but NOBODY can dial the node —
|
|
7974
|
+
* the counterparty's re-dial fails
|
|
7975
|
+
* `counterparty_dial_failed` and every message in
|
|
7976
|
+
* both directions has to go the relay park route
|
|
7977
|
+
* - this: one candidate at a time, each raced against its
|
|
7978
|
+
* own deadline, plain node as the floor
|
|
7979
|
+
*
|
|
7980
|
+
* The middle option is what shipped for one test run and it made the session half-dead: revived,
|
|
7981
|
+
* `active`, and unreachable. The first is what shipped before that and it hung. Neither is a
|
|
7982
|
+
* choice between "fast" and "reliable" — the per-candidate race is how `#startReceiverNode` has
|
|
7983
|
+
* always done it, and it is the shape that works in production every day.
|
|
7984
|
+
*
|
|
7985
|
+
* A FAILED CANDIDATE IS TORN DOWN AT SETTLEMENT. The first version awaited `stop()` immediately
|
|
7986
|
+
* and claimed that made seed reuse safe; it did not — `libp2p.stop()` returns at once unless the
|
|
7987
|
+
* node is `'started'`, and during the timeout window it is `'starting'` (review HIGH-3, verified
|
|
7988
|
+
* against libp2p 3.3.2). The teardown is now chained onto the candidate's OWN start promise, so it
|
|
7989
|
+
* runs whenever that settles, however late.
|
|
7990
|
+
*
|
|
7991
|
+
* A BRIEF OVERLAP IS THEREFORE POSSIBLE and is stated rather than denied: a candidate that grants
|
|
7992
|
+
* at 4s comes up on this session's peer id and is stopped immediately after. What is guaranteed is
|
|
7993
|
+
* that it dies, not that it never lives. The receiver path avoids even that by minting a seed per
|
|
7994
|
+
* candidate; here the identity is fixed, which is the whole point of a revival, so that option
|
|
7995
|
+
* does not exist.
|
|
7996
|
+
*
|
|
7997
|
+
* The floor is a plain node: a session that is usable over the relay park route beats no session.
|
|
7998
|
+
*/
|
|
7999
|
+
async #buildRevivedNode(sessionId, gater, seed, candidateAddrs, agentName) {
|
|
8000
|
+
for (const circuitAddr of candidateAddrs.slice(0, REVIVE_RESERVATION_CANDIDATES)) {
|
|
8001
|
+
const candidate = await this.#factory.createNode({
|
|
8002
|
+
sessionId,
|
|
8003
|
+
connectionGater: gater,
|
|
8004
|
+
nodeType: "session",
|
|
8005
|
+
inboundReachable: true,
|
|
8006
|
+
transportPrivateKey: seed,
|
|
8007
|
+
circuitRelayListenAddrs: [circuitAddr],
|
|
8008
|
+
});
|
|
8009
|
+
// KEEP THE START PROMISE. Review HIGH-3: `libp2p.stop()` opens with
|
|
8010
|
+
// `if (this.status !== 'started') return`, and during the whole timeout window the status is
|
|
8011
|
+
// `'starting'` — so awaiting `stop()` on a timed-out candidate stopped nothing and waited for
|
|
8012
|
+
// nothing. The abandoned `start()` stayed in flight, and if the relay answered late the node
|
|
8013
|
+
// went live holding THIS SESSION'S peer id, sharing the gater (so it admits the counterparty)
|
|
8014
|
+
// with no content handler registered, and with no reference left to stop it. Verified against
|
|
8015
|
+
// libp2p 3.3.2 rather than assumed.
|
|
8016
|
+
const startP = candidate.start();
|
|
8017
|
+
let startError;
|
|
8018
|
+
const started = await Promise.race([
|
|
8019
|
+
startP.then(() => true),
|
|
8020
|
+
new Promise((res) => setTimeout(() => res(false), REVIVE_RESERVATION_TIMEOUT_MS).unref?.()),
|
|
8021
|
+
]).catch((err) => { startError = err; return false; });
|
|
8022
|
+
if (started && candidate.listenAddresses().some((a) => a.includes("/p2p-circuit"))) {
|
|
8023
|
+
this.#logger.info("session.revive.reservation.granted", { agentName, sessionId });
|
|
8024
|
+
return candidate;
|
|
8025
|
+
}
|
|
8026
|
+
// Started but granted nothing, or never started. Either way this node is not the one.
|
|
8027
|
+
//
|
|
8028
|
+
// Review MEDIUM-5: name WHICH of the three causes this was, the way `#startReceiverNode` does.
|
|
8029
|
+
// "declined" alone stood for a relay that is full, a relay that is unreachable, and a relay
|
|
8030
|
+
// that is merely slow — three different problems with three different responses, and the
|
|
8031
|
+
// thrown error was discarded entirely.
|
|
8032
|
+
const declineReason = started
|
|
8033
|
+
? "relay_granted_no_reservation"
|
|
8034
|
+
: startError !== undefined
|
|
8035
|
+
? "relay_unreachable"
|
|
8036
|
+
: "reservation_did_not_complete_in_time";
|
|
8037
|
+
const isLast = circuitAddr === candidateAddrs.slice(0, REVIVE_RESERVATION_CANDIDATES).at(-1);
|
|
8038
|
+
this.#logger.warn("session.revive.reservation.declined", {
|
|
8039
|
+
agentName,
|
|
8040
|
+
sessionId,
|
|
8041
|
+
circuitAddr,
|
|
8042
|
+
reason: declineReason,
|
|
8043
|
+
...(startError !== undefined ? { error: extractErrorMessage(startError) } : {}),
|
|
8044
|
+
impact: isLast
|
|
8045
|
+
? "no relay granted; the session comes up reachable only via the relay park route"
|
|
8046
|
+
: "trying the next relay",
|
|
8047
|
+
});
|
|
8048
|
+
// Teardown at SETTLEMENT, not now: a `stop()` issued while the node is still starting is a
|
|
8049
|
+
// no-op (see above), so the only way to guarantee this node dies is to wait for its own start
|
|
8050
|
+
// to finish first. Not awaited, so a hung start cannot hold the revival up — the point is that
|
|
8051
|
+
// the teardown eventually happens, not that it happens before the next candidate.
|
|
8052
|
+
void startP.then(() => candidate.stop().catch(() => { }), () => { });
|
|
8053
|
+
}
|
|
8054
|
+
// THE FLOOR. No reservation, so the counterparty cannot dial us directly — but their messages
|
|
8055
|
+
// park at the relay and drain, which is how every message in the 2026-08-18 test arrived. A
|
|
8056
|
+
// session usable one way beats a session that never comes back.
|
|
8057
|
+
const plain = await this.#factory.createNode({
|
|
8058
|
+
sessionId,
|
|
8059
|
+
connectionGater: gater,
|
|
8060
|
+
nodeType: "session",
|
|
8061
|
+
inboundReachable: true,
|
|
8062
|
+
transportPrivateKey: seed,
|
|
8063
|
+
});
|
|
8064
|
+
await plain.start();
|
|
8065
|
+
if (candidateAddrs.length > 0) {
|
|
8066
|
+
this.#logger.warn("session.revive.reservation.none", {
|
|
8067
|
+
agentName,
|
|
8068
|
+
sessionId,
|
|
8069
|
+
candidates: candidateAddrs.length,
|
|
8070
|
+
impact: "the revived session holds no circuit address — the counterparty cannot dial it, so "
|
|
8071
|
+
+ "delivery in both directions depends on relay store-and-forward until it is rebuilt",
|
|
8072
|
+
});
|
|
8073
|
+
}
|
|
8074
|
+
return plain;
|
|
8075
|
+
}
|
|
8076
|
+
/**
|
|
8077
|
+
* DOD-M12B-REVIVE-RELAY-1 — reconnect the session's relay WITNESS, which revival never did.
|
|
8078
|
+
*
|
|
8079
|
+
* THE FIRST-PRINCIPLES DEFECT, and the one that explains every symptom chased separately before
|
|
8080
|
+
* it. Establishment does five things: build the node, register the content handler, wire liveness,
|
|
8081
|
+
* **connect the relay**, and dial the counterparty. Revival did the first three. A revived session
|
|
8082
|
+
* was therefore not a session — it looked live, reported `active`, and had no live inbound path at
|
|
8083
|
+
* all.
|
|
8084
|
+
*
|
|
8085
|
+
* MEASURED 2026-08-18 with two real agents: a message on a reconnected session took **three
|
|
8086
|
+
* minutes**, against seconds on a fresh one, because only the five-minute mailbox backstop ever
|
|
8087
|
+
* found it. Doorbells stopped firing for the same reason — the relay stream is what rings them.
|
|
8088
|
+
* And `#parkContent` refuses without `entry.relayClient`, so sends could not park either.
|
|
8089
|
+
*
|
|
8090
|
+
* NO ASSIGNMENT IS PRESENTED, and that is by design rather than omission: `RelayConnectParams`
|
|
8091
|
+
* documents the reconnect mode itself — *"absent … on the restart/persisted reconnect path (the
|
|
8092
|
+
* relay already recorded the session at first establishment) — the client then just reconnects
|
|
8093
|
+
* without re-recording."* A revival is exactly that path.
|
|
8094
|
+
*
|
|
8095
|
+
* Best-effort and non-fatal: a session that comes back without its witness is still better than
|
|
8096
|
+
* one that does not come back, and the failure is named rather than silent.
|
|
8097
|
+
*/
|
|
8098
|
+
async #reconnectRevivedSessionRelay(agentName, sessionId, node, gater, correlationId, ep) {
|
|
8099
|
+
if (!ep) {
|
|
8100
|
+
this.#logger.warn("session.revive.relay.absent", {
|
|
8101
|
+
agentName,
|
|
8102
|
+
sessionId,
|
|
8103
|
+
impact: "no relay is recorded for this session, so it comes back with no live inbound path — "
|
|
8104
|
+
+ "messages arrive only on the periodic mailbox poll, and a failed send cannot park and is "
|
|
8105
|
+
+ "reported lost",
|
|
8106
|
+
});
|
|
8107
|
+
return;
|
|
8108
|
+
}
|
|
8109
|
+
try {
|
|
8110
|
+
// The gater admits only the counterparty inbound; the relay is a third peer and must be
|
|
8111
|
+
// permitted OUTBOUND or our own gate refuses the dial (INV-5 keeps inbound counterparty-only).
|
|
8112
|
+
gater.setAllowedOutboundPeer(ep.relayPeerId);
|
|
8113
|
+
const clientKey = `${agentName}::${ep.relayPeerId}`;
|
|
8114
|
+
let client = this.#relayClients.get(clientKey);
|
|
8115
|
+
if (!client) {
|
|
8116
|
+
if (!this.#relayReceiptStore && this.#db)
|
|
8117
|
+
this.#relayReceiptStore = new RelayReceiptStore(this.#db, this.#logger);
|
|
8118
|
+
if (!this.#sealLeafStore && this.#db)
|
|
8119
|
+
this.#sealLeafStore = new SessionSealLeafStore(this.#db, this.#logger);
|
|
8120
|
+
client = this.#detachedRelayClientBuilder?.(agentName, ep.relayPeerId, [...ep.relayAddrs], {
|
|
8121
|
+
receiptStore: this.#relayReceiptStore ?? undefined,
|
|
8122
|
+
sealLeafStore: this.#sealLeafStore ?? undefined,
|
|
8123
|
+
});
|
|
8124
|
+
if (!client) {
|
|
8125
|
+
this.#logger.warn("session.revive.relay.builder_absent", {
|
|
8126
|
+
agentName,
|
|
8127
|
+
sessionId,
|
|
8128
|
+
impact: "no relay client could be built, so this revived session has no live inbound path",
|
|
8129
|
+
});
|
|
8130
|
+
return;
|
|
8131
|
+
}
|
|
8132
|
+
this.#relayClients.set(clientKey, client);
|
|
8133
|
+
}
|
|
8134
|
+
client.registerSession(sessionId, node, this.#relayLeafHandler(agentName, sessionId, correlationId));
|
|
8135
|
+
const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
|
|
8136
|
+
if (entry) {
|
|
8137
|
+
entry.relayClient = client;
|
|
8138
|
+
entry.relaySessionIdBytes = Uint8Array.from(Buffer.from(sessionId, "hex"));
|
|
8139
|
+
entry.relayClientKey = clientKey;
|
|
8140
|
+
}
|
|
8141
|
+
/**
|
|
8142
|
+
* THE STEP THIS METHOD IS NAMED AFTER, and the first version did not take it (review HIGH-2).
|
|
8143
|
+
*
|
|
8144
|
+
* `registerSession` files a handler in a Map. It opens nothing — no dial, no auth, no reader
|
|
8145
|
+
* loop. `#connectSessionRelay` ends with exactly this line and the reconnect ended without it,
|
|
8146
|
+
* so a revived session registered a handler on a client whose stream was `null` and then
|
|
8147
|
+
* logged that its live inbound path was back. It was not: the counterparty's leaves queued at
|
|
8148
|
+
* the relay, no doorbell fired, and delivery fell back to the five-minute mailbox poll — the
|
|
8149
|
+
* three-minutes-versus-seconds symptom the whole unit exists to remove.
|
|
8150
|
+
*
|
|
8151
|
+
* Worse, the client is usually BRAND NEW here: `markInterruptedWithDetails` closes and drops
|
|
8152
|
+
* the client for the last session on a relay, so a single-session agent always lands in the
|
|
8153
|
+
* build branch above with a fresh, unconnected client.
|
|
8154
|
+
*
|
|
8155
|
+
* `#ensureConnected` is idempotent, so this also repairs the cached-but-dead-stream case.
|
|
8156
|
+
*/
|
|
8157
|
+
await client.connect(node);
|
|
8158
|
+
this.#logger.info("session.revive.relay.connected", {
|
|
8159
|
+
agentName,
|
|
8160
|
+
sessionId,
|
|
8161
|
+
relayPeerId: ep.relayPeerId,
|
|
8162
|
+
impact: "the revived session has its live inbound path back — messages arrive promptly "
|
|
8163
|
+
+ "instead of waiting for the periodic mailbox poll",
|
|
8164
|
+
});
|
|
8165
|
+
}
|
|
8166
|
+
catch (err) {
|
|
8167
|
+
this.#logger.warn("session.revive.relay.failed", {
|
|
8168
|
+
agentName,
|
|
8169
|
+
sessionId,
|
|
8170
|
+
error: err instanceof Error ? err.message : String(err),
|
|
8171
|
+
impact: "the session is back but without its witness — delivery falls back to the periodic poll",
|
|
8172
|
+
});
|
|
8173
|
+
}
|
|
8174
|
+
}
|
|
7729
8175
|
/**
|
|
7730
8176
|
* DOD-M12B-SESSION-SEED-1 — bring an interrupted session back on the peer id it already has.
|
|
7731
8177
|
*
|
|
@@ -7764,6 +8210,31 @@ export class SessionNodeManager {
|
|
|
7764
8210
|
guidance: `Session is '${record.status}'. A session that has ended cannot be revived; start a new one.`,
|
|
7765
8211
|
};
|
|
7766
8212
|
}
|
|
8213
|
+
/**
|
|
8214
|
+
* THE CAP APPLIES TO A REVIVAL TOO (review: parity gap). Establishment refuses at
|
|
8215
|
+
* `MAX_SESSION_NODES` because each node is a real libp2p instance with listeners, connections
|
|
8216
|
+
* and a relay reservation. A revival builds exactly the same thing, so letting it past the cap
|
|
8217
|
+
* would let a daemon walk over the limit one reconnect at a time — and the limit exists to stop
|
|
8218
|
+
* a machine being taken down by its own session count.
|
|
8219
|
+
*
|
|
8220
|
+
* Refused by name, so the caller can say something true: this is a local resource limit, not a
|
|
8221
|
+
* problem with the session or the counterparty.
|
|
8222
|
+
*/
|
|
8223
|
+
if (this.#activeNodes.size >= MAX_SESSION_NODES) {
|
|
8224
|
+
this.#logger.warn("session.revive.cap.reached", {
|
|
8225
|
+
agentName,
|
|
8226
|
+
sessionId,
|
|
8227
|
+
activeCount: this.#activeNodes.size,
|
|
8228
|
+
maxCount: MAX_SESSION_NODES,
|
|
8229
|
+
impact: "this session stays interrupted until another session ends and frees a node slot",
|
|
8230
|
+
});
|
|
8231
|
+
return {
|
|
8232
|
+
ok: false,
|
|
8233
|
+
reason: "session_node_cap_reached",
|
|
8234
|
+
guidance: `This daemon already holds ${MAX_SESSION_NODES} active session nodes, so this session ` +
|
|
8235
|
+
"cannot be brought back yet. Close a session you have finished with and try again.",
|
|
8236
|
+
};
|
|
8237
|
+
}
|
|
7767
8238
|
const identity = this.#sessionSeeds.get(key);
|
|
7768
8239
|
if (identity === undefined) {
|
|
7769
8240
|
// The honest case: the daemon restarted, so the keypair is genuinely gone. That is
|
|
@@ -7788,19 +8259,22 @@ export class SessionNodeManager {
|
|
|
7788
8259
|
for (const relayPeerId of reservations.relayPeerIds)
|
|
7789
8260
|
gater.setAllowedOutboundPeer(relayPeerId);
|
|
7790
8261
|
let node;
|
|
8262
|
+
const t0 = Date.now();
|
|
8263
|
+
this.#logger.info("session.revive.node.building", {
|
|
8264
|
+
agentName,
|
|
8265
|
+
sessionId,
|
|
8266
|
+
circuitAddrs: reservations.addrs.length,
|
|
8267
|
+
relayPeerIds: reservations.relayPeerIds.length,
|
|
8268
|
+
});
|
|
7791
8269
|
try {
|
|
7792
|
-
node = await this.#
|
|
8270
|
+
node = await this.#buildRevivedNode(sessionId, gater, identity.seed, reservations.addrs, agentName);
|
|
8271
|
+
this.#logger.info("session.revive.node.started", {
|
|
8272
|
+
agentName,
|
|
7793
8273
|
sessionId,
|
|
7794
|
-
|
|
7795
|
-
|
|
7796
|
-
|
|
7797
|
-
// review HIGH-1: reachable, and holding a circuit reservation. A revived node that binds
|
|
7798
|
-
// loopback preserves an identity nobody can dial — the counterparty's next message parks
|
|
7799
|
-
// again and case B is only half delivered.
|
|
7800
|
-
inboundReachable: true,
|
|
7801
|
-
...(reservations.addrs.length > 0 ? { circuitRelayListenAddrs: reservations.addrs } : {}),
|
|
8274
|
+
startMs: Date.now() - t0,
|
|
8275
|
+
listenAddrs: node.listenAddresses().length,
|
|
8276
|
+
circuitListen: node.listenAddresses().filter((a) => a.includes("/p2p-circuit")).length,
|
|
7802
8277
|
});
|
|
7803
|
-
await node.start();
|
|
7804
8278
|
}
|
|
7805
8279
|
catch (err) {
|
|
7806
8280
|
this.#logger.error("session.revive.node.failed", {
|
|
@@ -7818,7 +8292,38 @@ export class SessionNodeManager {
|
|
|
7818
8292
|
probers: this.#autoNatProbers(),
|
|
7819
8293
|
});
|
|
7820
8294
|
autoNat.emitInitialResult();
|
|
8295
|
+
// DOD-M12B-SESSION-SEED-1: give the re-dial its addresses back BEFORE the session goes active,
|
|
8296
|
+
// so the first send after a revival has somewhere to go. Without this the send fails instantly
|
|
8297
|
+
// on a connection that was never made, and — measured live — is lost rather than parked.
|
|
8298
|
+
if (identity.counterpartyAddrs.length > 0) {
|
|
8299
|
+
this.#counterpartyAddrs.set(key, [...identity.counterpartyAddrs]);
|
|
8300
|
+
}
|
|
7821
8301
|
const correlationId = randomUUID();
|
|
8302
|
+
/**
|
|
8303
|
+
* DOD-M12B-REVIVE-PARK-1 — RESTORE THE RELAY, or a revived session cannot park and every failed
|
|
8304
|
+
* send is declared lost.
|
|
8305
|
+
*
|
|
8306
|
+
* This is the defect behind five identical live failures on 2026-08-18. `#parkContent` opens
|
|
8307
|
+
* with `if (!hook || !entry || !entry.relayPeerId || !entry.relayAddrs) return "unconfigured"`,
|
|
8308
|
+
* and a revived entry carried none of it — so the park was skipped and the send fell through to
|
|
8309
|
+
* *"could NOT be queued for retry — it is lost. Send it again."* The relay was recorded on the
|
|
8310
|
+
* session row the whole time, and store-and-forward would have delivered the message: the
|
|
8311
|
+
* counterparty's own sends park through it successfully in the same minute.
|
|
8312
|
+
*
|
|
8313
|
+
* What it cost the operator: their reply was accepted, discarded, and they were told to retype
|
|
8314
|
+
* it — which is how a transcript gets duplicates of a message that was never lost in the first
|
|
8315
|
+
* place.
|
|
8316
|
+
*
|
|
8317
|
+
* Read from the row rather than carried in the revival record on purpose: the row is where the
|
|
8318
|
+
* relay assignment is durable, and it is the same source `getPersistedRelayEndpoint` already
|
|
8319
|
+
* serves the startup flush from — a path that exists precisely because in-memory entries are
|
|
8320
|
+
* gone by then, which is exactly the situation a revival is in.
|
|
8321
|
+
*/
|
|
8322
|
+
// ONE lookup, and ONE event for the absent case (review LOW-8). This used to read the endpoint
|
|
8323
|
+
// here and again inside the relay reconnect, and both logged `session.revive.relay.absent` with
|
|
8324
|
+
// different `impact` text — one event name standing for two meanings, fired twice for a single
|
|
8325
|
+
// condition. `#reconnectRevivedSessionRelay` takes it as a parameter now.
|
|
8326
|
+
const persistedRelay = this.getPersistedRelayEndpoint(agentName, sessionId);
|
|
7822
8327
|
this.#activeNodes.set(key, {
|
|
7823
8328
|
node,
|
|
7824
8329
|
agentName,
|
|
@@ -7828,6 +8333,9 @@ export class SessionNodeManager {
|
|
|
7828
8333
|
correlationId,
|
|
7829
8334
|
counterpartySessionPeerId: identity.counterpartyPeerId,
|
|
7830
8335
|
autoNat,
|
|
8336
|
+
...(persistedRelay
|
|
8337
|
+
? { relayPeerId: persistedRelay.relayPeerId, relayAddrs: persistedRelay.relayAddrs }
|
|
8338
|
+
: {}),
|
|
7831
8339
|
});
|
|
7832
8340
|
await this.#registerContentHandler(agentName, sessionId, node, identity.counterpartyPubkey);
|
|
7833
8341
|
/**
|
|
@@ -7841,6 +8349,9 @@ export class SessionNodeManager {
|
|
|
7841
8349
|
* to trigger the next revival either.
|
|
7842
8350
|
*/
|
|
7843
8351
|
this.#wireSessionLiveness(agentName, sessionId, node, identity.counterpartyPubkey, correlationId, identity.counterpartyPeerId);
|
|
8352
|
+
// DOD-M12B-REVIVE-RELAY-1: the step revival skipped. Establishment connects the relay witness
|
|
8353
|
+
// here; without it the session comes back with no live inbound path at all.
|
|
8354
|
+
await this.#reconnectRevivedSessionRelay(agentName, sessionId, node, gater, correlationId, persistedRelay);
|
|
7844
8355
|
// THE REVERSE EDGE. A transport event took this session out of `active` and nothing has ever
|
|
7845
8356
|
// put one back. Written after the node is live and its handler registered, so the row never
|
|
7846
8357
|
// claims `active` for a session that cannot yet receive.
|
|
@@ -7862,6 +8373,21 @@ export class SessionNodeManager {
|
|
|
7862
8373
|
"down rather than left live under an interrupted row. The daemon logged the cause.",
|
|
7863
8374
|
};
|
|
7864
8375
|
}
|
|
8376
|
+
// The messages that failed while this session was down were queued on a promise of "retried on
|
|
8377
|
+
// reconnect". This is that reconnect — fire it before anyone is told the session is back.
|
|
8378
|
+
if (this.#retryDrainHook !== null) {
|
|
8379
|
+
try {
|
|
8380
|
+
this.#retryDrainHook(agentName, sessionId);
|
|
8381
|
+
}
|
|
8382
|
+
catch (err) {
|
|
8383
|
+
this.#logger.warn("session.revive.retry_drain.failed", {
|
|
8384
|
+
agentName,
|
|
8385
|
+
sessionId,
|
|
8386
|
+
error: err instanceof Error ? err.message : String(err),
|
|
8387
|
+
impact: "messages queued while this session was down are still queued",
|
|
8388
|
+
});
|
|
8389
|
+
}
|
|
8390
|
+
}
|
|
7865
8391
|
const peerId = node.getPeerId();
|
|
7866
8392
|
this.#logger.info("session.revived", {
|
|
7867
8393
|
agentName,
|
|
@@ -7876,8 +8402,8 @@ export class SessionNodeManager {
|
|
|
7876
8402
|
/**
|
|
7877
8403
|
* DOD-M12B-SESSION-SEED-1 — the DEMAND edge: a send on an interrupted session revives it.
|
|
7878
8404
|
*
|
|
7879
|
-
*
|
|
7880
|
-
* rather than
|
|
8405
|
+
* One of TWO production callers of `reviveSessionNode` — `reviveIfNeededForRead` is the other —
|
|
8406
|
+
* and both are deliberately demand paths rather than timers. The `REDIAL-1` discipline and Andre's tenet say the same thing from two
|
|
7881
8407
|
* directions: nothing may re-open on its own, because a background rebuilder would hold a dialable
|
|
7882
8408
|
* endpoint open for a session nobody is using — the *"open connection a malicious agent can farm
|
|
7883
8409
|
* for"*. The operator sending is the demand; there is no other trigger.
|
|
@@ -7970,11 +8496,45 @@ export class SessionNodeManager {
|
|
|
7970
8496
|
this.#fireParkedDrain(agentName, "session_revived");
|
|
7971
8497
|
return { ok: true };
|
|
7972
8498
|
}
|
|
8499
|
+
/** DOD-M12B-REVIVE-PARK-1 test seam: the relay the live entry will park to. Not otherwise
|
|
8500
|
+
* observable — `#activeNodes` is private and the park's own refusal is silent about which of its
|
|
8501
|
+
* four preconditions was missing. */
|
|
8502
|
+
getSessionRelayForTest(agentName, sessionId) {
|
|
8503
|
+
const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
|
|
8504
|
+
if (!entry)
|
|
8505
|
+
return null;
|
|
8506
|
+
return {
|
|
8507
|
+
...(entry.relayPeerId !== undefined ? { relayPeerId: entry.relayPeerId } : {}),
|
|
8508
|
+
...(entry.relayAddrs !== undefined ? { relayAddrs: entry.relayAddrs } : {}),
|
|
8509
|
+
};
|
|
8510
|
+
}
|
|
8511
|
+
/** DOD-M12B-SESSION-SEED-1 test seams: the counterparty addresses a re-dial depends on. Not
|
|
8512
|
+
* otherwise observable — they are set from a signed relay assignment that a fixture cannot mint. */
|
|
8513
|
+
setCounterpartyAddrsForTest(agentName, sessionId, addrs) {
|
|
8514
|
+
this.#counterpartyAddrs.set(this.#k(agentName, sessionId), [...addrs]);
|
|
8515
|
+
}
|
|
8516
|
+
getCounterpartyAddrsForTest(agentName, sessionId) {
|
|
8517
|
+
return this.#counterpartyAddrs.get(this.#k(agentName, sessionId)) ?? [];
|
|
8518
|
+
}
|
|
7973
8519
|
/** DOD-M12B-SESSION-SEED-1 test seam: drop a seed WITHOUT zeroing or a status change — what a
|
|
7974
8520
|
* process restart does to it. The refusal that follows is the one an operator most needs named. */
|
|
7975
8521
|
forgetSessionSeedForTest(agentName, sessionId) {
|
|
7976
8522
|
this.#sessionSeeds.delete(this.#k(agentName, sessionId));
|
|
7977
8523
|
}
|
|
8524
|
+
/**
|
|
8525
|
+
* DOD-M12B-SESSION-SEED-1 — drain the direct-resend queue when a session comes back.
|
|
8526
|
+
*
|
|
8527
|
+
* `retryQueue.drainSession` had NO production caller. The send path enqueues into it on a failed
|
|
8528
|
+
* delivery and tells the operator the message will be "retried on reconnect", and nothing ever
|
|
8529
|
+
* reconnected it — the row sat there until the session went terminal and was reaped. Measured
|
|
8530
|
+
* live 2026-08-18: two of the operator's messages went in, and the response told them both were
|
|
8531
|
+
* lost and to send again, which is how a transcript gets duplicates.
|
|
8532
|
+
*
|
|
8533
|
+
* A revival IS the reconnect that sentence promised. This is the hook that makes it true.
|
|
8534
|
+
*/
|
|
8535
|
+
setRetryDrainHook(fn) {
|
|
8536
|
+
this.#retryDrainHook = fn;
|
|
8537
|
+
}
|
|
7978
8538
|
/** DOD-M12B-SESSION-SEED-1 test seam: does this session still hold a revivable identity? */
|
|
7979
8539
|
hasSessionSeedForTest(agentName, sessionId) {
|
|
7980
8540
|
return this.#sessionSeeds.has(this.#k(agentName, sessionId));
|
|
@@ -8059,6 +8619,34 @@ export class SessionNodeManager {
|
|
|
8059
8619
|
/** CC-5/F21: count of RECEIVED messages on a session — the "did the counterparty ever speak"
|
|
8060
8620
|
* signal the dead-half-open reaper uses (message_count also counts our own auto-"Dispatched." ack,
|
|
8061
8621
|
* so it is NOT a reliable half-open discriminator). Mirrors #getReceivedBytesTotal. */
|
|
8622
|
+
/**
|
|
8623
|
+
* DOD-M12B-REAP-HELD-1 — did the counterparty EVER establish? Counted from every place their
|
|
8624
|
+
* messages can be, not just the one.
|
|
8625
|
+
*
|
|
8626
|
+
* OBSERVED LIVE 2026-08-18: the half-open reaper abandoned session `d28db475…` — twenty leaves in
|
|
8627
|
+
* the chain and sixteen more frames verified and held, ten of them from the counterparty — while
|
|
8628
|
+
* the restart-seal resolver was actively trying to notarize it. The receipt was forfeited.
|
|
8629
|
+
*
|
|
8630
|
+
* `countReceivedMessages` asks the TRANSCRIPT, and **held content never reaches the transcript**;
|
|
8631
|
+
* it sits in `held_content` until it can join the chain. So the very condition that holds content
|
|
8632
|
+
* — an interrupted session — is the condition that makes the counterparty's messages invisible,
|
|
8633
|
+
* and a fully-established conversation reads identically to an offer nobody ever answered.
|
|
8634
|
+
*
|
|
8635
|
+
* `origin = 'received'` is load-bearing. Our OWN held frames prove nothing about them, and
|
|
8636
|
+
* counting those would make every session we ever spoke into un-reapable — which is exactly the
|
|
8637
|
+
* clutter the reaper exists to clear. D18 also depends on the zero case staying zero: reaping only
|
|
8638
|
+
* genuinely 0-received ghosts is what stops a stranger whose first handshakes died from being
|
|
8639
|
+
* locked out by the acceptance bound forever.
|
|
8640
|
+
*/
|
|
8641
|
+
countEstablishedReceived(agentName, sessionId) {
|
|
8642
|
+
if (!this.#db)
|
|
8643
|
+
return 0;
|
|
8644
|
+
const agentId = this.#requireAgentId(agentName);
|
|
8645
|
+
const held = this.#db
|
|
8646
|
+
.prepare("SELECT COUNT(*) AS n FROM held_content WHERE agent_id = ? AND session_id = ? AND origin = 'received'")
|
|
8647
|
+
.get(agentId, sessionId);
|
|
8648
|
+
return this.countReceivedMessages(agentName, sessionId) + (held?.n ?? 0);
|
|
8649
|
+
}
|
|
8062
8650
|
countReceivedMessages(agentName, sessionId) {
|
|
8063
8651
|
if (!this.#db)
|
|
8064
8652
|
return 0;
|