@cello-protocol/daemon 0.0.169 → 0.0.170
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-id-migration.d.ts.map +1 -1
- package/dist/agent-id-migration.js +16 -0
- package/dist/agent-id-migration.js.map +1 -1
- package/dist/away-detection.d.ts +62 -15
- package/dist/away-detection.d.ts.map +1 -1
- package/dist/away-detection.js +77 -20
- package/dist/away-detection.js.map +1 -1
- package/dist/close-session-handler.d.ts.map +1 -1
- package/dist/close-session-handler.js +69 -3
- package/dist/close-session-handler.js.map +1 -1
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +291 -38
- package/dist/daemon.js.map +1 -1
- package/dist/delivery-open-registry.d.ts +92 -0
- package/dist/delivery-open-registry.d.ts.map +1 -0
- package/dist/delivery-open-registry.js +121 -0
- package/dist/delivery-open-registry.js.map +1 -0
- package/dist/document-delivery-transport.d.ts +10 -1
- package/dist/document-delivery-transport.d.ts.map +1 -1
- package/dist/document-delivery-transport.js +10 -1
- package/dist/document-delivery-transport.js.map +1 -1
- package/dist/document-layer.d.ts +21 -0
- package/dist/document-layer.d.ts.map +1 -1
- package/dist/document-layer.js +34 -1
- package/dist/document-layer.js.map +1 -1
- package/dist/document-reconcile-scheduler.d.ts +33 -0
- package/dist/document-reconcile-scheduler.d.ts.map +1 -1
- package/dist/document-reconcile-scheduler.js +73 -0
- package/dist/document-reconcile-scheduler.js.map +1 -1
- package/dist/inbound-sessions.d.ts +7 -0
- package/dist/inbound-sessions.d.ts.map +1 -1
- package/dist/inbound-sessions.js +27 -6
- package/dist/inbound-sessions.js.map +1 -1
- package/dist/notification-handlers.d.ts.map +1 -1
- package/dist/notification-handlers.js +39 -2
- package/dist/notification-handlers.js.map +1 -1
- package/dist/session-content-handlers.d.ts.map +1 -1
- package/dist/session-content-handlers.js +134 -8
- package/dist/session-content-handlers.js.map +1 -1
- package/dist/session-node-manager.d.ts +153 -8
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +1376 -35
- package/dist/session-node-manager.js.map +1 -1
- package/dist/types.d.ts +64 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +5 -5
|
@@ -52,6 +52,51 @@ import { GATEWAY_UNAVAILABLE, GOVERNANCE_TIMEOUT, } from "@cello-protocol/gatewa
|
|
|
52
52
|
*/
|
|
53
53
|
const AUTOACK_BROKER_GRACE_MS = 30_000;
|
|
54
54
|
const MAX_REFUSED_PARKED_ENTRIES = 512;
|
|
55
|
+
/**
|
|
56
|
+
* DOD-M12B-ACK-1 — inbound `/cello/content/1.0.0` streams allowed per connection.
|
|
57
|
+
*
|
|
58
|
+
* libp2p's registrar default is 32, and it enforces the cap AFTER protocol negotiation has already
|
|
59
|
+
* answered, so exceeding it resets a stream the sender believes it just opened and the sender's
|
|
60
|
+
* next write fails with a message that names nothing. Content delivery is bursty by design (a
|
|
61
|
+
* document sweep opened 99 events in one second on a live daemon), and a slot stays occupied for
|
|
62
|
+
* the whole of ingest — which awaits SQLCipher and the security gateway. 32 is simply too close to
|
|
63
|
+
* normal traffic to be a safety limit.
|
|
64
|
+
*
|
|
65
|
+
* This is headroom, NOT the fix. The fix is that #handleContentStream now closes what it opens; a
|
|
66
|
+
* raised cap without that would only move the cliff. Kept finite on purpose: an unbounded cap would
|
|
67
|
+
* let a peer pin memory by opening streams it never uses, and the ceiling is what makes a future
|
|
68
|
+
* leak of this shape show up as a bounded failure instead of a heap.
|
|
69
|
+
*/
|
|
70
|
+
const CONTENT_MAX_INBOUND_STREAMS = 512;
|
|
71
|
+
/**
|
|
72
|
+
* DOD-M12B-ACK-1 — how long an inbound content stream may stay open after we have closed our end.
|
|
73
|
+
*
|
|
74
|
+
* Closing our write end retires the stream only once the PEER has closed its end too, so a peer
|
|
75
|
+
* that opens streams and never closes them still fills our inbound slots — a guard that runs only
|
|
76
|
+
* on the party it constrains is not a guard. After this window we reset it ourselves, which the
|
|
77
|
+
* muxer honours unilaterally.
|
|
78
|
+
*
|
|
79
|
+
* It cannot be zero: an immediate reset would land while a well-behaved sender is still inside its
|
|
80
|
+
* own `await stream.close()`, rejecting that close and turning every ordinary send into a park.
|
|
81
|
+
* The frame is already ingested long before this fires, so nothing waits on it.
|
|
82
|
+
*/
|
|
83
|
+
const CONTENT_STREAM_LINGER_MS = 30_000;
|
|
84
|
+
/**
|
|
85
|
+
* DOD-M12B-SHUTDOWN-1 — how long one teardown step may block the daemon's exit.
|
|
86
|
+
*
|
|
87
|
+
* Chosen against the surface that complains: `cello logout` gives up and reports the daemon still
|
|
88
|
+
* running after 5 s, so a step that can burn longer than that guarantees the message the operator
|
|
89
|
+
* saw. Two steps at 2 s each stay inside it.
|
|
90
|
+
*/
|
|
91
|
+
const SHUTDOWN_STEP_DEADLINE_MS = 2_000;
|
|
92
|
+
/**
|
|
93
|
+
* DOD-M12B-REDIAL-1 — the shortest gap between two re-dials of one session.
|
|
94
|
+
*
|
|
95
|
+
* Long enough that a burst of sends against a peer that is genuinely gone costs one dial rather
|
|
96
|
+
* than one per message; short enough that a peer coming back is picked up on the next thing the
|
|
97
|
+
* operator says. It is cleared on a successful dial, so it never delays a live counterparty.
|
|
98
|
+
*/
|
|
99
|
+
const REDIAL_COOLDOWN_MS = 15_000;
|
|
55
100
|
// Persistence bounds are TIER-GRADUATED via DEFAULT_TIER_BOUNDS (contacts-tier-migration). The two
|
|
56
101
|
// consts below DERIVE from the grid's UNKNOWN row rather than restating it — the grid is the single
|
|
57
102
|
// source (DOD-TIER-2 AC4), so these can never drift from it.
|
|
@@ -237,9 +282,36 @@ export class SessionNodeManager {
|
|
|
237
282
|
#autoNatProbers;
|
|
238
283
|
// M7-SESSION-003: per-session direct-path counterparty liveness, observed on the
|
|
239
284
|
// session node's onPeerConnect ('alive') / onPeerDisconnect ('gone'). This is
|
|
240
|
-
// the liveness authority for direct sessions
|
|
241
|
-
//
|
|
285
|
+
// the liveness authority for direct sessions (relay sessions query the relay
|
|
286
|
+
// instead). NEVER the directory (SI-002). Read by exactly three consumers: the
|
|
287
|
+
// half-open reaper, both status surfaces, and cello_receive. No seal path reads
|
|
288
|
+
// it — the coupling to sealing runs through the receive guidance, which turns
|
|
289
|
+
// 'gone' into "call cello_close_session".
|
|
290
|
+
// DOD-M12B-ACK-1 adds 'impaired': connection up, our writes on it failing. It sits BELOW
|
|
291
|
+
// 'alive' and never above 'gone' — see #markSessionImpaired.
|
|
242
292
|
#sessionLiveness = new Map();
|
|
293
|
+
// DOD-M12B-ACK-1: WHY a session is impaired and what became of the content. Separate from the
|
|
294
|
+
// state above because the state is what surfaces print and this is what they must explain.
|
|
295
|
+
#impairmentCause = new Map();
|
|
296
|
+
// DOD-M12B-STRAND-1: sessions whose durable holds have been read back. One read per session per
|
|
297
|
+
// process; the Map is the working copy from then on.
|
|
298
|
+
#heldRestored = new Set();
|
|
299
|
+
// DOD-M12B-SEAL-STUCK-1: sessions whose post-restore release has been attempted. Separate from
|
|
300
|
+
// #heldRestored so a READ-ONLY probe can hydrate without performing (or consuming) the release.
|
|
301
|
+
#heldReleased = new Set();
|
|
302
|
+
// DOD-M12B-SEAL-STUCK-1: sessions whose ordering state THIS PROCESS has observed — a relay
|
|
303
|
+
// witness recorded for it. `#witnessedSeq` is memory-only, so for a session that predates this
|
|
304
|
+
// daemon "no gap recorded" means "not recorded", not "no gap", and that difference decides
|
|
305
|
+
// whether we may tell an operator the session is safe to close.
|
|
306
|
+
#orderingObserved = new Set();
|
|
307
|
+
// DOD-M12B-INDEX-1: sessions whose tree and whose relay counter have provably parted. A diverged
|
|
308
|
+
// session can never produce a root the counterparty agrees with, so it must never be reported as
|
|
309
|
+
// safe to close — the close would be signed, refused as `leaf_count_mismatch`, and the receipt
|
|
310
|
+
// lost for good.
|
|
311
|
+
#diverged = new Set();
|
|
312
|
+
// DOD-M12B-ACK-1: pending linger-resets for inbound content streams the peer has not closed.
|
|
313
|
+
// Held so shutdown can drop them rather than leave timers pointing at a torn-down node.
|
|
314
|
+
#lingeringStreams = new Set();
|
|
243
315
|
// M7-UPGRADE-002: sessions whose content integrity could NOT be verified (a content_hash
|
|
244
316
|
// mismatch = tamper was observed). The auto-acknowledge gate (SI-002) refuses to auto-co-sign
|
|
245
317
|
// for a desynced session — B must never blind-sign a tail it cannot verify. Keyed by sessionId hex.
|
|
@@ -341,6 +413,20 @@ export class SessionNodeManager {
|
|
|
341
413
|
// with its agent away), and the park deposit that follows has to be refused. One without the
|
|
342
414
|
// other reproduces nothing.
|
|
343
415
|
#sendFaultRemaining = 0;
|
|
416
|
+
// DOD-M12B-ACK-1 — the same seam for the delivery-ACK write. See injectAckFault.
|
|
417
|
+
#ackFaultRemaining = 0;
|
|
418
|
+
// DOD-M12B-REDIAL-1 — makes the next N `newStream` calls report the connection as gone, BEFORE
|
|
419
|
+
// the node is touched. The sibling of injectSendFault for the one condition that used to end a
|
|
420
|
+
// conversation permanently; a real connection drop is not reproducible in-process.
|
|
421
|
+
#connectionLossRemaining = 0;
|
|
422
|
+
// DOD-M12B-REDIAL-1: the counterparty addresses this session dialled, kept so it can dial them
|
|
423
|
+
// again. They arrived in the FROST-signed assignment and were used once and dropped, which is
|
|
424
|
+
// why nothing could ever re-dial.
|
|
425
|
+
#counterpartyAddrs = new Map();
|
|
426
|
+
// DOD-M12B-REDIAL-1: when this session may next attempt a re-dial. Continuous re-dialling is what
|
|
427
|
+
// produced the 2026-08-17 notification storm, so a burst of sends against a peer that is gone
|
|
428
|
+
// costs one attempt, not one per message.
|
|
429
|
+
#redialNotBefore = new Map();
|
|
344
430
|
/** Arm the park-deposit fault. Returns the count now armed. */
|
|
345
431
|
injectParkFault(count, cause) {
|
|
346
432
|
this.#parkFaultRemaining = Math.max(0, count);
|
|
@@ -353,6 +439,20 @@ export class SessionNodeManager {
|
|
|
353
439
|
this.#sendFaultRemaining = Math.max(0, count);
|
|
354
440
|
return this.#sendFaultRemaining;
|
|
355
441
|
}
|
|
442
|
+
/** DOD-M12B-ACK-1: arm the delivery-ACK write fault — the sibling of injectSendFault for the
|
|
443
|
+
* path that fails on a LISTENING agent. Without it the ACK failure branch (which impairs the
|
|
444
|
+
* session and, until this milestone, could never clear it again) is unreachable from a test:
|
|
445
|
+
* a listener sends no content, so the direct-send fault never fires for it. */
|
|
446
|
+
injectAckFault(count) {
|
|
447
|
+
this.#ackFaultRemaining = Math.max(0, count);
|
|
448
|
+
return this.#ackFaultRemaining;
|
|
449
|
+
}
|
|
450
|
+
/** DOD-M12B-REDIAL-1: arm the connection-loss fault — the next N direct sends find no open
|
|
451
|
+
* connection, exactly as they do after any blip. See #connectionLossRemaining. */
|
|
452
|
+
injectConnectionLoss(count) {
|
|
453
|
+
this.#connectionLossRemaining = Math.max(0, count);
|
|
454
|
+
return this.#connectionLossRemaining;
|
|
455
|
+
}
|
|
356
456
|
getSendFaultRemaining() {
|
|
357
457
|
return this.#sendFaultRemaining;
|
|
358
458
|
}
|
|
@@ -639,6 +739,10 @@ export class SessionNodeManager {
|
|
|
639
739
|
// watermark: this records "operator acknowledged via dismiss", not "operator received via
|
|
640
740
|
// cello_receive". NULL = not yet dismissed.
|
|
641
741
|
"ALTER TABLE sessions ADD COLUMN read_at INTEGER",
|
|
742
|
+
// DOD-M12B-ABANDON-NOTIFY-1: epoch-ms when the counterparty told us they force-abandoned.
|
|
743
|
+
// Deliberately NOT a status — the session stays sealable, so the operator can still take a
|
|
744
|
+
// unilateral receipt. It stops this side calling them, nothing more.
|
|
745
|
+
"ALTER TABLE sessions ADD COLUMN counterparty_abandoned_at INTEGER",
|
|
642
746
|
]) {
|
|
643
747
|
try {
|
|
644
748
|
this.#db.exec(ddl);
|
|
@@ -733,6 +837,69 @@ export class SessionNodeManager {
|
|
|
733
837
|
PRIMARY KEY (agent_id, session_id, leaf_index)
|
|
734
838
|
)
|
|
735
839
|
`);
|
|
840
|
+
// DOD-M12B-STRAND-1 — content we RECEIVED and VERIFIED but cannot append yet.
|
|
841
|
+
//
|
|
842
|
+
// Held content used to live only in `#heldContent`, a Map that died with the session node. The
|
|
843
|
+
// teardown path said so itself: "the content is unrecoverable by the time we are here."
|
|
844
|
+
// Measured on one daemon in one morning: 367 held, 8 released, **24 destroyed**. Each
|
|
845
|
+
// destruction is permanent and one-sided — the sender was never acknowledged, so it believes
|
|
846
|
+
// the message is merely pending, while the only copy the receiver will ever see is gone and
|
|
847
|
+
// every later message in that session is stuck behind a gap nothing can fill.
|
|
848
|
+
//
|
|
849
|
+
// `canonical_seq` is the RELAY's position, not a local counter, and it is part of the key: that
|
|
850
|
+
// is what lets a frame come back after a restart and land at its OWN index rather than the next
|
|
851
|
+
// free slot. Appending it anywhere else would change the root the seal signs over.
|
|
852
|
+
//
|
|
853
|
+
// Keyed on agent_id, never agent_name — agent_name is a mutable display label (see the repo
|
|
854
|
+
// guide). `content_blob` is the SCREENED copy that gets delivered; `original_blob` is the peer's
|
|
855
|
+
// raw bytes, which the release path needs because classification reads byte 0 and the screened
|
|
856
|
+
// copy is no longer a CBOR map header for a document frame.
|
|
857
|
+
this.#db.exec(`
|
|
858
|
+
CREATE TABLE IF NOT EXISTS held_content (
|
|
859
|
+
agent_id TEXT NOT NULL,
|
|
860
|
+
session_id TEXT NOT NULL,
|
|
861
|
+
canonical_seq INTEGER NOT NULL,
|
|
862
|
+
content_blob BLOB NOT NULL,
|
|
863
|
+
original_blob BLOB,
|
|
864
|
+
content_hash_hex TEXT NOT NULL,
|
|
865
|
+
screened_out INTEGER NOT NULL DEFAULT 0,
|
|
866
|
+
correlation_id TEXT,
|
|
867
|
+
held_at INTEGER NOT NULL,
|
|
868
|
+
-- DOD-M12B-INDEX-1: 'received' (default) or 'sent'. A held frame of OUR OWN must be
|
|
869
|
+
-- released down the sent path — appended and transcribed as sent — never down the received
|
|
870
|
+
-- path, which would put our words in the counterparty's mouth in the sealed record and hand
|
|
871
|
+
-- them back to our own agent through cello_receive as though they had just arrived.
|
|
872
|
+
origin TEXT NOT NULL DEFAULT 'received',
|
|
873
|
+
-- DOD-M12B-INDEX-1: 'msg' or 'doc'. A held document leaf must come back as a document leaf.
|
|
874
|
+
leaf_kind TEXT NOT NULL DEFAULT 'msg',
|
|
875
|
+
PRIMARY KEY (agent_id, session_id, canonical_seq)
|
|
876
|
+
)
|
|
877
|
+
`);
|
|
878
|
+
// DOD-M12B-INDEX-1: `CREATE TABLE IF NOT EXISTS` is a NO-OP against a table that already
|
|
879
|
+
// exists, so a database created between DOD-M12B-STRAND-1 and this change has `held_content`
|
|
880
|
+
// WITHOUT `origin`. On those every insert throws and every restore throws — holds go back to
|
|
881
|
+
// memory-only, silently at the surface, and that now includes our own sent messages, which
|
|
882
|
+
// nobody else holds a copy of. Loud in the log is not the same as visible.
|
|
883
|
+
try {
|
|
884
|
+
this.#db.exec("ALTER TABLE held_content ADD COLUMN origin TEXT NOT NULL DEFAULT 'received'");
|
|
885
|
+
}
|
|
886
|
+
catch (err) {
|
|
887
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
888
|
+
if (!/duplicate column name/i.test(msg))
|
|
889
|
+
throw err;
|
|
890
|
+
}
|
|
891
|
+
// DOD-M12B-INDEX-1: and the LEAF KIND. `#releaseHeld` used to append every held frame as "msg",
|
|
892
|
+
// so a document leaf that had to wait for its position came back as a conversation message —
|
|
893
|
+
// the distinction survived the immediate append and was destroyed by the hold, unrecoverably
|
|
894
|
+
// after a restart.
|
|
895
|
+
try {
|
|
896
|
+
this.#db.exec("ALTER TABLE held_content ADD COLUMN leaf_kind TEXT NOT NULL DEFAULT 'msg'");
|
|
897
|
+
}
|
|
898
|
+
catch (err) {
|
|
899
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
900
|
+
if (!/duplicate column name/i.test(msg))
|
|
901
|
+
throw err;
|
|
902
|
+
}
|
|
736
903
|
// DOD-LOG-1 (PERSIST-LOG-001) / PERSIST-002 (AC-010): the durable, ENCRYPTED-at-rest readable
|
|
737
904
|
// transcript. Each row is keyed by the canonical leaf `sequence`, so it JOINS to
|
|
738
905
|
// session_tree_leaves(leaf_index) — a stored message is provably behind a committed hash-chain
|
|
@@ -1548,6 +1715,10 @@ export class SessionNodeManager {
|
|
|
1548
1715
|
* otherwise let multiple held chunks each individually pass the size gate while cumulatively
|
|
1549
1716
|
* exceeding it once #releaseHeld drains them. */
|
|
1550
1717
|
#getHeldBytesTotal(agentName, sessionId) {
|
|
1718
|
+
// DOD-M12B-STRAND-1: hydrate first. Reading the Map before the durable holds are back
|
|
1719
|
+
// under-counts, and this gate exists to stop several held chunks each passing the size cap
|
|
1720
|
+
// individually while cumulatively exceeding it — an under-count is the bypass.
|
|
1721
|
+
this.#ensureHeldRestored(agentName, sessionId);
|
|
1551
1722
|
const held = this.#heldContent.get(this.#k(agentName, sessionId));
|
|
1552
1723
|
if (!held)
|
|
1553
1724
|
return 0;
|
|
@@ -1978,10 +2149,21 @@ export class SessionNodeManager {
|
|
|
1978
2149
|
};
|
|
1979
2150
|
}
|
|
1980
2151
|
// Log observability event (session.node.created)
|
|
2152
|
+
//
|
|
2153
|
+
// `counterpartySessionPeerId` IS LOGGED because it is recorded here ONCE and never refreshed,
|
|
2154
|
+
// while a standing receiver is rebuilt with a fresh libp2p keypair on every signaling reconnect
|
|
2155
|
+
// and every lost reservation. If the peer rebuilds between advertising its endpoint and this
|
|
2156
|
+
// handoff, we record an identity that no longer exists — and since `newStream` never dials, it
|
|
2157
|
+
// only ever looks for an ALREADY-OPEN connection filed under exactly this string, so every send
|
|
2158
|
+
// in this direction parks forever while the reverse direction works fine.
|
|
2159
|
+
//
|
|
2160
|
+
// Both sides of a local session log this event, so recording the id we will dial makes that
|
|
2161
|
+
// mismatch a direct comparison in the log instead of an unfalsifiable hypothesis.
|
|
1981
2162
|
this.#logger.info("session.node.created", {
|
|
1982
2163
|
sessionId,
|
|
1983
2164
|
agentName,
|
|
1984
2165
|
sessionPeerId: peerId,
|
|
2166
|
+
counterpartySessionPeerId: counterpartyPeerId,
|
|
1985
2167
|
correlationId,
|
|
1986
2168
|
});
|
|
1987
2169
|
// Add to active map (keyed by (agentName, sessionId) — DOD-LOOP-1)
|
|
@@ -2232,10 +2414,164 @@ export class SessionNodeManager {
|
|
|
2232
2414
|
/**
|
|
2233
2415
|
* M7-SESSION-003: read the direct-path counterparty liveness for a session.
|
|
2234
2416
|
* 'unknown' when no session node observation has occurred yet.
|
|
2417
|
+
*
|
|
2418
|
+
* DOD-M12B-ACK-1: 'impaired' is DAEMON-LOCAL and deliberately not on the relay's
|
|
2419
|
+
* SessionLiveness wire type — the relay answers a different question (does it hold the
|
|
2420
|
+
* recipient's standing connection) and its three states are a deployed bilateral contract.
|
|
2235
2421
|
*/
|
|
2236
2422
|
getSessionLiveness(agentName, sessionId) {
|
|
2237
2423
|
return this.#sessionLiveness.get(this.#k(agentName, sessionId)) ?? "unknown";
|
|
2238
2424
|
}
|
|
2425
|
+
/**
|
|
2426
|
+
* DOD-M12B-ACK-1 — live `/cello/content/1.0.0` stream counts on a session's direct path, or null
|
|
2427
|
+
* when the session has no active node.
|
|
2428
|
+
*
|
|
2429
|
+
* Answerable at runtime on purpose, in the same spirit as getConnectionMonitorPolicy: the count
|
|
2430
|
+
* is what decides whether the next send survives, and until this existed it could only be
|
|
2431
|
+
* recovered by measuring a log after the fact. It is also what lets the regression assert that a
|
|
2432
|
+
* slot was RELEASED, rather than that some particular number of messages happened to fit.
|
|
2433
|
+
*/
|
|
2434
|
+
countSessionContentStreams(agentName, sessionId) {
|
|
2435
|
+
const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
|
|
2436
|
+
if (!entry || typeof entry.node.countProtocolStreams !== "function")
|
|
2437
|
+
return null;
|
|
2438
|
+
return entry.node.countProtocolStreams(entry.counterpartySessionPeerId, CELLO_CONTENT_PROTOCOL_ID);
|
|
2439
|
+
}
|
|
2440
|
+
/**
|
|
2441
|
+
* DOD-M12B-ACK-1 — the live content-stream counts for a peer, as log context.
|
|
2442
|
+
*
|
|
2443
|
+
* A diagnostic must not be able to break the failure path it describes, so a node that predates
|
|
2444
|
+
* `countProtocolStreams` (test fakes do) yields no fields rather than throwing.
|
|
2445
|
+
*/
|
|
2446
|
+
#streamCensus(node, peerId) {
|
|
2447
|
+
if (typeof node.countProtocolStreams !== "function")
|
|
2448
|
+
return {};
|
|
2449
|
+
try {
|
|
2450
|
+
const { inbound, outbound } = node.countProtocolStreams(peerId, CELLO_CONTENT_PROTOCOL_ID);
|
|
2451
|
+
return { contentStreamsInbound: inbound, contentStreamsOutbound: outbound, contentStreamsInboundCap: CONTENT_MAX_INBOUND_STREAMS };
|
|
2452
|
+
}
|
|
2453
|
+
catch {
|
|
2454
|
+
return {};
|
|
2455
|
+
}
|
|
2456
|
+
}
|
|
2457
|
+
/**
|
|
2458
|
+
* DOD-M12B-ACK-1 — the connection is up and delivery on it is not working.
|
|
2459
|
+
*
|
|
2460
|
+
* Liveness is otherwise driven ONLY by libp2p peer-connect/peer-disconnect, so it answers "is
|
|
2461
|
+
* there a connection object?" while every surface that prints it is read as "can I talk to them?".
|
|
2462
|
+
* Measured 2026-08-17: one session reported `alive` for 70 minutes after every write had started
|
|
2463
|
+
* failing, another never stopped.
|
|
2464
|
+
*
|
|
2465
|
+
* ONLY 'gone' is protected, and 'gone' is NOT protected because a seal gate reads it — nothing in
|
|
2466
|
+
* the code does. It is protected because the receive surface turns 'gone' into "call
|
|
2467
|
+
* cello_close_session", and a failed write must never be able to produce that instruction.
|
|
2468
|
+
*
|
|
2469
|
+
* 'unknown' is DOWNGRADED just like 'alive', which is not obvious and is the point. A session
|
|
2470
|
+
* whose recorded `counterpartySessionPeerId` has gone stale never sees a matching peer-connect,
|
|
2471
|
+
* so it sits at 'unknown' while every send fails forever — the exact case documented at
|
|
2472
|
+
* #wireSessionLiveness — and the receive surface renders 'unknown' as healthy-and-quiet, which is
|
|
2473
|
+
* the 70-minute lie relocated one lane over. 'unknown' claims nothing; the surface built on it does.
|
|
2474
|
+
*/
|
|
2475
|
+
#markSessionImpaired(agentName, sessionId, opts) {
|
|
2476
|
+
const key = this.#k(agentName, sessionId);
|
|
2477
|
+
const prior = this.#sessionLiveness.get(key);
|
|
2478
|
+
if (prior === "gone") {
|
|
2479
|
+
// Declining is a decision, so it is logged. A silent early return here is the shape that let
|
|
2480
|
+
// the original defect hide for a day: nothing recorded that writes were failing on a session
|
|
2481
|
+
// every surface was still calling healthy.
|
|
2482
|
+
this.#logger.debug("session.liveness.impairment.declined", {
|
|
2483
|
+
sessionId, liveness: prior, cause: opts.cause, error: opts.error, correlationId: opts.correlationId,
|
|
2484
|
+
});
|
|
2485
|
+
return;
|
|
2486
|
+
}
|
|
2487
|
+
// The CAUSE is refreshed even when the state does not move, because the receive surface builds
|
|
2488
|
+
// its guidance from it and a stale cause would describe the wrong failure.
|
|
2489
|
+
this.#impairmentCause.set(key, { cause: opts.cause, retained: "unknown" });
|
|
2490
|
+
if (prior === "impaired")
|
|
2491
|
+
return;
|
|
2492
|
+
this.#sessionLiveness.set(key, "impaired");
|
|
2493
|
+
this.#logger.warn("session.liveness.changed", {
|
|
2494
|
+
sessionId,
|
|
2495
|
+
counterpartyPubkey: this.getSessionRecord(agentName, sessionId)?.counterparty_pubkey,
|
|
2496
|
+
transportPath: "direct",
|
|
2497
|
+
liveness: "impaired",
|
|
2498
|
+
observedBy: opts.cause,
|
|
2499
|
+
priorLiveness: prior ?? "unknown",
|
|
2500
|
+
// `reason` is a contract string and `error` is the message — the convention every other
|
|
2501
|
+
// failure log in this file follows. Collapsing them makes grouping by `reason` useless.
|
|
2502
|
+
reason: "write_failed",
|
|
2503
|
+
error: opts.error,
|
|
2504
|
+
correlationId: opts.correlationId,
|
|
2505
|
+
});
|
|
2506
|
+
}
|
|
2507
|
+
/**
|
|
2508
|
+
* DOD-M12B-ACK-1 — what became of the content whose send caused the impairment.
|
|
2509
|
+
*
|
|
2510
|
+
* The receive surface has no memory of the last send, so without this it can only guess — and the
|
|
2511
|
+
* guess it would make ("it was parked, do not resend") is FALSE in the two cases that matter
|
|
2512
|
+
* most: a refused park whose durable enqueue was dropped, and one that threw. In both the message
|
|
2513
|
+
* is gone and `cello_send` has already told the caller to send it again, so a receive that says
|
|
2514
|
+
* "do not resend" contradicts it later, while the agent is sitting there waiting.
|
|
2515
|
+
*/
|
|
2516
|
+
#noteImpairmentRetention(agentName, sessionId, retained) {
|
|
2517
|
+
const key = this.#k(agentName, sessionId);
|
|
2518
|
+
const current = this.#impairmentCause.get(key);
|
|
2519
|
+
if (!current)
|
|
2520
|
+
return;
|
|
2521
|
+
this.#impairmentCause.set(key, { cause: current.cause, retained });
|
|
2522
|
+
}
|
|
2523
|
+
/** DOD-M12B-ACK-1: why this session is impaired, for the surface that has to explain it. Null
|
|
2524
|
+
* when it is not impaired — a caller must not narrate a failure that is not current. */
|
|
2525
|
+
getSessionImpairment(agentName, sessionId) {
|
|
2526
|
+
const key = this.#k(agentName, sessionId);
|
|
2527
|
+
if (this.#sessionLiveness.get(key) !== "impaired")
|
|
2528
|
+
return null;
|
|
2529
|
+
return this.#impairmentCause.get(key) ?? null;
|
|
2530
|
+
}
|
|
2531
|
+
/**
|
|
2532
|
+
* DOD-M12B-ACK-1 — a delivery landed, so the impairment is over.
|
|
2533
|
+
*
|
|
2534
|
+
* Without this an `impaired` flag is a one-way door: one bad write would make a session report a
|
|
2535
|
+
* broken conversation for the rest of its life, which is the same class of lie in the other
|
|
2536
|
+
* direction. Called from BOTH send paths — an agent that mostly listens sends content rarely and
|
|
2537
|
+
* ACKs constantly, so clearing only on content would leave exactly those sessions impaired
|
|
2538
|
+
* forever. Only clears 'impaired': a successful write says nothing about a connection libp2p has
|
|
2539
|
+
* already declared 'gone'.
|
|
2540
|
+
*/
|
|
2541
|
+
#clearSessionImpairment(agentName, sessionId, observedBy, correlationId) {
|
|
2542
|
+
const key = this.#k(agentName, sessionId);
|
|
2543
|
+
if (this.#sessionLiveness.get(key) !== "impaired")
|
|
2544
|
+
return;
|
|
2545
|
+
this.#sessionLiveness.set(key, "alive");
|
|
2546
|
+
this.#impairmentCause.delete(key);
|
|
2547
|
+
this.#logger.info("session.liveness.changed", {
|
|
2548
|
+
sessionId,
|
|
2549
|
+
counterpartyPubkey: this.getSessionRecord(agentName, sessionId)?.counterparty_pubkey,
|
|
2550
|
+
transportPath: "direct",
|
|
2551
|
+
liveness: "alive",
|
|
2552
|
+
observedBy,
|
|
2553
|
+
reason: "write_succeeded",
|
|
2554
|
+
correlationId,
|
|
2555
|
+
});
|
|
2556
|
+
}
|
|
2557
|
+
/**
|
|
2558
|
+
* DOD-M12B-ABANDON-NOTIFY-1 — drive the REAL inbound content handler with one framed message and
|
|
2559
|
+
* a claimed peer identity.
|
|
2560
|
+
*
|
|
2561
|
+
* The handler is registered on a live libp2p node, so without this the only way to reach its
|
|
2562
|
+
* branches is a full two-node transport fixture — which is why the session-abandoned branch and
|
|
2563
|
+
* its peer pinning had no coverage at all. This feeds the same function the protocol handler
|
|
2564
|
+
* calls, including the authentication check, rather than a copy of its logic.
|
|
2565
|
+
*/
|
|
2566
|
+
async handleContentFrameForTest(agentName, sessionId, framedBytes, remotePeerId) {
|
|
2567
|
+
const source = {
|
|
2568
|
+
async *[Symbol.asyncIterator]() { yield framedBytes; },
|
|
2569
|
+
close: async () => { },
|
|
2570
|
+
abort: () => { },
|
|
2571
|
+
status: "closed",
|
|
2572
|
+
};
|
|
2573
|
+
await this.#handleContentStream(agentName, sessionId, source, remotePeerId);
|
|
2574
|
+
}
|
|
2239
2575
|
/** Test seam (same spirit as getDb()): seed per-session direct-path liveness, which is otherwise
|
|
2240
2576
|
* only set by the live node's onPeerConnect/onPeerDisconnect (#wireSessionLiveness). Lets a
|
|
2241
2577
|
* DB-seeded test exercise the CC-5 reaper's "alive counterparty must survive" gate without standing
|
|
@@ -2308,11 +2644,16 @@ export class SessionNodeManager {
|
|
|
2308
2644
|
"Check the daemon's database (disk space, permissions).",
|
|
2309
2645
|
};
|
|
2310
2646
|
}
|
|
2311
|
-
// Log observability event
|
|
2647
|
+
// Log observability event. `counterpartySessionPeerId` for the same reason as the initiator
|
|
2648
|
+
// side: this is the identity every later send will look for an open connection under, it is
|
|
2649
|
+
// never refreshed, and the peer's standing receiver may already have been rebuilt under a new
|
|
2650
|
+
// one. The RESPONDER is the side that can go stale — only the initiator dials, so this is the
|
|
2651
|
+
// half that inherits an id it never verified.
|
|
2312
2652
|
this.#logger.info("session.node.created", {
|
|
2313
2653
|
sessionId,
|
|
2314
2654
|
agentName,
|
|
2315
2655
|
sessionPeerId: peerId,
|
|
2656
|
+
counterpartySessionPeerId: initiatorPeerId,
|
|
2316
2657
|
correlationId,
|
|
2317
2658
|
});
|
|
2318
2659
|
// Remove this agent's standing receiver from the slot and add to active map. The handed-off
|
|
@@ -2500,22 +2841,69 @@ export class SessionNodeManager {
|
|
|
2500
2841
|
// destroyed with the session three seconds later. The sender then re-sent that envelope 90
|
|
2501
2842
|
// times against a ceiling of 5, and every surface reported the delivery as merely pending.
|
|
2502
2843
|
//
|
|
2503
|
-
//
|
|
2504
|
-
//
|
|
2844
|
+
// DOD-M12B-STRAND-1 — THIS IS NO LONGER AN EPITAPH. Dropping the Map now drops a CACHE: the
|
|
2845
|
+
// frames are rows in `held_content`, and #restoreHeldContent brings them back the next time
|
|
2846
|
+
// this session gets a node. `session.content.held.discarded` is kept, at WARN rather than
|
|
2847
|
+
// ERROR, and only for what it now means — a gap is still open on a session going away, which
|
|
2848
|
+
// is worth an alarm even though nothing is lost.
|
|
2849
|
+
//
|
|
2850
|
+
// It fires ONLY when the durable rows are confirmed present. A hold whose persist failed is a
|
|
2851
|
+
// genuine loss and must not be reported with the same event as a survivor, so it gets its own
|
|
2852
|
+
// error naming the count that is actually gone. The check is a COUNT against the store rather
|
|
2853
|
+
// than a belief about the write that ran earlier.
|
|
2505
2854
|
const strandedHolds = this.#heldContent.get(key);
|
|
2506
2855
|
if (strandedHolds && strandedHolds.size > 0) {
|
|
2507
|
-
|
|
2856
|
+
const canonicalSeqs = [...strandedHolds.keys()].sort((a, b) => a - b);
|
|
2857
|
+
// NULL means "we could not find out", and it must never render as "destroyed". The bare
|
|
2858
|
+
// catch this replaces coerced a failed COUNT to 0, which then claimed every held frame had
|
|
2859
|
+
// been lost — asserting a cause it had not established. It is not hypothetical:
|
|
2860
|
+
// #requireAgentId THROWS for a retired agent, on this exact path, so retiring an agent with
|
|
2861
|
+
// an open hold fabricated a data-loss alarm pointing at the persistence layer while the real
|
|
2862
|
+
// fault was name resolution.
|
|
2863
|
+
let durable = null;
|
|
2864
|
+
try {
|
|
2865
|
+
const row = this.#db?.prepare("SELECT COUNT(*) AS n FROM held_content WHERE agent_id = ? AND session_id = ?").get(this.#requireAgentId(agentName), sessionId);
|
|
2866
|
+
durable = row?.n ?? 0;
|
|
2867
|
+
}
|
|
2868
|
+
catch (err) {
|
|
2869
|
+
this.#logger.warn("session.content.held.durable_count.failed", {
|
|
2870
|
+
agentName, sessionId,
|
|
2871
|
+
impact: "cannot say whether the held frames are durable — reported as unknown, NOT as lost",
|
|
2872
|
+
error: err instanceof Error ? err.message : String(err),
|
|
2873
|
+
});
|
|
2874
|
+
}
|
|
2875
|
+
const lost = durable === null ? null : strandedHolds.size - durable;
|
|
2876
|
+
this.#logger.warn("session.content.held.discarded", {
|
|
2508
2877
|
agentName,
|
|
2509
2878
|
sessionId,
|
|
2510
2879
|
count: strandedHolds.size,
|
|
2511
|
-
|
|
2880
|
+
durable,
|
|
2881
|
+
canonicalSeqs,
|
|
2512
2882
|
// NULL when the tree was not cached at teardown. Honest, and cheap — reloading the leaf
|
|
2513
2883
|
// table to fill in a diagnostic field is not worth a disk read on every teardown, let
|
|
2514
2884
|
// alone the cache resurrection it caused.
|
|
2515
2885
|
treeSize: treeSizeBeforeEviction,
|
|
2516
2886
|
});
|
|
2887
|
+
if (lost !== null && lost > 0) {
|
|
2888
|
+
this.#logger.error("session.content.held.lost", {
|
|
2889
|
+
agentName,
|
|
2890
|
+
sessionId,
|
|
2891
|
+
lost,
|
|
2892
|
+
held: strandedHolds.size,
|
|
2893
|
+
impact: "verified content was NOT written to held_content and is destroyed by this teardown — the sender was never acknowledged and believes it is still pending",
|
|
2894
|
+
});
|
|
2895
|
+
}
|
|
2517
2896
|
}
|
|
2518
2897
|
this.#heldContent.delete(key);
|
|
2898
|
+
// DOD-M12B-STRAND-1: the hydration guard goes with the cache it guards. Without this, a session
|
|
2899
|
+
// torn down and given a node again inside ONE process would never re-read its durable holds —
|
|
2900
|
+
// the frames would sit in the table, unreleasable, which is indistinguishable from the loss
|
|
2901
|
+
// this unit exists to stop.
|
|
2902
|
+
this.#heldRestored.delete(key);
|
|
2903
|
+
this.#heldReleased.delete(key);
|
|
2904
|
+
this.#diverged.delete(key);
|
|
2905
|
+
this.#counterpartyAddrs.delete(key);
|
|
2906
|
+
this.#redialNotBefore.delete(key);
|
|
2519
2907
|
this.#highWaterSeq.delete(key);
|
|
2520
2908
|
}
|
|
2521
2909
|
/**
|
|
@@ -2523,6 +2911,37 @@ export class SessionNodeManager {
|
|
|
2523
2911
|
* Called from the SIGTERM / cello logout path (AC-009).
|
|
2524
2912
|
* SQLite writes complete before this method returns.
|
|
2525
2913
|
*/
|
|
2914
|
+
/**
|
|
2915
|
+
* DOD-M12B-SHUTDOWN-1 — wait for a teardown step, but never forever.
|
|
2916
|
+
*
|
|
2917
|
+
* Every step of shutdown used to be an unbounded `await` on libp2p. That is what makes "the
|
|
2918
|
+
* daemon acknowledged the request but is still running" possible: nothing on the daemon side
|
|
2919
|
+
* emits a word while it hangs, so the operator's own message ("it may be stuck closing sessions
|
|
2920
|
+
* or its database") was a guess. Past the deadline the step is ABANDONED and SAID — the resources
|
|
2921
|
+
* it was closing are reclaimed by the OS on exit, and an exit is worth more than a tidy one.
|
|
2922
|
+
*/
|
|
2923
|
+
async #boundedTeardown(work, step, count) {
|
|
2924
|
+
if (count === 0)
|
|
2925
|
+
return;
|
|
2926
|
+
const started = Date.now();
|
|
2927
|
+
let timer;
|
|
2928
|
+
const deadline = new Promise((resolve) => {
|
|
2929
|
+
timer = setTimeout(() => resolve("timeout"), SHUTDOWN_STEP_DEADLINE_MS);
|
|
2930
|
+
timer.unref?.();
|
|
2931
|
+
});
|
|
2932
|
+
const outcome = await Promise.race([work.then(() => "done"), deadline]);
|
|
2933
|
+
if (timer)
|
|
2934
|
+
clearTimeout(timer);
|
|
2935
|
+
if (outcome === "timeout") {
|
|
2936
|
+
this.#logger.error("session.shutdown.step.timeout", {
|
|
2937
|
+
step, count, waitedMs: Date.now() - started,
|
|
2938
|
+
impact: "this teardown step did not finish and was abandoned so the daemon can exit; the OS reclaims what it held",
|
|
2939
|
+
});
|
|
2940
|
+
}
|
|
2941
|
+
else {
|
|
2942
|
+
this.#logger.debug("session.shutdown.step.done", { step, count, tookMs: Date.now() - started });
|
|
2943
|
+
}
|
|
2944
|
+
}
|
|
2526
2945
|
async gracefulShutdown() {
|
|
2527
2946
|
// DOD-NAT-REACHABILITY-1: stop the reservation watchdog before anything is torn
|
|
2528
2947
|
// down — a tick landing mid-shutdown would try to rebuild a receiver we are in
|
|
@@ -2533,6 +2952,11 @@ export class SessionNodeManager {
|
|
|
2533
2952
|
}
|
|
2534
2953
|
// Signal any in-flight standing-receiver replacement to self-stop (review M2).
|
|
2535
2954
|
this.#shuttingDown = true;
|
|
2955
|
+
// DOD-M12B-ACK-1: drop the inbound-content linger resets. The nodes they would reset are being
|
|
2956
|
+
// torn down here anyway, so firing after this point is pure noise on the way out.
|
|
2957
|
+
for (const timer of this.#lingeringStreams)
|
|
2958
|
+
clearTimeout(timer);
|
|
2959
|
+
this.#lingeringStreams.clear();
|
|
2536
2960
|
// Cancel every armed awaiting-ACK timer so an un-acked send (e.g. a rejected /
|
|
2537
2961
|
// tampered frame that never produced a `persisted` ACK) does not leave a 20s
|
|
2538
2962
|
// timer pinning the content + this manager in memory past teardown (review M1).
|
|
@@ -2587,14 +3011,22 @@ export class SessionNodeManager {
|
|
|
2587
3011
|
});
|
|
2588
3012
|
}));
|
|
2589
3013
|
}
|
|
2590
|
-
|
|
3014
|
+
// DOD-M12B-SHUTDOWN-1: BOUNDED. `node.stop()` awaits libp2p's own teardown, which has no
|
|
3015
|
+
// deadline of its own — one connection that will not close holds this, and this holds the whole
|
|
3016
|
+
// daemon. Measured 2026-08-17: `cello logout` acknowledged, then the process was still alive
|
|
3017
|
+
// 30+ seconds later and needed a signal. An abandoned stop costs a socket the OS reclaims when
|
|
3018
|
+
// we exit; an unbounded wait costs the exit itself.
|
|
3019
|
+
await this.#boundedTeardown(Promise.all(stopPromises), "session_nodes", stopPromises.length);
|
|
2591
3020
|
this.#activeNodes.clear();
|
|
2592
3021
|
// Evict in-memory per-session caches (trees reload from SQLite; received-content
|
|
2593
3022
|
// plaintext must not survive shutdown in memory).
|
|
2594
3023
|
this.#trees.clear();
|
|
2595
3024
|
this.#receivedContent.clear();
|
|
2596
|
-
// Stop ALL per-agent standing receivers (DOD-LOOP-1).
|
|
2597
|
-
|
|
3025
|
+
// Stop ALL per-agent standing receivers (DOD-LOOP-1). In PARALLEL and BOUNDED: this was a
|
|
3026
|
+
// sequential await per agent with no deadline, so five agents meant five chances for one stuck
|
|
3027
|
+
// libp2p teardown to hold the exit — and it sits between the operator being told the daemon is
|
|
3028
|
+
// stopping and the process actually going.
|
|
3029
|
+
await this.#boundedTeardown(Promise.all([...this.#standingReceivers].map(async ([agentName, sr]) => {
|
|
2598
3030
|
sr.autoNat.stop();
|
|
2599
3031
|
try {
|
|
2600
3032
|
await sr.node.stop();
|
|
@@ -2607,7 +3039,7 @@ export class SessionNodeManager {
|
|
|
2607
3039
|
correlationId: "n/a",
|
|
2608
3040
|
});
|
|
2609
3041
|
}
|
|
2610
|
-
}
|
|
3042
|
+
})), "standing_receivers", this.#standingReceivers.size);
|
|
2611
3043
|
this.#standingReceivers.clear();
|
|
2612
3044
|
// Release the SQLite handle so the DB file is no longer held open after shutdown
|
|
2613
3045
|
// (review L5). Queries guard on `#db === null` and degrade to empty/null.
|
|
@@ -3174,6 +3606,9 @@ export class SessionNodeManager {
|
|
|
3174
3606
|
for (const addr of addrs) {
|
|
3175
3607
|
try {
|
|
3176
3608
|
await entry.node.dial(addr);
|
|
3609
|
+
// DOD-M12B-REDIAL-1: keep them. They arrived in the signed assignment and were used once
|
|
3610
|
+
// and dropped, which is the reason nothing could ever dial this counterparty again.
|
|
3611
|
+
this.#counterpartyAddrs.set(this.#k(agentName, sessionId), [...addrs]);
|
|
3177
3612
|
this.#logger.info("session.transport.connected", {
|
|
3178
3613
|
sessionId,
|
|
3179
3614
|
addr,
|
|
@@ -3246,6 +3681,8 @@ export class SessionNodeManager {
|
|
|
3246
3681
|
// the leaf_deliver witness stream / arrival order.
|
|
3247
3682
|
let orderingS1;
|
|
3248
3683
|
let orderingS2;
|
|
3684
|
+
// DOD-M12B-INDEX-1: the relay's answer to "where does this message go", carried to the caller.
|
|
3685
|
+
let assignedSeq;
|
|
3249
3686
|
// DOD-MP-SESSION-RETIRE-1 — the relay's answer SURVIVES to the caller even when the direct send
|
|
3250
3687
|
// then succeeds. `relay_session_gone` is deliberately not terminal (it also fires for perfectly
|
|
3251
3688
|
// live sessions whenever the relay restarts, because the relay stores sessions in memory), so
|
|
@@ -3263,9 +3700,20 @@ export class SessionNodeManager {
|
|
|
3263
3700
|
if (witnessed.ok) {
|
|
3264
3701
|
orderingS1 = witnessed.structure1_cbor;
|
|
3265
3702
|
orderingS2 = witnessed.structure2_cbor;
|
|
3703
|
+
// 1-BASED → 0-BASED. The relay numbers the first leaf of a session 1
|
|
3704
|
+
// (`relay-node.ts`: `const seq = state.seq_counter + 1`), and this tree is 0-indexed.
|
|
3705
|
+
// Every RECEIVE path in this file normalises with -1 and says so; the send path took the
|
|
3706
|
+
// raw number, which puts every comparison against `tree.size()` one position out — so a
|
|
3707
|
+
// perfectly healthy first message reads as "ahead of the tail" and is held behind a gap
|
|
3708
|
+
// that does not exist. Do not remove this without changing both receive sites too.
|
|
3709
|
+
assignedSeq = witnessed.sequence_number - 1;
|
|
3266
3710
|
this.#logger.info("session.relay.hash.submitted", {
|
|
3267
3711
|
sessionId,
|
|
3268
|
-
|
|
3712
|
+
// BOTH SPACES, NAMED. The relay's number is 1-based and the leaf index is 0-based, and
|
|
3713
|
+
// reading one as the other is the defect this milestone exists to stop — so a log that
|
|
3714
|
+
// carries only "sequenceNumber" invites exactly that mistake on the next investigation.
|
|
3715
|
+
relaySequence: witnessed.sequence_number,
|
|
3716
|
+
leafIndex: assignedSeq,
|
|
3269
3717
|
correlationId,
|
|
3270
3718
|
});
|
|
3271
3719
|
}
|
|
@@ -3353,8 +3801,14 @@ export class SessionNodeManager {
|
|
|
3353
3801
|
// resolves the awaiting timer; on failure (counterparty offline) the hash is already
|
|
3354
3802
|
// witnessed above, so the caller / TTF path parks the SEALED content to the relay
|
|
3355
3803
|
// store-and-forward backstop and the recipient recovers it at the witnessed sequence (2b).
|
|
3804
|
+
// Held outside the try so the catch can retire a stream that was opened and then failed to
|
|
3805
|
+
// write. Without it every failure leaks the OUTBOUND half of the stream the receiver-side
|
|
3806
|
+
// `finally` retires — same defect, other end, other cap (64 outbound per protocol per
|
|
3807
|
+
// connection). See the note on #handleContentStream's finally.
|
|
3808
|
+
let sendStream;
|
|
3356
3809
|
try {
|
|
3357
|
-
const stream = await
|
|
3810
|
+
const stream = await this.#openContentStream(agentName, sessionId, entry, correlationId);
|
|
3811
|
+
sendStream = stream;
|
|
3358
3812
|
// AC-001/AC-003: arm the TTF tracking BEFORE the frame goes on the wire. The
|
|
3359
3813
|
// receiver's `persisted` ACK can come back fast (in-process / low-latency
|
|
3360
3814
|
// transports), so registering the awaiting entry after send would let the ACK
|
|
@@ -3399,9 +3853,17 @@ export class SessionNodeManager {
|
|
|
3399
3853
|
// A close that failed for a benign reason costs a redundant park, which the receiver dedups
|
|
3400
3854
|
// on the content hash. A false delivered costs the message.
|
|
3401
3855
|
await stream.close();
|
|
3402
|
-
|
|
3856
|
+
this.#clearSessionImpairment(agentName, sessionId, "direct_send", correlationId);
|
|
3857
|
+
return { ok: true, delivered: true, ...(assignedSeq === undefined ? {} : { sequenceNumber: assignedSeq }), ...(relayRefusal === undefined ? {} : { relayRefusal }) };
|
|
3403
3858
|
}
|
|
3404
3859
|
catch (err) {
|
|
3860
|
+
this.#markSessionImpaired(agentName, sessionId, { cause: "direct_send", error: err instanceof Error ? err.message : String(err), correlationId });
|
|
3861
|
+
if (sendStream !== undefined) {
|
|
3862
|
+
try {
|
|
3863
|
+
sendStream.abort(err instanceof Error ? err : new Error(String(err)));
|
|
3864
|
+
}
|
|
3865
|
+
catch { /* already gone */ }
|
|
3866
|
+
}
|
|
3405
3867
|
// The send failed after (possibly) arming the awaiting tracking — drop it so a
|
|
3406
3868
|
// never-delivered frame does not later fire a spurious TTF park.
|
|
3407
3869
|
this.#untrackAwaitingAck(agentName, sessionId, contentHash);
|
|
@@ -3412,9 +3874,31 @@ export class SessionNodeManager {
|
|
|
3412
3874
|
// can be reported as "dispatched to relay" instead of a raw stream failure — the operator/
|
|
3413
3875
|
// agent sees the truth (the message IS in flight, just not direct), not a false negative.
|
|
3414
3876
|
const hashHex = Buffer.from(contentHash).toString("hex");
|
|
3877
|
+
// NAME THE CAUSE. This catch used to discard `err` outright, so a park reported only its exit
|
|
3878
|
+
// point — "dispatched to relay" — and never what went wrong. Measured 2026-08-17: 212 parks on
|
|
3879
|
+
// one daemon, not one of them recording a reason, which is what made a one-way session look
|
|
3880
|
+
// like a protocol mystery for a night.
|
|
3881
|
+
//
|
|
3882
|
+
// `counterpartySessionPeerId` is the load-bearing field. It is recorded ONCE at session
|
|
3883
|
+
// establishment and never refreshed, while a standing receiver is rebuilt with a fresh keypair
|
|
3884
|
+
// on every signaling reconnect — so if the two ever cross, every send goes one-way forever and
|
|
3885
|
+
// nothing says so. With this line that becomes a single grep instead of a night.
|
|
3886
|
+
this.#logger.warn("session.content.direct.send.failed", {
|
|
3887
|
+
agentName,
|
|
3888
|
+
sessionId,
|
|
3889
|
+
contentHash: hashHex,
|
|
3890
|
+
counterpartySessionPeerId: entry.counterpartySessionPeerId,
|
|
3891
|
+
error: err instanceof Error ? err.message : String(err),
|
|
3892
|
+
// "Cannot write to a stream that is closed" names where the write died, never why. The
|
|
3893
|
+
// why is almost always the per-protocol stream cap, and these two numbers are what turn
|
|
3894
|
+
// that from a log-measurement session into a grep.
|
|
3895
|
+
...this.#streamCensus(entry.node, entry.counterpartySessionPeerId),
|
|
3896
|
+
correlationId,
|
|
3897
|
+
});
|
|
3415
3898
|
const attempt = await this.#parkContent(agentName, sessionId, hashHex, content, orderingS1, orderingS2);
|
|
3416
3899
|
if (attempt.outcome === "parked") {
|
|
3417
|
-
|
|
3900
|
+
this.#noteImpairmentRetention(agentName, sessionId, "parked");
|
|
3901
|
+
return { ok: true, delivered: false, parked: true, ...(assignedSeq === undefined ? {} : { sequenceNumber: assignedSeq }), ...(relayRefusal === undefined ? {} : { relayRefusal }) };
|
|
3418
3902
|
}
|
|
3419
3903
|
// M12-P12: the deposit was refused, and #untrackAwaitingAck above already dropped the
|
|
3420
3904
|
// in-memory entry — so without this, NOTHING holds the content and the TTF timer that would
|
|
@@ -3491,10 +3975,18 @@ export class SessionNodeManager {
|
|
|
3491
3975
|
// F3: the two failures are NOT interchangeable to the caller. `reason` is a contract string
|
|
3492
3976
|
// and stays put; `guidance` carries the difference, because "we are retrying this" and "this
|
|
3493
3977
|
// message is gone, send it again" demand opposite actions from the operator.
|
|
3978
|
+
//
|
|
3979
|
+
// The same distinction is recorded on the session, because `cello_receive` will be asked
|
|
3980
|
+
// about this later and would otherwise have to guess — and its guess ("it was parked, do not
|
|
3981
|
+
// resend") is the exact opposite of what the lost case needs.
|
|
3982
|
+
this.#noteImpairmentRetention(agentName, sessionId, durable ? "durable" : "lost");
|
|
3494
3983
|
return {
|
|
3495
3984
|
ok: false,
|
|
3496
3985
|
reason: "session_stream_unavailable",
|
|
3497
3986
|
error: errMsg,
|
|
3987
|
+
// Carried on the failure path too: a DURABLY QUEUED message still owns the position the
|
|
3988
|
+
// relay witnessed for it before delivery was attempted, and its leaf must go there.
|
|
3989
|
+
...(assignedSeq === undefined ? {} : { sequenceNumber: assignedSeq }),
|
|
3498
3990
|
// M12-P13: the machine-readable half of the distinction below. M12-P12 shipped it in the
|
|
3499
3991
|
// guidance SENTENCE only, so the callers that have to ACT on it — commit the leaf for a
|
|
3500
3992
|
// queued message, never for a lost one — would have had to substring-match English. None
|
|
@@ -4129,6 +4621,7 @@ export class SessionNodeManager {
|
|
|
4129
4621
|
: this.#witnessedSeq.get(key)?.get(contentHashHex);
|
|
4130
4622
|
const nextExpected = this.getSessionTree(agentName, sessionId).size();
|
|
4131
4623
|
if (canonicalSeq !== undefined && canonicalSeq > nextExpected) {
|
|
4624
|
+
this.#ensureHeldRestored(agentName, sessionId);
|
|
4132
4625
|
let held = this.#heldContent.get(key);
|
|
4133
4626
|
if (!held) {
|
|
4134
4627
|
held = new Map();
|
|
@@ -4137,7 +4630,15 @@ export class SessionNodeManager {
|
|
|
4137
4630
|
// A terminal block out of canonical order is held WITHOUT delivery (screenedOut): #releaseHeld
|
|
4138
4631
|
// leafs it at its canonical index when the gap fills, but never buffers it for the agent. This
|
|
4139
4632
|
// keeps leafIndex === canonicalSeq for screened-out content too (code-review HIGH-1).
|
|
4140
|
-
|
|
4633
|
+
// THE PEER'S RAW BYTES RIDE ALONG. Classification (document frame vs conversation) reads
|
|
4634
|
+
// byte 0, and `deliverContent` is the SCREENED copy — for a CBOR frame that is no longer a
|
|
4635
|
+
// map header, so a held document frame was released into the CONVERSATION path: transcript,
|
|
4636
|
+
// doorbell, and `cello_receive` handing an agent raw CBOR as though a person typed it.
|
|
4637
|
+
// The in-order path has always passed these bytes; only the held path dropped them.
|
|
4638
|
+
held.set(canonicalSeq, { content: deliverContent, originalContent: content, contentHashHex, correlationId, ...(terminalBlock ? { screenedOut: true } : {}) });
|
|
4639
|
+
// DOD-M12B-STRAND-1: and to disk, before we answer. The in-memory Map is the working copy;
|
|
4640
|
+
// this row is the one that survives the teardown that used to destroy it.
|
|
4641
|
+
this.#persistHeldContent(agentName, sessionId, canonicalSeq, deliverContent, content, contentHashHex, terminalBlock === true, correlationId);
|
|
4141
4642
|
this.#logger.info("session.content.held", {
|
|
4142
4643
|
sessionId,
|
|
4143
4644
|
canonicalSeq,
|
|
@@ -4230,6 +4731,9 @@ export class SessionNodeManager {
|
|
|
4230
4731
|
if (sequenceNumber < 0)
|
|
4231
4732
|
return;
|
|
4232
4733
|
const key = this.#k(agentName, sessionId);
|
|
4734
|
+
// DOD-M12B-SEAL-STUCK-1: this process has now seen this session's ordering state, so an empty
|
|
4735
|
+
// witness map for it means "no gap" rather than "never looked".
|
|
4736
|
+
this.#orderingObserved.add(key);
|
|
4233
4737
|
let map = this.#witnessedSeq.get(key);
|
|
4234
4738
|
if (!map) {
|
|
4235
4739
|
map = new Map();
|
|
@@ -4286,6 +4790,16 @@ export class SessionNodeManager {
|
|
|
4286
4790
|
*/
|
|
4287
4791
|
sealReadiness(agentName, sessionId) {
|
|
4288
4792
|
const key = this.#k(agentName, sessionId);
|
|
4793
|
+
// DOD-M12B-STRAND-1: hydrate first. An under-counted `heldCount` reports a gapped session as
|
|
4794
|
+
// READY, and this gate's whole purpose is to stop a short chain being signed — the counterparty
|
|
4795
|
+
// answers `leaf_count_mismatch`, which is TERMINAL and costs the receipt permanently. Failing
|
|
4796
|
+
// open here is the one outcome worse than refusing a healthy close.
|
|
4797
|
+
//
|
|
4798
|
+
// Hydrate WITHOUT releasing: this is a read path now — the status surface asks it for every
|
|
4799
|
+
// active session — and a read that appends leaves, advances the root and rings the doorbell is
|
|
4800
|
+
// a diagnostic command that delivers messages. The release still runs on every path that was
|
|
4801
|
+
// going to mutate anyway.
|
|
4802
|
+
this.#ensureHeldRestored(agentName, sessionId, { release: false });
|
|
4289
4803
|
const treeSize = this.getSessionTree(agentName, sessionId).size();
|
|
4290
4804
|
const highWaterSeq = this.#highWaterSeq.get(key) ?? -1;
|
|
4291
4805
|
const heldCount = this.#heldContent.get(key)?.size ?? 0;
|
|
@@ -4305,7 +4819,107 @@ export class SessionNodeManager {
|
|
|
4305
4819
|
// ordering authority has committed and this tree has not — no arithmetic, no space mismatch,
|
|
4306
4820
|
// and it cannot go negative.
|
|
4307
4821
|
const missingLeaves = this.#witnessedSeq.get(key)?.size ?? 0;
|
|
4308
|
-
|
|
4822
|
+
// DOD-M12B-INDEX-1: our OWN held sends are counted separately. They block a seal just as a
|
|
4823
|
+
// received hold does, but they are not "a message from the counterparty that has not arrived" —
|
|
4824
|
+
// and a refusal that calls them that tells the operator to wait for something already in hand,
|
|
4825
|
+
// which is how a close-retry loop ends at force-abandon and no receipt.
|
|
4826
|
+
let heldOwn = 0;
|
|
4827
|
+
for (const e of this.#heldContent.get(key)?.values() ?? [])
|
|
4828
|
+
if (e.origin === "sent")
|
|
4829
|
+
heldOwn++;
|
|
4830
|
+
return {
|
|
4831
|
+
ready: missingLeaves === 0 && heldCount === 0,
|
|
4832
|
+
treeSize, highWaterSeq, heldCount, missingLeaves,
|
|
4833
|
+
heldOwn, heldReceived: heldCount - heldOwn,
|
|
4834
|
+
};
|
|
4835
|
+
}
|
|
4836
|
+
/**
|
|
4837
|
+
* DOD-M12B-SEAL-STUCK-1 — the operator-facing answer to "can this session be closed?".
|
|
4838
|
+
*
|
|
4839
|
+
* THREE STATES, because there are three answers. `sealReadiness` above returns a boolean plus raw
|
|
4840
|
+
* counters, and both of its counters are easy to misread on a surface:
|
|
4841
|
+
*
|
|
4842
|
+
* - `missingLeaves` is `#witnessedSeq.size`, which is every position the relay witnessed that
|
|
4843
|
+
* this tree has not appended — and a HELD frame keeps its witness entry. So it INCLUDES the
|
|
4844
|
+
* held ones. Reporting it beside `heldCount` counts the same message twice and labels one copy
|
|
4845
|
+
* "never received" when it is sitting on our own disk. Split here into what each actually is.
|
|
4846
|
+
* - Neither counter survives a restart on its own: `#witnessedSeq` is memory-only. Held content
|
|
4847
|
+
* is durable since DOD-M12B-STRAND-1, but a position the relay witnessed for content that
|
|
4848
|
+
* never arrived leaves no trace. So for a session carrying leaves this process did not watch
|
|
4849
|
+
* arrive, "clean" is unknowable — and saying `ready` there invites a close that gets
|
|
4850
|
+
* `leaf_count_mismatch` back, which is terminal and costs the receipt for good.
|
|
4851
|
+
*/
|
|
4852
|
+
sealReadinessView(agentName, sessionId) {
|
|
4853
|
+
const key = this.#k(agentName, sessionId);
|
|
4854
|
+
const r = this.sealReadiness(agentName, sessionId);
|
|
4855
|
+
if (!r.ready) {
|
|
4856
|
+
const oldestHeldMs = this.#oldestHeldMs(agentName, sessionId);
|
|
4857
|
+
return {
|
|
4858
|
+
state: "blocked",
|
|
4859
|
+
// The witness map counts a held frame until it is appended, so subtract the RECEIVED holds
|
|
4860
|
+
// to avoid reporting one message twice. NOT the own-sends: the witness map only ever
|
|
4861
|
+
// carries counterparty leaves, so subtracting ours would push this count below the truth.
|
|
4862
|
+
awaitingArrival: Math.max(0, r.missingLeaves - r.heldReceived),
|
|
4863
|
+
heldBehindGap: r.heldCount,
|
|
4864
|
+
oldestHeldMs,
|
|
4865
|
+
};
|
|
4866
|
+
}
|
|
4867
|
+
if (this.#diverged.has(key)) {
|
|
4868
|
+
// NOT `ready`. The tree is ahead of the relay's counter for good, so a close here signs a root
|
|
4869
|
+
// the counterparty answers `leaf_count_mismatch` to — terminal, and the receipt is gone. The
|
|
4870
|
+
// raw counters cannot see this: nothing is missing and nothing is held.
|
|
4871
|
+
return { state: "unknown", reason: "record_diverged_from_relay" };
|
|
4872
|
+
}
|
|
4873
|
+
if (r.treeSize > 0 && !this.#orderingObserved.has(key)) {
|
|
4874
|
+
return {
|
|
4875
|
+
state: "unknown",
|
|
4876
|
+
reason: "witness_state_predates_daemon_start",
|
|
4877
|
+
};
|
|
4878
|
+
}
|
|
4879
|
+
return { state: "ready" };
|
|
4880
|
+
}
|
|
4881
|
+
/** DOD-M12B-INDEX-1 — this agent's own K_local pubkey, for attributing its own held content.
|
|
4882
|
+
* Null when it cannot be resolved: an UNATTRIBUTED annex row is true, a falsely attributed one
|
|
4883
|
+
* is not, and this is the record that outlives the session. */
|
|
4884
|
+
#ownPubkeyHex(agentName) {
|
|
4885
|
+
if (!this.#db)
|
|
4886
|
+
return null;
|
|
4887
|
+
try {
|
|
4888
|
+
// BY agent_id, never by agent_name. The name is a mutable, reuse-freed display label, and
|
|
4889
|
+
// scoping on it hands one identity's rows to another keypair (DOD-AGENT-ID-JOINKEY-1).
|
|
4890
|
+
const row = this.#db
|
|
4891
|
+
.prepare("SELECT k_local_pubkey FROM agents WHERE agent_id = ?")
|
|
4892
|
+
.get(this.#requireAgentId(agentName));
|
|
4893
|
+
return row?.k_local_pubkey ?? null;
|
|
4894
|
+
}
|
|
4895
|
+
catch (err) {
|
|
4896
|
+
// An unattributed annex row is truthful; an unattributed row nobody knows about is not. This
|
|
4897
|
+
// throws for a retired agent, and without a line here EVERY own held message would land in
|
|
4898
|
+
// the record that outlives the session with no sender and no explanation.
|
|
4899
|
+
this.#logger.warn("session.own_pubkey.unresolved", {
|
|
4900
|
+
agentName,
|
|
4901
|
+
error: err instanceof Error ? err.message : String(err),
|
|
4902
|
+
impact: "this agent's own held content will be annexed without a sender",
|
|
4903
|
+
});
|
|
4904
|
+
return null;
|
|
4905
|
+
}
|
|
4906
|
+
}
|
|
4907
|
+
/** DOD-M12B-SEAL-STUCK-1 — how long the oldest held frame for this session has been waiting, or
|
|
4908
|
+
* null when nothing is held. This is what separates "stuck since this morning" from "in flight
|
|
4909
|
+
* 40 ms ago", and without it a healthy mid-conversation window reads as a stranded session. */
|
|
4910
|
+
#oldestHeldMs(agentName, sessionId) {
|
|
4911
|
+
if (!this.#db)
|
|
4912
|
+
return null;
|
|
4913
|
+
try {
|
|
4914
|
+
const row = this.#db.prepare("SELECT MIN(held_at) AS oldest FROM held_content WHERE agent_id = ? AND session_id = ?").get(this.#requireAgentId(agentName), sessionId);
|
|
4915
|
+
if (!row || row.oldest === null)
|
|
4916
|
+
return null;
|
|
4917
|
+
return Date.now() - row.oldest;
|
|
4918
|
+
}
|
|
4919
|
+
catch {
|
|
4920
|
+
// A diagnostic detail must not be able to break the surface it decorates.
|
|
4921
|
+
return null;
|
|
4922
|
+
}
|
|
4309
4923
|
}
|
|
4310
4924
|
/** DOD-MSG-4 / DAEMON-004: append a verified message leaf and buffer it for cello_receive. */
|
|
4311
4925
|
#appendVerifiedContent(agentName, sessionId, content, contentHashHex, senderPubkey, correlationId,
|
|
@@ -4429,6 +5043,485 @@ export class SessionNodeManager {
|
|
|
4429
5043
|
}
|
|
4430
5044
|
return { leafIndex };
|
|
4431
5045
|
}
|
|
5046
|
+
/**
|
|
5047
|
+
* DOD-M12B-ABANDON-NOTIFY-1 — tell the counterparty we have hung up. Best effort, never blocking.
|
|
5048
|
+
*
|
|
5049
|
+
* A force-abandon marks the session terminal HERE and did nothing else, so the other side kept
|
|
5050
|
+
* its half live, kept retrying delivery into it, and kept trying to re-establish — forever,
|
|
5051
|
+
* because nothing would ever answer. That is what produced the 2026-08-17 notification storm:
|
|
5052
|
+
* surviving halves calling continuously while the operator saw connection requests from agents
|
|
5053
|
+
* nobody was driving.
|
|
5054
|
+
*
|
|
5055
|
+
* BEST EFFORT, and every caller must treat it that way. A peer that is offline cannot be told, so
|
|
5056
|
+
* this is an improvement on silence rather than a guarantee — and it must never delay or fail the
|
|
5057
|
+
* abandon, which is the operator's escape hatch out of a session that can never seal.
|
|
5058
|
+
*/
|
|
5059
|
+
async notifyCounterpartyAbandon(agentName, sessionId, correlationId) {
|
|
5060
|
+
const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
|
|
5061
|
+
if (!entry) {
|
|
5062
|
+
// NAMES ITS CAUSE, and it is not the network. An `interrupted` session has no node — the
|
|
5063
|
+
// restart sweep and markInterrupted both tear it down — and `interrupted` is exactly the
|
|
5064
|
+
// status force-abandon exists for. Reporting this as "could not be reached" sends the
|
|
5065
|
+
// operator to debug a connection when the answer is in our own process. At INFO, not debug,
|
|
5066
|
+
// because it is the common case and it changes what the operator is told.
|
|
5067
|
+
this.#logger.info("session.abandon.notice.skipped", {
|
|
5068
|
+
agentName, sessionId, reason: "no_local_node", correlationId,
|
|
5069
|
+
impact: "this side had already torn the session down, so there was nothing to send on — the counterparty was not told",
|
|
5070
|
+
});
|
|
5071
|
+
return { told: false, reason: "no_local_node" };
|
|
5072
|
+
}
|
|
5073
|
+
let stream;
|
|
5074
|
+
try {
|
|
5075
|
+
// Through the RE-DIAL path, not a bare newStream. A session worth force-abandoning is very
|
|
5076
|
+
// often one whose connection blipped — the peer is online and calling us, which is the whole
|
|
5077
|
+
// complaint — so one demand-driven dial is the difference between telling them and not.
|
|
5078
|
+
stream = await this.#openContentStream(agentName, sessionId, entry, correlationId);
|
|
5079
|
+
// Typed against protocol-types so the shape cannot drift from the declaration the receiving
|
|
5080
|
+
// side (and any second client implementation) reads.
|
|
5081
|
+
const notice = {
|
|
5082
|
+
type: "session_abandoned_notice",
|
|
5083
|
+
session_id: sessionId,
|
|
5084
|
+
...(correlationId === undefined ? {} : { correlation_id: correlationId }),
|
|
5085
|
+
};
|
|
5086
|
+
const frame = encodeCbor(notice);
|
|
5087
|
+
stream.send(lp.encode.single(frame));
|
|
5088
|
+
await stream.close();
|
|
5089
|
+
this.#logger.info("session.abandon.notice.sent", { agentName, sessionId, correlationId });
|
|
5090
|
+
return { told: true, reason: "sent" };
|
|
5091
|
+
}
|
|
5092
|
+
catch (err) {
|
|
5093
|
+
if (stream !== undefined) {
|
|
5094
|
+
try {
|
|
5095
|
+
stream.abort(err instanceof Error ? err : new Error(String(err)));
|
|
5096
|
+
}
|
|
5097
|
+
catch { /* already gone */ }
|
|
5098
|
+
}
|
|
5099
|
+
this.#logger.warn("session.abandon.notice.failed", {
|
|
5100
|
+
agentName, sessionId, correlationId,
|
|
5101
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5102
|
+
impact: "the counterparty was not told and may keep calling until it gives up",
|
|
5103
|
+
});
|
|
5104
|
+
return { told: false, reason: "send_failed" };
|
|
5105
|
+
}
|
|
5106
|
+
}
|
|
5107
|
+
/**
|
|
5108
|
+
* DOD-M12B-ABANDON-NOTIFY-1 — the receiving half: our counterparty has abandoned, so retire.
|
|
5109
|
+
*
|
|
5110
|
+
* RETIRING IS NOT DELETING. The counterparty walking away forfeits the notarized receipt; it must
|
|
5111
|
+
* not also cost the operator the record of what was actually said. The transcript and the tree
|
|
5112
|
+
* stay exactly as they are.
|
|
5113
|
+
*
|
|
5114
|
+
* Only an `active` or `interrupted` session moves. A SEALED session has a notarized receipt and
|
|
5115
|
+
* must never be turned into an abandoned one by a late or duplicated notice — that would destroy
|
|
5116
|
+
* the artifact this protocol exists to produce. An unknown session is refused rather than
|
|
5117
|
+
* created: an authenticated stream proves who is speaking, not that a session exists.
|
|
5118
|
+
*/
|
|
5119
|
+
async retireOnCounterpartyAbandon(agentName, sessionId, correlationId) {
|
|
5120
|
+
const record = this.getSessionRecord(agentName, sessionId);
|
|
5121
|
+
if (!record) {
|
|
5122
|
+
this.#logger.warn("session.abandon.notice.unknown_session", { agentName, sessionId, correlationId });
|
|
5123
|
+
return false;
|
|
5124
|
+
}
|
|
5125
|
+
if (record.status !== "active" && record.status !== "interrupted") {
|
|
5126
|
+
this.#logger.debug("session.abandon.notice.ignored", {
|
|
5127
|
+
agentName, sessionId, status: record.status, correlationId,
|
|
5128
|
+
reason: "session already terminal",
|
|
5129
|
+
});
|
|
5130
|
+
return false;
|
|
5131
|
+
}
|
|
5132
|
+
// THE TRANSPORT IS RETIRED. THE SESSION IS NOT.
|
|
5133
|
+
//
|
|
5134
|
+
// The first build flipped the status to `abandoned`, and that was wrong twice over. It handed
|
|
5135
|
+
// the abandoning party a button that DENIES US OUR RECEIPT: the unilateral seal exists for
|
|
5136
|
+
// exactly this case — "the counterparty never co-closes" — and produces a notarized certificate
|
|
5137
|
+
// after a grace period, but `cello_close_session` refuses an `abandoned` session outright. So
|
|
5138
|
+
// one frame from them destroyed a recovery path that already existed, remotely and for free.
|
|
5139
|
+
// Today the abandoner can only go silent, and going silent is what the unilateral seal was
|
|
5140
|
+
// built to survive.
|
|
5141
|
+
//
|
|
5142
|
+
// What the DoD actually asks for is that we stop calling them. That is a transport concern:
|
|
5143
|
+
// mark it, stop re-dialling, stop retrying delivery — and leave the session sealable.
|
|
5144
|
+
const marked = this.#markCounterpartyAbandoned(agentName, sessionId);
|
|
5145
|
+
if (!marked)
|
|
5146
|
+
return false;
|
|
5147
|
+
// The addresses go, so the demand-driven re-dial has nothing to dial. This is the storm.
|
|
5148
|
+
const key = this.#k(agentName, sessionId);
|
|
5149
|
+
this.#counterpartyAddrs.delete(key);
|
|
5150
|
+
this.#logger.warn("session.counterparty.abandoned", {
|
|
5151
|
+
agentName, sessionId, priorStatus: record.status, correlationId,
|
|
5152
|
+
impact: "the counterparty ended this session on their side, so nothing more will arrive and replies cannot reach them — this side stops calling. The session is NOT terminal: a unilateral seal is still available, and the transcript is intact",
|
|
5153
|
+
});
|
|
5154
|
+
// AWAITED, and `retireSessionNode` NOT `destroySessionNode`. The latter writes the status back
|
|
5155
|
+
// — `error` maps to `interrupted` — a few hundred milliseconds later, which silently undid the
|
|
5156
|
+
// whole unit; the former is the method that tears a node down without touching the status, and
|
|
5157
|
+
// it is what the local force-abandon path already uses.
|
|
5158
|
+
await this.retireSessionNode(agentName, sessionId);
|
|
5159
|
+
return true;
|
|
5160
|
+
}
|
|
5161
|
+
/** DOD-M12B-ABANDON-NOTIFY-1 — durable "they hung up" marker. Not a status: the session stays
|
|
5162
|
+
* sealable, which is the whole point of not making this terminal. */
|
|
5163
|
+
#markCounterpartyAbandoned(agentName, sessionId) {
|
|
5164
|
+
if (!this.#db)
|
|
5165
|
+
return false;
|
|
5166
|
+
try {
|
|
5167
|
+
const res = this.#db
|
|
5168
|
+
.prepare("UPDATE sessions SET counterparty_abandoned_at = ?, updated_at = ? WHERE agent_id = ? AND session_id = ? AND counterparty_abandoned_at IS NULL")
|
|
5169
|
+
.run(Date.now(), Date.now(), this.#requireAgentId(agentName), sessionId);
|
|
5170
|
+
// No rows changed means it was already marked — a duplicated notice, which must not
|
|
5171
|
+
// re-announce. "Did not throw" is not "landed"; the row count is the answer.
|
|
5172
|
+
return Number(res?.changes ?? 0) > 0;
|
|
5173
|
+
}
|
|
5174
|
+
catch (err) {
|
|
5175
|
+
this.#logger.error("session.counterparty.abandoned.write.failed", {
|
|
5176
|
+
agentName, sessionId,
|
|
5177
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5178
|
+
impact: "this side will go on trying to reach a counterparty that has hung up",
|
|
5179
|
+
});
|
|
5180
|
+
return false;
|
|
5181
|
+
}
|
|
5182
|
+
}
|
|
5183
|
+
/** DOD-M12B-ABANDON-NOTIFY-1: has the counterparty told us they hung up? */
|
|
5184
|
+
counterpartyAbandonedAt(agentName, sessionId) {
|
|
5185
|
+
if (!this.#db)
|
|
5186
|
+
return null;
|
|
5187
|
+
try {
|
|
5188
|
+
const row = this.#db
|
|
5189
|
+
.prepare("SELECT counterparty_abandoned_at FROM sessions WHERE agent_id = ? AND session_id = ?")
|
|
5190
|
+
.get(this.#requireAgentId(agentName), sessionId);
|
|
5191
|
+
return row?.counterparty_abandoned_at ?? null;
|
|
5192
|
+
}
|
|
5193
|
+
catch {
|
|
5194
|
+
return null;
|
|
5195
|
+
}
|
|
5196
|
+
}
|
|
5197
|
+
/**
|
|
5198
|
+
* DOD-M12B-REDIAL-1 — open a content stream, re-dialling once if the connection has gone.
|
|
5199
|
+
*
|
|
5200
|
+
* `newStream` never dials. It looks for an already-open connection filed under the recorded peer
|
|
5201
|
+
* id and throws `connection_lost` when there is none — and NOTHING re-dialled: not on
|
|
5202
|
+
* `session.liveness.changed → gone`, not on signaling reconnect, not on agent offline→online, not
|
|
5203
|
+
* in the drain hook. So one blip and that session parked EVERY message for the rest of its life,
|
|
5204
|
+
* on both sides, permanently. The relay backstop kept the messages moving, which is exactly why
|
|
5205
|
+
* it hid: nothing was lost, the conversation just stopped being a conversation.
|
|
5206
|
+
*
|
|
5207
|
+
* DEMAND-DRIVEN, never a timer. A background re-dial loop is what produced the 2026-08-17
|
|
5208
|
+
* notification storm — surviving halves of abandoned sessions dialling continuously while the
|
|
5209
|
+
* operator saw connection requests from agents nobody was driving. This fires only when a send
|
|
5210
|
+
* actually needs the connection, and a cooldown bounds a burst against a peer that is genuinely
|
|
5211
|
+
* gone: five sends cost one dial, not five.
|
|
5212
|
+
*/
|
|
5213
|
+
async #openContentStream(agentName, sessionId, entry, correlationId) {
|
|
5214
|
+
const attempt = async () => {
|
|
5215
|
+
if (this.#connectionLossRemaining > 0) {
|
|
5216
|
+
this.#connectionLossRemaining -= 1;
|
|
5217
|
+
throw { reason: "no_connection", message: "injected connection loss" };
|
|
5218
|
+
}
|
|
5219
|
+
return entry.node.newStream(entry.counterpartySessionPeerId, CELLO_CONTENT_PROTOCOL_ID);
|
|
5220
|
+
};
|
|
5221
|
+
try {
|
|
5222
|
+
return await attempt();
|
|
5223
|
+
}
|
|
5224
|
+
catch (err) {
|
|
5225
|
+
const reason = err?.reason;
|
|
5226
|
+
// ONLY for a missing connection, and `no_connection` is the reason that means exactly that.
|
|
5227
|
+
// NOT `connection_lost`, which is the transport's catch-all default and therefore also covers
|
|
5228
|
+
// a stream that failed on a healthy connection — the per-protocol stream cap of
|
|
5229
|
+
// DOD-M12B-ACK-1. Dialling there fixes nothing and shows the counterparty a connection
|
|
5230
|
+
// request caused by a defect on this side, which is the storm this unit exists to avoid.
|
|
5231
|
+
if (reason !== "no_connection")
|
|
5232
|
+
throw err;
|
|
5233
|
+
const key = this.#k(agentName, sessionId);
|
|
5234
|
+
const addrs = this.#counterpartyAddrs.get(key);
|
|
5235
|
+
if (!addrs || addrs.length === 0) {
|
|
5236
|
+
// ABSENT IS NOT FINE, and it is not silent. A session we never dialled — the responder's
|
|
5237
|
+
// half — has no addresses to dial back with, and that is a real limitation the operator
|
|
5238
|
+
// should be able to see rather than infer from a park.
|
|
5239
|
+
this.#logger.warn("session.transport.redial.unavailable", {
|
|
5240
|
+
sessionId, agentName, correlationId,
|
|
5241
|
+
impact: "the direct path is down and this side holds no address for the counterparty, so every send parks until they re-establish",
|
|
5242
|
+
});
|
|
5243
|
+
throw err;
|
|
5244
|
+
}
|
|
5245
|
+
const now = Date.now();
|
|
5246
|
+
const notBefore = this.#redialNotBefore.get(key) ?? 0;
|
|
5247
|
+
if (now < notBefore) {
|
|
5248
|
+
this.#logger.debug("session.transport.redial.cooldown", {
|
|
5249
|
+
sessionId, agentName, retryInMs: notBefore - now, correlationId,
|
|
5250
|
+
});
|
|
5251
|
+
throw err;
|
|
5252
|
+
}
|
|
5253
|
+
this.#redialNotBefore.set(key, now + REDIAL_COOLDOWN_MS);
|
|
5254
|
+
this.#logger.info("session.transport.redial.attempted", { sessionId, agentName, addrs: addrs.length, correlationId });
|
|
5255
|
+
const reconnected = await this.connectToCounterparty(agentName, sessionId, addrs);
|
|
5256
|
+
if (!reconnected.ok) {
|
|
5257
|
+
this.#logger.warn("session.transport.redial.failed", {
|
|
5258
|
+
sessionId, agentName, reason: reconnected.reason, error: reconnected.error, correlationId,
|
|
5259
|
+
});
|
|
5260
|
+
throw err;
|
|
5261
|
+
}
|
|
5262
|
+
this.#logger.info("session.transport.redial.succeeded", { sessionId, agentName, correlationId });
|
|
5263
|
+
// Cleared so the NEXT blip is repaired immediately: the cooldown exists to bound a dead peer,
|
|
5264
|
+
// not to make a live one wait.
|
|
5265
|
+
this.#redialNotBefore.delete(key);
|
|
5266
|
+
return attempt();
|
|
5267
|
+
}
|
|
5268
|
+
}
|
|
5269
|
+
/**
|
|
5270
|
+
* DOD-M12B-INDEX-1 — commit THIS agent's own leaf at the position the relay assigned it.
|
|
5271
|
+
*
|
|
5272
|
+
* The receiver has always enforced "leaf index === canonical position": content witnessed ahead
|
|
5273
|
+
* of the next expected leaf is held, not appended out of order. The sender never did. It had the
|
|
5274
|
+
* position in hand — the relay answers about 4 ms before the append — and called a push-only
|
|
5275
|
+
* append that puts the leaf at the tail whatever the tail happens to be. While its own tree has
|
|
5276
|
+
* no gap the two agree and nothing shows; the first gap puts its leaf at someone else's index,
|
|
5277
|
+
* parts its root from the counterparty's, and the next seal gets `leaf_count_mismatch`, which is
|
|
5278
|
+
* terminal.
|
|
5279
|
+
*
|
|
5280
|
+
* DELIVERY IS NOT DEFERRED BY THIS. The caller has already put the bytes on the wire; only the
|
|
5281
|
+
* leaf waits for its slot, exactly as a received message does. Holding our own send is only
|
|
5282
|
+
* affordable because holds are durable (DOD-M12B-STRAND-1) — before that it would have risked
|
|
5283
|
+
* losing the message outright.
|
|
5284
|
+
*
|
|
5285
|
+
* `assignedSeq` absent means no ordering authority answered. That is the documented degradation
|
|
5286
|
+
* and it appends in arrival order as before: with no position there is no discipline to enforce,
|
|
5287
|
+
* and refusing would take messaging down whenever the relay is unreachable.
|
|
5288
|
+
*/
|
|
5289
|
+
placeOwnLeaf(agentName, sessionId, contentHashHex, sentBytes, assignedSeq, correlationId, kind = "msg") {
|
|
5290
|
+
// Hydrate before reading the frontier: a durable hold this process has not read back yet would
|
|
5291
|
+
// make the tree look further along than it is.
|
|
5292
|
+
this.#ensureHeldRestored(agentName, sessionId);
|
|
5293
|
+
const nextExpected = this.getSessionTree(agentName, sessionId).size();
|
|
5294
|
+
if (assignedSeq === undefined) {
|
|
5295
|
+
const { leafIndex } = this.appendSessionLeaf(agentName, sessionId, kind, contentHashHex, correlationId);
|
|
5296
|
+
return { placed: true, leafIndex };
|
|
5297
|
+
}
|
|
5298
|
+
if (assignedSeq === nextExpected) {
|
|
5299
|
+
const { leafIndex } = this.appendSessionLeaf(agentName, sessionId, kind, contentHashHex, correlationId);
|
|
5300
|
+
return { placed: true, leafIndex };
|
|
5301
|
+
}
|
|
5302
|
+
if (assignedSeq < nextExpected) {
|
|
5303
|
+
// THE TREE AND THE RELAY HAVE ALREADY DIVERGED, and refusing here does not undo that.
|
|
5304
|
+
//
|
|
5305
|
+
// This side is AHEAD of the relay's counter, which happens by design: a message whose relay
|
|
5306
|
+
// submit failed still appends unwitnessed (the documented degradation). From then on every
|
|
5307
|
+
// ack comes back behind our frontier and the two can never agree again — the seal was already
|
|
5308
|
+
// lost at the unwitnessed append, not here.
|
|
5309
|
+
//
|
|
5310
|
+
// So the choice is between a record that is short by every subsequent message and one that is
|
|
5311
|
+
// complete but skewed. Appending at the tail keeps the operator's own words in their own
|
|
5312
|
+
// transcript, which is worth more than a tidiness the roots cannot recover anyway; writing
|
|
5313
|
+
// over the assigned slot is the one thing never done, because that rewrites a leaf a root has
|
|
5314
|
+
// already been computed over. The divergence is reported at ERROR and carried to the caller
|
|
5315
|
+
// rather than dressed up as an ordinary success.
|
|
5316
|
+
this.#logger.error("session.tree.position_behind_frontier", {
|
|
5317
|
+
agentName, sessionId, assignedSeq, nextExpected, contentHash: contentHashHex, correlationId,
|
|
5318
|
+
impact: "this side's tree is ahead of the relay's counter, so the two can no longer agree on a root — the message is kept in the local record and this session can no longer be sealed bilaterally",
|
|
5319
|
+
});
|
|
5320
|
+
this.#diverged.add(this.#k(agentName, sessionId));
|
|
5321
|
+
const { leafIndex } = this.appendSessionLeaf(agentName, sessionId, kind, contentHashHex, correlationId);
|
|
5322
|
+
return { placed: true, leafIndex, diverged: true };
|
|
5323
|
+
}
|
|
5324
|
+
// Ahead of the tail: hold it, exactly as the receiver holds theirs, and let #releaseHeld put it
|
|
5325
|
+
// in at its own index when the gap fills.
|
|
5326
|
+
const key = this.#k(agentName, sessionId);
|
|
5327
|
+
let held = this.#heldContent.get(key);
|
|
5328
|
+
if (!held) {
|
|
5329
|
+
held = new Map();
|
|
5330
|
+
this.#heldContent.set(key, held);
|
|
5331
|
+
}
|
|
5332
|
+
held.set(assignedSeq, { content: sentBytes, contentHashHex, correlationId, origin: "sent", kind });
|
|
5333
|
+
this.#persistHeldContent(agentName, sessionId, assignedSeq, sentBytes, sentBytes, contentHashHex, false, correlationId, "sent", kind);
|
|
5334
|
+
this.#logger.info("session.content.held", {
|
|
5335
|
+
sessionId, canonicalSeq: assignedSeq, nextExpected, gap: assignedSeq - nextExpected,
|
|
5336
|
+
origin: "sent", correlationId,
|
|
5337
|
+
});
|
|
5338
|
+
return { placed: false, heldAt: assignedSeq };
|
|
5339
|
+
}
|
|
5340
|
+
/**
|
|
5341
|
+
* DOD-M12B-STRAND-1 — write one held frame to the durable store.
|
|
5342
|
+
*
|
|
5343
|
+
* LOGS LOUD, does not refuse. The caller answers `held: true` either way, and that is correct:
|
|
5344
|
+
* held content is never `persisted`-acked, so the sender keeps its copy and retries whether or
|
|
5345
|
+
* not this row lands. What a failure costs is the restart case — the frame is memory-only again,
|
|
5346
|
+
* exactly as it was before this unit — so it is reported at ERROR here and counted again by the
|
|
5347
|
+
* teardown alarm, and never allowed to look like a success.
|
|
5348
|
+
*/
|
|
5349
|
+
#persistHeldContent(agentName, sessionId, canonicalSeq, deliverContent, originalContent, contentHashHex, screenedOut, correlationId, origin = "received", leafKind = "msg") {
|
|
5350
|
+
if (!this.#db)
|
|
5351
|
+
return;
|
|
5352
|
+
try {
|
|
5353
|
+
// A position may legitimately be re-written by a redelivery of the SAME frame. Different
|
|
5354
|
+
// content at the same relay position means the relay contradicted itself, and destroying the
|
|
5355
|
+
// first copy silently is not an option for verified content.
|
|
5356
|
+
const existing = this.#db.prepare("SELECT content_hash_hex FROM held_content WHERE agent_id = ? AND session_id = ? AND canonical_seq = ?").get(this.#requireAgentId(agentName), sessionId, canonicalSeq);
|
|
5357
|
+
if (existing && existing.content_hash_hex !== contentHashHex) {
|
|
5358
|
+
this.#logger.error("session.content.held.position_conflict", {
|
|
5359
|
+
agentName, sessionId, canonicalSeq, correlationId,
|
|
5360
|
+
existingContentHash: existing.content_hash_hex, incomingContentHash: contentHashHex,
|
|
5361
|
+
impact: "two different frames claim one canonical position — the earlier held copy is being replaced",
|
|
5362
|
+
});
|
|
5363
|
+
}
|
|
5364
|
+
this.#db.prepare(`INSERT OR REPLACE INTO held_content
|
|
5365
|
+
(agent_id, session_id, canonical_seq, content_blob, original_blob, content_hash_hex, screened_out, correlation_id, held_at, origin, leaf_kind)
|
|
5366
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(this.#requireAgentId(agentName), sessionId, canonicalSeq, Buffer.from(deliverContent), Buffer.from(originalContent), contentHashHex, screenedOut ? 1 : 0, correlationId ?? null, Date.now(), origin, leafKind);
|
|
5367
|
+
}
|
|
5368
|
+
catch (err) {
|
|
5369
|
+
this.#logger.error("session.content.held.persist.failed", {
|
|
5370
|
+
agentName, sessionId, canonicalSeq, contentHash: contentHashHex, correlationId,
|
|
5371
|
+
impact: "this frame is held IN MEMORY ONLY and will be destroyed if the daemon restarts",
|
|
5372
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5373
|
+
});
|
|
5374
|
+
}
|
|
5375
|
+
}
|
|
5376
|
+
/** DOD-M12B-STRAND-1 — drop one held frame from the durable store, on release or on a refusal
|
|
5377
|
+
* that supersedes it. A row that outlives its release re-appends on the next boot. */
|
|
5378
|
+
#deleteHeldContent(agentName, sessionId, canonicalSeq) {
|
|
5379
|
+
if (!this.#db)
|
|
5380
|
+
return;
|
|
5381
|
+
try {
|
|
5382
|
+
this.#db.prepare("DELETE FROM held_content WHERE agent_id = ? AND session_id = ? AND canonical_seq = ?").run(this.#requireAgentId(agentName), sessionId, canonicalSeq);
|
|
5383
|
+
}
|
|
5384
|
+
catch (err) {
|
|
5385
|
+
this.#logger.error("session.content.held.delete.failed", {
|
|
5386
|
+
agentName, sessionId, canonicalSeq,
|
|
5387
|
+
impact: "the released frame's durable row survives and will be re-appended on the next boot",
|
|
5388
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5389
|
+
});
|
|
5390
|
+
}
|
|
5391
|
+
}
|
|
5392
|
+
/**
|
|
5393
|
+
* DOD-M12B-STRAND-1 — restore this session's held frames into memory.
|
|
5394
|
+
*
|
|
5395
|
+
* Called when a session node is (re)created, which is the moment the session becomes able to
|
|
5396
|
+
* append again. Loading at daemon boot instead would be wrong for the same reason the old code
|
|
5397
|
+
* was wrong: a frame is only releasable against a tree, and the tree is loaded per session.
|
|
5398
|
+
*
|
|
5399
|
+
* A frame whose content the tree ALREADY HOLDS at that position is dropped — re-appending it
|
|
5400
|
+
* would change the root a seal signs over. That test is `hashAt`, not `canonical_seq < frontier`:
|
|
5401
|
+
* the two counters are different spaces and this file documents them drifting, so the index
|
|
5402
|
+
* comparison alone would destroy a frame the tree never held. See the drift branch below.
|
|
5403
|
+
*/
|
|
5404
|
+
#ensureHeldRestored(agentName, sessionId, opts) {
|
|
5405
|
+
const key = this.#k(agentName, sessionId);
|
|
5406
|
+
if (!this.#heldRestored.has(key)) {
|
|
5407
|
+
// Set BEFORE restoring, so the #ensureHeldRestored inside #releaseHeld below returns straight
|
|
5408
|
+
// away instead of recursing.
|
|
5409
|
+
this.#heldRestored.add(key);
|
|
5410
|
+
this.#restoreHeldContent(agentName, sessionId);
|
|
5411
|
+
}
|
|
5412
|
+
// A RESTORED FRAME MAY ALREADY BE IN ORDER, and nothing else would ever notice.
|
|
5413
|
+
//
|
|
5414
|
+
// #releaseHeld has one caller: the tail of a successful inbound ingest. Every other way the tree
|
|
5415
|
+
// grows — an outbound send leaf, a queued or rejected leaf — advances the frontier without
|
|
5416
|
+
// draining. While holds died with the session node that cost seconds; now the hold is durable,
|
|
5417
|
+
// so the stall is durable too: the counterparty's message sits on disk at exactly the next slot,
|
|
5418
|
+
// is never delivered, and `sealReadiness` counts it, so the session cannot close either.
|
|
5419
|
+
// Undeliverable AND unsealable, forever, from one restart.
|
|
5420
|
+
//
|
|
5421
|
+
// TRACKED SEPARATELY FROM THE HYDRATION. A read-only caller (the status surface) hydrates and
|
|
5422
|
+
// must NOT release — otherwise `cello status` appends leaves, advances the session root, writes
|
|
5423
|
+
// transcript rows and rings the doorbell, which makes a diagnostic command the thing that
|
|
5424
|
+
// delivers messages. One shared flag would also let that read CONSUME the release the next real
|
|
5425
|
+
// ingest was going to perform, which is the stall above, reintroduced.
|
|
5426
|
+
if (opts?.release === false)
|
|
5427
|
+
return;
|
|
5428
|
+
if (this.#heldReleased.has(key))
|
|
5429
|
+
return;
|
|
5430
|
+
this.#heldReleased.add(key);
|
|
5431
|
+
if (this.#heldContent.get(key)?.size) {
|
|
5432
|
+
const counterparty = this.getSessionRecord(agentName, sessionId)?.counterparty_pubkey;
|
|
5433
|
+
if (counterparty)
|
|
5434
|
+
this.#releaseHeld(agentName, sessionId, counterparty);
|
|
5435
|
+
}
|
|
5436
|
+
}
|
|
5437
|
+
#restoreHeldContent(agentName, sessionId) {
|
|
5438
|
+
if (!this.#db)
|
|
5439
|
+
return;
|
|
5440
|
+
let rows;
|
|
5441
|
+
try {
|
|
5442
|
+
rows = this.#db.prepare(`SELECT canonical_seq, content_blob, original_blob, content_hash_hex, screened_out, correlation_id, origin, leaf_kind
|
|
5443
|
+
FROM held_content WHERE agent_id = ? AND session_id = ? ORDER BY canonical_seq ASC`).all(this.#requireAgentId(agentName), sessionId);
|
|
5444
|
+
}
|
|
5445
|
+
catch (err) {
|
|
5446
|
+
this.#logger.error("session.content.held.restore.failed", {
|
|
5447
|
+
agentName, sessionId,
|
|
5448
|
+
impact: "verified content held before the restart is not in memory and cannot be released",
|
|
5449
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5450
|
+
});
|
|
5451
|
+
return;
|
|
5452
|
+
}
|
|
5453
|
+
if (rows.length === 0)
|
|
5454
|
+
return;
|
|
5455
|
+
const key = this.#k(agentName, sessionId);
|
|
5456
|
+
let held = this.#heldContent.get(key);
|
|
5457
|
+
if (!held) {
|
|
5458
|
+
held = new Map();
|
|
5459
|
+
this.#heldContent.set(key, held);
|
|
5460
|
+
}
|
|
5461
|
+
const tree = this.getSessionTree(agentName, sessionId);
|
|
5462
|
+
const frontier = tree.size();
|
|
5463
|
+
let restored = 0;
|
|
5464
|
+
let superseded = 0;
|
|
5465
|
+
let drifted = 0;
|
|
5466
|
+
for (const row of rows) {
|
|
5467
|
+
// ASK THE TREE WHAT IS AT THAT POSITION — do not infer it from the index.
|
|
5468
|
+
//
|
|
5469
|
+
// `canonical_seq` is the RELAY's sequence space; `frontier` is this tree's msg-leaf count.
|
|
5470
|
+
// The two drift, on purpose and by documented cases: the relay counts CTRL leaves the tree
|
|
5471
|
+
// never appends, and a first message whose relay submit failed leaves the tree one ahead.
|
|
5472
|
+
// Under drift `canonical_seq < frontier` is TRUE for a frame the tree has never held, and
|
|
5473
|
+
// deleting on that comparison destroys verified content while reporting it as tidy-up —
|
|
5474
|
+
// the exact failure this unit exists to end, reintroduced on the recovery path.
|
|
5475
|
+
const occupant = tree.hashAt(row.canonical_seq);
|
|
5476
|
+
if (occupant === row.content_hash_hex) {
|
|
5477
|
+
this.#deleteHeldContent(agentName, sessionId, row.canonical_seq);
|
|
5478
|
+
superseded++;
|
|
5479
|
+
continue;
|
|
5480
|
+
}
|
|
5481
|
+
if (row.canonical_seq < frontier) {
|
|
5482
|
+
// The position is taken by DIFFERENT content. The frame cannot be appended (that would
|
|
5483
|
+
// rewrite a committed leaf) and must not be deleted (it is verified content nobody else
|
|
5484
|
+
// holds), so it goes to the annex that exists for exactly this — content that arrived for
|
|
5485
|
+
// a chain that can no longer carry it — and only then does the row go.
|
|
5486
|
+
const annexed = this.recordSealedAnnex(agentName, sessionId, row.content_hash_hex, new Uint8Array(row.content_blob),
|
|
5487
|
+
// DOD-M12B-INDEX-1: our own held send is attributed to US, never to the counterparty.
|
|
5488
|
+
row.origin === "sent"
|
|
5489
|
+
? this.#ownPubkeyHex(agentName)
|
|
5490
|
+
: this.getSessionRecord(agentName, sessionId)?.counterparty_pubkey ?? null);
|
|
5491
|
+
this.#logger.error("session.content.held.position_drifted", {
|
|
5492
|
+
agentName, sessionId, canonicalSeq: row.canonical_seq, frontier,
|
|
5493
|
+
contentHash: row.content_hash_hex, occupant, annexed,
|
|
5494
|
+
correlationId: row.correlation_id ?? undefined,
|
|
5495
|
+
impact: annexed
|
|
5496
|
+
? "the relay's position for this frame is occupied by different content — it cannot join the chain and is readable only from the annex"
|
|
5497
|
+
: "the relay's position for this frame is occupied by different content AND the annex write failed — the durable row is kept rather than destroyed",
|
|
5498
|
+
});
|
|
5499
|
+
if (annexed)
|
|
5500
|
+
this.#deleteHeldContent(agentName, sessionId, row.canonical_seq);
|
|
5501
|
+
drifted++;
|
|
5502
|
+
continue;
|
|
5503
|
+
}
|
|
5504
|
+
held.set(row.canonical_seq, {
|
|
5505
|
+
content: new Uint8Array(row.content_blob),
|
|
5506
|
+
...(row.original_blob ? { originalContent: new Uint8Array(row.original_blob) } : {}),
|
|
5507
|
+
contentHashHex: row.content_hash_hex,
|
|
5508
|
+
...(row.correlation_id ? { correlationId: row.correlation_id } : {}),
|
|
5509
|
+
...(row.screened_out ? { screenedOut: true } : {}),
|
|
5510
|
+
...(row.origin === "sent" ? { origin: "sent" } : {}),
|
|
5511
|
+
...(row.leaf_kind === "doc" ? { kind: "doc" } : {}),
|
|
5512
|
+
});
|
|
5513
|
+
restored++;
|
|
5514
|
+
}
|
|
5515
|
+
if (held.size === 0)
|
|
5516
|
+
this.#heldContent.delete(key);
|
|
5517
|
+
this.#logger.info("session.content.held.restored", {
|
|
5518
|
+
agentName, sessionId, restored, superseded, drifted, frontier,
|
|
5519
|
+
canonicalSeqs: [...held.keys()].sort((a, b) => a - b),
|
|
5520
|
+
// The flow ids of the frames that came back, so a restored message ties to the
|
|
5521
|
+
// `session.content.held` that opened its flow before the restart.
|
|
5522
|
+
correlationIds: rows.map((r) => r.correlation_id).filter((c) => c !== null),
|
|
5523
|
+
});
|
|
5524
|
+
}
|
|
4432
5525
|
/**
|
|
4433
5526
|
* DOD-MSG-4: drain held out-of-order content in canonical order. After a leaf is appended, any
|
|
4434
5527
|
* held entry whose canonical sequence equals the new next-expected index is now in order — append
|
|
@@ -4436,6 +5529,10 @@ export class SessionNodeManager {
|
|
|
4436
5529
|
*/
|
|
4437
5530
|
#releaseHeld(agentName, sessionId, senderPubkey) {
|
|
4438
5531
|
const key = this.#k(agentName, sessionId);
|
|
5532
|
+
// DOD-M12B-STRAND-1: hydrate before scanning. Restoring eagerly at session-node creation put
|
|
5533
|
+
// it behind writes that can fail — one failed `sessions` row upsert and the frames stayed on
|
|
5534
|
+
// disk, invisible, which is the same outcome as losing them.
|
|
5535
|
+
this.#ensureHeldRestored(agentName, sessionId);
|
|
4439
5536
|
const held = this.#heldContent.get(key);
|
|
4440
5537
|
if (!held)
|
|
4441
5538
|
return 0;
|
|
@@ -4446,13 +5543,47 @@ export class SessionNodeManager {
|
|
|
4446
5543
|
if (!entry)
|
|
4447
5544
|
break;
|
|
4448
5545
|
held.delete(nextExpected);
|
|
4449
|
-
//
|
|
4450
|
-
//
|
|
4451
|
-
|
|
5546
|
+
// DOD-M12B-STRAND-1: released content leaves the durable store in the same breath. A row
|
|
5547
|
+
// that outlives its release would re-append the same content on the next boot — growing the
|
|
5548
|
+
// tree and changing a root that has already been signed.
|
|
5549
|
+
this.#deleteHeldContent(agentName, sessionId, nextExpected);
|
|
5550
|
+
// DOD-M12B-INDEX-1: OUR OWN held message. It leafs at its canonical index and is transcribed
|
|
5551
|
+
// as SENT — never routed down the received path, which would attribute our words to the
|
|
5552
|
+
// counterparty in the sealed record and hand them back to our own agent as inbound.
|
|
5553
|
+
if (entry.origin === "sent") {
|
|
5554
|
+
// The KIND the leaf was placed with, not a hardcoded "msg" — a document leaf that had to
|
|
5555
|
+
// wait for its position must come back as a document leaf, or the two sides disagree about
|
|
5556
|
+
// what the chain contains.
|
|
5557
|
+
this.appendSessionLeaf(agentName, sessionId, entry.kind ?? "msg", entry.contentHashHex, entry.correlationId);
|
|
5558
|
+
// A DOCUMENT frame takes a leaf and NO transcript row — matching what the immediate-append
|
|
5559
|
+
// path does for one. Writing one would put raw CBOR into the operator's transcript as
|
|
5560
|
+
// something they said, which is the same attribution failure as releasing it inbound.
|
|
5561
|
+
if (entry.kind === "doc") {
|
|
5562
|
+
released++;
|
|
5563
|
+
this.#logger.info("session.content.released", {
|
|
5564
|
+
sessionId, sequenceNumber: nextExpected, leafKind: "doc", correlationId: entry.correlationId,
|
|
5565
|
+
});
|
|
5566
|
+
if (held.size === 0) {
|
|
5567
|
+
this.#heldContent.delete(key);
|
|
5568
|
+
break;
|
|
5569
|
+
}
|
|
5570
|
+
continue;
|
|
5571
|
+
}
|
|
5572
|
+
// OBSERVED, not assumed — the received path already does this. The leaf commits either way,
|
|
5573
|
+
// so a dropped transcript write means the operator's OWN message is missing from their own
|
|
5574
|
+
// transcript with the chain saying it is there, and nothing anywhere said so.
|
|
5575
|
+
if (!this.recordTranscriptMessage(agentName, sessionId, nextExpected, "sent", entry.content, entry.correlationId)) {
|
|
5576
|
+
this.#logger.error("session.content.released.transcript.failed", {
|
|
5577
|
+
agentName, sessionId, sequenceNumber: nextExpected, correlationId: entry.correlationId,
|
|
5578
|
+
impact: "this side's own message is committed to the chain but missing from its transcript",
|
|
5579
|
+
});
|
|
5580
|
+
}
|
|
5581
|
+
}
|
|
5582
|
+
else if (entry.screenedOut) {
|
|
4452
5583
|
this.appendSessionLeaf(agentName, sessionId, "msg", entry.contentHashHex, entry.correlationId);
|
|
4453
5584
|
}
|
|
4454
5585
|
else {
|
|
4455
|
-
this.#appendVerifiedContent(agentName, sessionId, entry.content, entry.contentHashHex, senderPubkey, entry.correlationId);
|
|
5586
|
+
this.#appendVerifiedContent(agentName, sessionId, entry.content, entry.contentHashHex, senderPubkey, entry.correlationId, entry.originalContent);
|
|
4456
5587
|
}
|
|
4457
5588
|
released++;
|
|
4458
5589
|
this.#logger.info("session.content.released", {
|
|
@@ -4652,10 +5783,35 @@ export class SessionNodeManager {
|
|
|
4652
5783
|
*/
|
|
4653
5784
|
async #sendDeliveryAck(agentName, sessionId, contentHash, correlationId) {
|
|
4654
5785
|
const entry = this.#activeNodes.get(this.#k(agentName, sessionId));
|
|
4655
|
-
if (!entry)
|
|
5786
|
+
if (!entry) {
|
|
5787
|
+
// NOT a silent return. No ACK is exactly this milestone's symptom — the sender's TTF expires
|
|
5788
|
+
// and the message parks — so the one case where we knowingly decline to send one has to say
|
|
5789
|
+
// so, or it is indistinguishable from the defect.
|
|
5790
|
+
this.#logger.debug("content.delivery.ack.skipped", {
|
|
5791
|
+
agentName,
|
|
5792
|
+
sessionId,
|
|
5793
|
+
contentHash: Buffer.from(contentHash).toString("hex"),
|
|
5794
|
+
reason: "session_node_gone",
|
|
5795
|
+
correlationId,
|
|
5796
|
+
});
|
|
4656
5797
|
return;
|
|
5798
|
+
}
|
|
5799
|
+
// Held outside the try so the catch can retire a stream that was opened and then failed to
|
|
5800
|
+
// write. Without it every failure leaks the OUTBOUND half of the stream the receiver-side
|
|
5801
|
+
// `finally` retires — same defect, other end, other cap. See the note on #handleContentStream.
|
|
5802
|
+
// Assigned IMMEDIATELY after newStream: anything between the two is a window where a throw
|
|
5803
|
+
// leaks the stream because the catch cannot see it.
|
|
5804
|
+
let ackStream;
|
|
4657
5805
|
try {
|
|
4658
5806
|
const stream = await entry.node.newStream(entry.counterpartySessionPeerId, CELLO_CONTENT_PROTOCOL_ID);
|
|
5807
|
+
ackStream = stream;
|
|
5808
|
+
// Injected ACK-write failure — thrown from inside the try so it lands in exactly the catch a
|
|
5809
|
+
// real reset lands in, and the whole downstream path (impair → abort → log) runs unmodified.
|
|
5810
|
+
if (this.#ackFaultRemaining > 0) {
|
|
5811
|
+
this.#ackFaultRemaining -= 1;
|
|
5812
|
+
this.#logger.warn("content.delivery.ack.fault.injected", { sessionId });
|
|
5813
|
+
throw new Error("connection_lost: injected delivery-ack fault");
|
|
5814
|
+
}
|
|
4659
5815
|
const frame = encodeCbor({
|
|
4660
5816
|
type: "content_delivery_ack",
|
|
4661
5817
|
session_id: sessionId,
|
|
@@ -4664,27 +5820,47 @@ export class SessionNodeManager {
|
|
|
4664
5820
|
correlation_id: correlationId,
|
|
4665
5821
|
});
|
|
4666
5822
|
stream.send(lp.encode.single(frame));
|
|
4667
|
-
//
|
|
4668
|
-
//
|
|
4669
|
-
//
|
|
4670
|
-
//
|
|
5823
|
+
// NOT SWALLOWED, for the same reason the direct-send path stopped swallowing it: `close()`
|
|
5824
|
+
// waits for the write buffer to drain, so a reset mid-flush throws HERE and that is exactly
|
|
5825
|
+
// the case where the bytes never left. A swallowed close made two things happen at once —
|
|
5826
|
+
// this log claimed the ACK went out while the sender's TTF fired and parked, and the abort in
|
|
5827
|
+
// the catch below (the thing that frees the stream slot) became unreachable.
|
|
5828
|
+
await stream.close();
|
|
5829
|
+
// AFTER the close, because that is when it is true. The receiver-side counterpart to the
|
|
5830
|
+
// sender's content.delivery.acked: B has acknowledged this content `persisted`, so the sender
|
|
5831
|
+
// stops retrying/parking. Emitted for BOTH a normally delivered message AND a terminal-screen
|
|
5832
|
+
// block (the block is a definitive receipt — the leaf is recorded, so the sender must stop) —
|
|
5833
|
+
// and deliberately NOT for a transient hold.
|
|
4671
5834
|
this.#logger.info("content.delivery.ack.sent", {
|
|
4672
5835
|
sessionId,
|
|
4673
5836
|
contentHash: Buffer.from(contentHash).toString("hex"),
|
|
4674
5837
|
correlationId,
|
|
4675
5838
|
});
|
|
4676
|
-
|
|
4677
|
-
|
|
4678
|
-
|
|
4679
|
-
|
|
5839
|
+
// An agent that mostly LISTENS sends content rarely and ACKs constantly. Clearing only on the
|
|
5840
|
+
// content path would leave exactly those sessions reporting a broken conversation forever
|
|
5841
|
+
// after one bad ACK — the one-way door, on the other send path.
|
|
5842
|
+
this.#clearSessionImpairment(agentName, sessionId, "delivery_ack", correlationId);
|
|
4680
5843
|
}
|
|
4681
5844
|
catch (err) {
|
|
4682
5845
|
this.#logger.warn("content.delivery.ack.send.failed", {
|
|
4683
5846
|
sessionId,
|
|
4684
5847
|
contentHash: Buffer.from(contentHash).toString("hex"),
|
|
4685
5848
|
error: err instanceof Error ? err.message : String(err),
|
|
5849
|
+
// "Cannot write to a stream that is closed" names where the write died, never why. The
|
|
5850
|
+
// why is almost always the per-protocol stream cap, and these two numbers are what turn
|
|
5851
|
+
// that from a log-measurement session into a grep.
|
|
5852
|
+
...this.#streamCensus(entry.node, entry.counterpartySessionPeerId),
|
|
4686
5853
|
correlationId,
|
|
4687
5854
|
});
|
|
5855
|
+
// The ACK travels the same direct path as our own content, so a failure here is the same
|
|
5856
|
+
// evidence: writes to this counterparty are not landing.
|
|
5857
|
+
this.#markSessionImpaired(agentName, sessionId, { cause: "delivery_ack", error: err instanceof Error ? err.message : String(err), correlationId });
|
|
5858
|
+
if (ackStream !== undefined) {
|
|
5859
|
+
try {
|
|
5860
|
+
ackStream.abort(err instanceof Error ? err : new Error(String(err)));
|
|
5861
|
+
}
|
|
5862
|
+
catch { /* already gone */ }
|
|
5863
|
+
}
|
|
4688
5864
|
}
|
|
4689
5865
|
}
|
|
4690
5866
|
/** Cancel and drop a single awaiting-ACK entry (e.g. the send failed after arming). */
|
|
@@ -4742,9 +5918,17 @@ export class SessionNodeManager {
|
|
|
4742
5918
|
// the fragile dependency on that internal timing (review L4).
|
|
4743
5919
|
async #registerContentHandler(agentName, sessionId, node, _counterpartyPubkey) {
|
|
4744
5920
|
try {
|
|
4745
|
-
await node.handle(CELLO_CONTENT_PROTOCOL_ID, (stream) => {
|
|
4746
|
-
|
|
4747
|
-
|
|
5921
|
+
await node.handle(CELLO_CONTENT_PROTOCOL_ID, (stream, remotePeerId) => {
|
|
5922
|
+
// `.catch` is not decoration: the handler builds its length-prefixed decoder before its own
|
|
5923
|
+
// try, and a throw there would otherwise become an unhandled rejection that takes the
|
|
5924
|
+
// daemon down for one malformed inbound stream.
|
|
5925
|
+
void this.#handleContentStream(agentName, sessionId, stream, remotePeerId).catch((err) => {
|
|
5926
|
+
this.#logger.warn("session.content.stream.handler.failed", {
|
|
5927
|
+
sessionId,
|
|
5928
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5929
|
+
});
|
|
5930
|
+
});
|
|
5931
|
+
}, { maxInboundStreams: CONTENT_MAX_INBOUND_STREAMS });
|
|
4748
5932
|
}
|
|
4749
5933
|
catch (err) {
|
|
4750
5934
|
this.#logger.error("session.content.handler.register.failed", {
|
|
@@ -4930,9 +6114,26 @@ export class SessionNodeManager {
|
|
|
4930
6114
|
// No verified position — the caller falls back to the announced hash-dedup path.
|
|
4931
6115
|
return null;
|
|
4932
6116
|
}
|
|
4933
|
-
async #handleContentStream(agentName, sessionId, stream) {
|
|
4934
|
-
|
|
6117
|
+
async #handleContentStream(agentName, sessionId, stream, remotePeerId) {
|
|
6118
|
+
// CLOSING THIS STREAM IS WHAT KEEPS THE SESSION ALIVE PAST ITS 33RD MESSAGE.
|
|
6119
|
+
//
|
|
6120
|
+
// Every content frame and every delivery ACK opens a fresh /cello/content/1.0.0 stream on the
|
|
6121
|
+
// one muxed connection the session holds, and libp2p caps INBOUND streams per protocol per
|
|
6122
|
+
// connection. It enforces that cap AFTER multistream-select has answered, so an over-cap stream
|
|
6123
|
+
// negotiates fine and is reset an instant later, and the SENDER's next `stream.send(...)`
|
|
6124
|
+
// throws "Cannot write to a stream that is closed" — an error that names the exit point and not
|
|
6125
|
+
// one thing about the cause.
|
|
6126
|
+
//
|
|
6127
|
+
// A stream leaves the muxer's set only on its `close` event, and closing our write end triggers
|
|
6128
|
+
// that only once the peer has closed its end too. So a handler that reads its frame and returns
|
|
6129
|
+
// leaves the stream half-open for the life of the connection and the count only ever rises.
|
|
6130
|
+
// Measured on a live daemon: 115 failures over 3.5 hours, with EXACTLY 32 successful streams
|
|
6131
|
+
// before the first one on both affected sessions (M12B Entry 10).
|
|
6132
|
+
//
|
|
6133
|
+
// The decoder is built INSIDE the try so a malformed stream cannot throw past the close below.
|
|
6134
|
+
let iter;
|
|
4935
6135
|
try {
|
|
6136
|
+
iter = lp.decode(stream)[Symbol.asyncIterator]();
|
|
4936
6137
|
const result = await iter.next();
|
|
4937
6138
|
if (result.done || result.value === undefined)
|
|
4938
6139
|
return;
|
|
@@ -4952,6 +6153,40 @@ export class SessionNodeManager {
|
|
|
4952
6153
|
}
|
|
4953
6154
|
return;
|
|
4954
6155
|
}
|
|
6156
|
+
// DOD-M12B-ABANDON-NOTIFY-1: the counterparty force-abandoned. Handled here, on the same
|
|
6157
|
+
// authenticated stream the delivery acknowledgement rides, and AFTER the session-id check
|
|
6158
|
+
// below cannot be skipped — the frame names its session and the handler is bound to one.
|
|
6159
|
+
if (frame["type"] === "session_abandoned_notice") {
|
|
6160
|
+
// PINNED TO THE COUNTERPARTY. This frame ENDS a conversation, so the stream being
|
|
6161
|
+
// authenticated is not enough — a session node is a promoted standing receiver, and a
|
|
6162
|
+
// standing receiver accepts everyone. libp2p's gater runs at connection establishment and
|
|
6163
|
+
// does not close connections that already exist, so a peer that dialled this node earlier
|
|
6164
|
+
// still holds a live connection after `setAllowedPeer` narrows it. Without this check that
|
|
6165
|
+
// peer could hang up a session it is not party to.
|
|
6166
|
+
//
|
|
6167
|
+
// `remotePeerId` is the Noise-authenticated transport identity, which the handler was
|
|
6168
|
+
// throwing away. Absent means we cannot prove who is speaking, and an unprovable claim to
|
|
6169
|
+
// end a session is refused.
|
|
6170
|
+
const expected = this.#activeNodes.get(this.#k(agentName, sessionId))?.counterpartySessionPeerId;
|
|
6171
|
+
if (!remotePeerId || !expected || remotePeerId !== expected) {
|
|
6172
|
+
this.#logger.warn("session.content.peer_mismatch", {
|
|
6173
|
+
sessionId, frameType: "session_abandoned_notice",
|
|
6174
|
+
remotePeerId: remotePeerId ?? "(absent)", expected: expected ?? "(unknown)",
|
|
6175
|
+
});
|
|
6176
|
+
return;
|
|
6177
|
+
}
|
|
6178
|
+
// REQUIRED and equal — absence is not a pass. The frame names its session and the handler
|
|
6179
|
+
// is bound to one; treating a missing field as agreement is how a guard stops guarding.
|
|
6180
|
+
const claimed = frame["session_id"];
|
|
6181
|
+
if (typeof claimed !== "string" || claimed !== sessionId) {
|
|
6182
|
+
this.#logger.warn("session.content.session_mismatch", {
|
|
6183
|
+
sessionId, claimedSessionId: typeof claimed === "string" ? claimed : "(absent)",
|
|
6184
|
+
});
|
|
6185
|
+
return;
|
|
6186
|
+
}
|
|
6187
|
+
void this.retireOnCounterpartyAbandon(agentName, sessionId, correlationId);
|
|
6188
|
+
return;
|
|
6189
|
+
}
|
|
4955
6190
|
if (frame["type"] !== "content_frame") {
|
|
4956
6191
|
// LOGGED, not silently dropped. This handler is bound to one session, and a frame it does
|
|
4957
6192
|
// not understand arriving on that stream is either a peer speaking a newer protocol or a
|
|
@@ -5026,6 +6261,48 @@ export class SessionNodeManager {
|
|
|
5026
6261
|
error: err instanceof Error ? err.message : String(err),
|
|
5027
6262
|
});
|
|
5028
6263
|
}
|
|
6264
|
+
finally {
|
|
6265
|
+
// `close()` waits only for OUR write buffer, which is empty here, so this cannot stall the
|
|
6266
|
+
// handler; it runs on every exit above, and there are several early returns.
|
|
6267
|
+
try {
|
|
6268
|
+
await stream.close();
|
|
6269
|
+
}
|
|
6270
|
+
catch (err) {
|
|
6271
|
+
// NOT SILENT. A close that fails here is the signature of the cap biting from the other
|
|
6272
|
+
// side, and it was the absence of exactly this line that turned the original diagnosis
|
|
6273
|
+
// into a 6,451-record log measurement.
|
|
6274
|
+
this.#logger.warn("session.content.stream.close.failed", {
|
|
6275
|
+
sessionId,
|
|
6276
|
+
error: err instanceof Error ? err.message : String(err),
|
|
6277
|
+
});
|
|
6278
|
+
try {
|
|
6279
|
+
stream.abort(err instanceof Error ? err : new Error(String(err)));
|
|
6280
|
+
}
|
|
6281
|
+
catch { /* already gone */ }
|
|
6282
|
+
return;
|
|
6283
|
+
}
|
|
6284
|
+
// OUR CLOSE ALONE DOES NOT FREE THE SLOT — the peer has to close its end too, and a peer
|
|
6285
|
+
// owns its own daemon. Without this, someone who opens content streams and never closes them
|
|
6286
|
+
// pins every inbound slot we have and puts us straight back into the defect above, with the
|
|
6287
|
+
// same unreadable error. `abort` resets unilaterally, so it works regardless of the peer;
|
|
6288
|
+
// the delay is what keeps it from landing while a well-behaved sender is still inside its
|
|
6289
|
+
// own `close()`. Unref'd so it can never hold the process open at shutdown, and tracked so
|
|
6290
|
+
// teardown can drop it.
|
|
6291
|
+
if (stream.status === "open" || stream.status === "closing") {
|
|
6292
|
+
const linger = setTimeout(() => {
|
|
6293
|
+
this.#lingeringStreams.delete(linger);
|
|
6294
|
+
if (stream.status !== "open" && stream.status !== "closing")
|
|
6295
|
+
return;
|
|
6296
|
+
this.#logger.debug("session.content.stream.linger.reset", { sessionId });
|
|
6297
|
+
try {
|
|
6298
|
+
stream.abort(new Error("inbound content stream not closed by peer"));
|
|
6299
|
+
}
|
|
6300
|
+
catch { /* already gone */ }
|
|
6301
|
+
}, CONTENT_STREAM_LINGER_MS);
|
|
6302
|
+
linger.unref?.();
|
|
6303
|
+
this.#lingeringStreams.add(linger);
|
|
6304
|
+
}
|
|
6305
|
+
}
|
|
5029
6306
|
}
|
|
5030
6307
|
/**
|
|
5031
6308
|
* M7-SESSION-001 AC-004/AC-005: Register a relay stream for an active session.
|
|
@@ -5681,6 +6958,62 @@ export class SessionNodeManager {
|
|
|
5681
6958
|
return flipped;
|
|
5682
6959
|
}
|
|
5683
6960
|
/** @returns true iff the UPDATE was executed without error (a failed write is logged, never thrown). */
|
|
6961
|
+
/**
|
|
6962
|
+
* DOD-M12B-STRAND-1 — move a terminal session's held frames to the annex.
|
|
6963
|
+
*
|
|
6964
|
+
* A held frame is content this agent RECEIVED and VERIFIED. When its session ends it can never
|
|
6965
|
+
* join that chain (appending behind a committed root is not an option, and ingest refuses a
|
|
6966
|
+
* terminal session outright), but it is still the operator's mail and no other copy exists —
|
|
6967
|
+
* the sender was never acknowledged for it. `sealed_session_annex` is where M12-P17 already puts
|
|
6968
|
+
* content that arrives for an ended session; this is the same content arriving slightly earlier.
|
|
6969
|
+
*
|
|
6970
|
+
* ANNEX FIRST, DELETE SECOND, per row. A crash between them costs a duplicate the annex's
|
|
6971
|
+
* INSERT OR IGNORE absorbs; the other order costs the message. A row whose annex write fails is
|
|
6972
|
+
* KEPT — the retention sweep will find it again, and a leftover row is cheaper than a lost one.
|
|
6973
|
+
*/
|
|
6974
|
+
#annexHeldContentOnTerminal(agentName, sessionId, status) {
|
|
6975
|
+
if (!this.#db)
|
|
6976
|
+
return;
|
|
6977
|
+
let rows;
|
|
6978
|
+
try {
|
|
6979
|
+
rows = this.#db.prepare(`SELECT canonical_seq, content_blob, content_hash_hex, held_at, origin
|
|
6980
|
+
FROM held_content WHERE agent_id = ? AND session_id = ? ORDER BY canonical_seq ASC`).all(this.#requireAgentId(agentName), sessionId);
|
|
6981
|
+
}
|
|
6982
|
+
catch (err) {
|
|
6983
|
+
this.#logger.error("session.content.held.annex.scan.failed", {
|
|
6984
|
+
agentName, sessionId, status,
|
|
6985
|
+
impact: "held frames for a terminal session were not moved to the annex and remain unreadable",
|
|
6986
|
+
error: err instanceof Error ? err.message : String(err),
|
|
6987
|
+
});
|
|
6988
|
+
return;
|
|
6989
|
+
}
|
|
6990
|
+
if (rows.length === 0)
|
|
6991
|
+
return;
|
|
6992
|
+
const counterparty = this.getSessionRecord(agentName, sessionId)?.counterparty_pubkey ?? null;
|
|
6993
|
+
let annexed = 0;
|
|
6994
|
+
let kept = 0;
|
|
6995
|
+
for (const row of rows) {
|
|
6996
|
+
// DOD-M12B-INDEX-1: ATTRIBUTION. A `sent` row is our own message. Stamping the counterparty's
|
|
6997
|
+
// pubkey on it would put our words in their mouth in the one record that survives the
|
|
6998
|
+
// session — the same failure the release path was changed to avoid, on the drain it did not
|
|
6999
|
+
// touch. `#ownPubkeyHex` is null only when the identity cannot be resolved, and a null sender
|
|
7000
|
+
// reads as "unattributed", which is true, rather than as a false attribution.
|
|
7001
|
+
const sender = row.origin === "sent" ? this.#ownPubkeyHex(agentName) : counterparty;
|
|
7002
|
+
if (this.recordSealedAnnex(agentName, sessionId, row.content_hash_hex, new Uint8Array(row.content_blob), sender)) {
|
|
7003
|
+
this.#deleteHeldContent(agentName, sessionId, row.canonical_seq);
|
|
7004
|
+
annexed++;
|
|
7005
|
+
}
|
|
7006
|
+
else {
|
|
7007
|
+
kept++;
|
|
7008
|
+
}
|
|
7009
|
+
}
|
|
7010
|
+
this.#logger.warn("session.content.held.annexed", {
|
|
7011
|
+
agentName, sessionId, status, annexed, kept,
|
|
7012
|
+
// The consumer of `held_at`: how long the oldest frame waited before its session ended.
|
|
7013
|
+
oldestHeldMs: Date.now() - Math.min(...rows.map((r) => r.held_at)),
|
|
7014
|
+
impact: "these messages arrived and verified but never joined the chain — they are readable from the annex, not the transcript",
|
|
7015
|
+
});
|
|
7016
|
+
}
|
|
5684
7017
|
#updateSessionStatus(agentName, sessionId, status) {
|
|
5685
7018
|
if (!this.#db)
|
|
5686
7019
|
return false;
|
|
@@ -5708,6 +7041,14 @@ export class SessionNodeManager {
|
|
|
5708
7041
|
// is still, on disk, drainable. 'interrupted' and 'seal_interrupted_pending' are deliberately
|
|
5709
7042
|
// NOT terminal — both can still complete, and reaping them would destroy live content.
|
|
5710
7043
|
if (status === "sealed" || status === "abandoned") {
|
|
7044
|
+
// DOD-M12B-STRAND-1: held frames outlive the chain that could have carried them.
|
|
7045
|
+
//
|
|
7046
|
+
// Once a session is terminal, `ingestReceivedContent` refuses it — and #releaseHeld is only
|
|
7047
|
+
// reachable from ingest — so no code path that exists can ever release a held frame again.
|
|
7048
|
+
// Left alone the rows sit on disk, unreachable by any surface, while the teardown alarm
|
|
7049
|
+
// reports `lost: 0`: a success message for content that has just become permanently
|
|
7050
|
+
// unreadable. The annex is the store built for exactly this shape.
|
|
7051
|
+
this.#annexHeldContentOnTerminal(agentName, sessionId, status);
|
|
5711
7052
|
try {
|
|
5712
7053
|
this.#onSessionTerminal?.(sessionId, status);
|
|
5713
7054
|
}
|