@cello-protocol/daemon 0.0.195 → 0.0.197
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/held-content.d.ts +7 -3
- package/dist/held-content.d.ts.map +1 -1
- package/dist/held-content.js.map +1 -1
- package/dist/session-assignment-parser.d.ts +11 -6
- package/dist/session-assignment-parser.d.ts.map +1 -1
- package/dist/session-assignment-parser.js +11 -6
- package/dist/session-assignment-parser.js.map +1 -1
- package/dist/session-content-context.d.ts +144 -0
- package/dist/session-content-context.d.ts.map +1 -0
- package/dist/session-content-context.js +2 -0
- package/dist/session-content-context.js.map +1 -0
- package/dist/session-content-ingest.d.ts +208 -0
- package/dist/session-content-ingest.d.ts.map +1 -0
- package/dist/session-content-ingest.js +2216 -0
- package/dist/session-content-ingest.js.map +1 -0
- package/dist/session-content-send.d.ts +184 -0
- package/dist/session-content-send.d.ts.map +1 -0
- package/dist/session-content-send.js +1300 -0
- package/dist/session-content-send.js.map +1 -0
- package/dist/session-lifecycle.d.ts +303 -0
- package/dist/session-lifecycle.d.ts.map +1 -0
- package/dist/session-lifecycle.js +1643 -0
- package/dist/session-lifecycle.js.map +1 -0
- package/dist/session-node-manager.d.ts +67 -766
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +966 -7940
- package/dist/session-node-manager.js.map +1 -1
- package/dist/session-node-types.d.ts +22 -0
- package/dist/session-node-types.d.ts.map +1 -1
- package/dist/session-node-types.js.map +1 -1
- package/dist/session-relay.d.ts +361 -0
- package/dist/session-relay.d.ts.map +1 -0
- package/dist/session-relay.js +1471 -0
- package/dist/session-relay.js.map +1 -0
- package/dist/session-salts.d.ts +13 -0
- package/dist/session-salts.d.ts.map +1 -1
- package/dist/session-salts.js +13 -0
- package/dist/session-salts.js.map +1 -1
- package/dist/session-seal.d.ts +336 -0
- package/dist/session-seal.d.ts.map +1 -0
- package/dist/session-seal.js +948 -0
- package/dist/session-seal.js.map +1 -0
- package/package.json +5 -5
|
@@ -0,0 +1,1471 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CELLO Daemon — THE BLIND WITNESS, AND STAYING REACHABLE THROUGH IT
|
|
3
|
+
*
|
|
4
|
+
* Split out of `session-node-manager.ts`. A relay countersigns the ORDER of a conversation and is
|
|
5
|
+
* given only hashes, never plaintext — so what lives here is everything about talking to one:
|
|
6
|
+
* connecting a session to its witness, proving key possession to it, holding a circuit reservation
|
|
7
|
+
* open so a counterparty behind a NAT can still reach us, watching that reservation decay and
|
|
8
|
+
* renewing it, quarantining a relay that misbehaves, and detaching cleanly when a session ends.
|
|
9
|
+
*
|
|
10
|
+
* **Moved verbatim, comments included.**
|
|
11
|
+
*
|
|
12
|
+
* ⚠️ **A RELAY IS NOT TRUSTED, AND THE CODE HERE IS WHERE THAT IS ENFORCED.** It authenticates to
|
|
13
|
+
* us as much as we authenticate to it; it is quarantined rather than believed when it misbehaves;
|
|
14
|
+
* and the one thing it must never learn — the operator's own address — is why the circuit-dial
|
|
15
|
+
* authorisation below is narrow rather than convenient. Anything added here that widens what a
|
|
16
|
+
* relay can see or do is a protocol change, not a refactor.
|
|
17
|
+
*/
|
|
18
|
+
import { randomUUID } from "node:crypto";
|
|
19
|
+
import * as lp from "it-length-prefixed";
|
|
20
|
+
import { decode } from "cbor-x";
|
|
21
|
+
import { isValidMultiaddr } from "@cello-protocol/transport";
|
|
22
|
+
import { contentHashFor } from "./wire-content-hash.js";
|
|
23
|
+
import { extractErrorMessage } from "./error-message.js";
|
|
24
|
+
import { RelayReceiptStore } from "./relay-receipt-store.js";
|
|
25
|
+
import { SessionSealLeafStore } from "./session-seal-leaf-store.js";
|
|
26
|
+
import { AgentRelayClient } from "./session-relay-client.js";
|
|
27
|
+
import { RELAY_QUARANTINE_MS, SR_RESERVATION_MAX_RETRIES, heldRelayIdsOf, relayPeerIdOf, } from "./session-node-types.js";
|
|
28
|
+
export class SessionRelay {
|
|
29
|
+
#ctx;
|
|
30
|
+
constructor(ctx) {
|
|
31
|
+
this.#ctx = ctx;
|
|
32
|
+
}
|
|
33
|
+
/** A getter so the moved queries still read `this.#db` and narrow exactly as they did. */
|
|
34
|
+
get #db() {
|
|
35
|
+
return this.#ctx.db();
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* DOD-M15-RELAYAUTH-1: authenticate a fresh standing receiver to its reservation relay over the
|
|
39
|
+
* CELLO relay protocol — proof of K_local key possession, not a session. Reuses the SAME
|
|
40
|
+
* `#relayClients` cache `connectSessionRelay`/`#resolveSealTransport` read from, keyed
|
|
41
|
+
* identically (`${agentName}::${relayPeerId}`), so a session created moments later on this same
|
|
42
|
+
* relay finds an already-authenticated client instead of dialing and authenticating twice.
|
|
43
|
+
*
|
|
44
|
+
* The manager holds no K_local (M12-P15's own rationale for `#detachedRelayClientBuilder`) —
|
|
45
|
+
* without a builder wired (a narrow startup race, or a test harness that never wires one), this
|
|
46
|
+
* is a no-op and the relay's own grace-window revoke is the backstop, not a defect in this path.
|
|
47
|
+
*/
|
|
48
|
+
async authenticateStandingReceiver(agentName, node, relayPeerId, heldCircuitAddr, correlationId) {
|
|
49
|
+
const clientKey = `${agentName}::${relayPeerId}`;
|
|
50
|
+
let client = this.#ctx.relayClients.get(clientKey);
|
|
51
|
+
if (!client) {
|
|
52
|
+
if (!this.#ctx.relayReceiptStore && this.#db)
|
|
53
|
+
this.#ctx.relayReceiptStore = new RelayReceiptStore(this.#db, this.#ctx.logger);
|
|
54
|
+
if (!this.#ctx.sealLeafStore && this.#db)
|
|
55
|
+
this.#ctx.sealLeafStore = new SessionSealLeafStore(this.#db, this.#ctx.logger);
|
|
56
|
+
/**
|
|
57
|
+
* ⚠️ SPLIT, NOT AN ANCHORED STRIP. The held address is
|
|
58
|
+
* `/ip4/…/tcp/…/p2p/<relay>/p2p-circuit/p2p/<self>` — the `/p2p-circuit` marker is in the
|
|
59
|
+
* MIDDLE, not at the end, so a `/\/p2p-circuit$/` replace matches nothing and silently
|
|
60
|
+
* hands the relay client a circuit address as its DIAL address. Measured, not assumed:
|
|
61
|
+
* that first version failed this file's own test because the client could not dial.
|
|
62
|
+
*/
|
|
63
|
+
const baseRelayAddr = heldCircuitAddr.split("/p2p-circuit")[0] ?? heldCircuitAddr;
|
|
64
|
+
client = this.#ctx.detachedRelayClientBuilder?.(agentName, relayPeerId, [baseRelayAddr], {
|
|
65
|
+
receiptStore: this.#ctx.relayReceiptStore ?? undefined,
|
|
66
|
+
sealLeafStore: this.#ctx.sealLeafStore ?? undefined,
|
|
67
|
+
ownChainStore: this.#ctx.ownChainStore ?? undefined,
|
|
68
|
+
// DOD-M15-RELAYSLOTS-1: read at each auth, never snapshotted — the token expires hourly.
|
|
69
|
+
onlineToken: () => this.#ctx.getDirectoryOnlineToken(agentName),
|
|
70
|
+
});
|
|
71
|
+
if (!client) {
|
|
72
|
+
/**
|
|
73
|
+
* Review M5: this was a `debug` line, and it is not a debug-level event.
|
|
74
|
+
*
|
|
75
|
+
* If no builder is wired we return without proving key possession, the relay revokes this
|
|
76
|
+
* receiver's reservation about fifteen seconds later, and the agent stops being reachable
|
|
77
|
+
* from behind NAT — while reporting itself perfectly healthy. Calling that "a narrow startup
|
|
78
|
+
* race" in a comment concedes it happens in production, and a system that is unreachable
|
|
79
|
+
* must not be quieter about it than a system that is merely slow.
|
|
80
|
+
*/
|
|
81
|
+
this.#ctx.logger.warn("session.standing_receiver.relay_auth.no_builder", {
|
|
82
|
+
agentName,
|
|
83
|
+
relayPeerId,
|
|
84
|
+
correlationId,
|
|
85
|
+
impact: "this receiver cannot prove key possession, so the relay will revoke its reservation and " +
|
|
86
|
+
"the agent becomes unreachable from behind NAT — nobody can start a session with it — " +
|
|
87
|
+
"even though it still reports itself online.",
|
|
88
|
+
});
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
this.#ctx.relayClients.set(clientKey, client);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* ⚠️ `proveReservation`, NOT `connect`. Review HIGH-1: `connect()` short-circuits on the
|
|
95
|
+
* client's cached stream, which belongs to whichever node connected FIRST — so every
|
|
96
|
+
* REPLACEMENT standing receiver (the one built behind each new session) sent nothing, the relay
|
|
97
|
+
* never saw its transport identity, and its reservation was revoked ~15s later. The agent then
|
|
98
|
+
* churned reserve→revoke→rebuild for the life of the conversation, holding no usable circuit
|
|
99
|
+
* address, so while you were talking to one person nobody else could reach you.
|
|
100
|
+
*
|
|
101
|
+
* `proveReservation` always opens its own stream from THIS node, and marks it so the relay
|
|
102
|
+
* proves possession without rebinding the agent's delivery target away from the live session.
|
|
103
|
+
*/
|
|
104
|
+
const proven = await client.proveReservation(node);
|
|
105
|
+
/**
|
|
106
|
+
* DOD-M15-RELAYSLOTS-1: keep the relay's refusal where the OPERATOR can reach it.
|
|
107
|
+
*
|
|
108
|
+
* The log line below is a good log line and it is not an answer to "why is my agent
|
|
109
|
+
* unreachable?" — nobody opens the file. The relay now refuses for reasons a person can act on
|
|
110
|
+
* (no token yet, too many sessions still open, this relay is misconfigured), each with its own
|
|
111
|
+
* next step, and every one of them is useless if it stops at a log.
|
|
112
|
+
*/
|
|
113
|
+
if (!proven) {
|
|
114
|
+
const refusal = client.getLastAuthRefusal();
|
|
115
|
+
/**
|
|
116
|
+
* Review L2: the `else` is not symmetry for its own sake. `proveReservation` also fails for
|
|
117
|
+
* transport reasons, which leave `getLastAuthRefusal()` null — and without this branch the
|
|
118
|
+
* PREVIOUS refusal stayed in the map, so `cello_status` went on showing a cause and an
|
|
119
|
+
* affordance for something that was no longer what was wrong.
|
|
120
|
+
*/
|
|
121
|
+
if (refusal)
|
|
122
|
+
this.#ctx.srRelayRefusal.set(agentName, { ...this.#ctx.withDirectoryCause(agentName, refusal), relayPeerId });
|
|
123
|
+
else
|
|
124
|
+
this.#ctx.srRelayRefusal.delete(agentName);
|
|
125
|
+
/**
|
|
126
|
+
* DOD-M15-RELAYSLOTS-1 clause 9 — **ACT on the classification, do not merely record it.**
|
|
127
|
+
* A relay-side fault means a different relay will work now, so quarantine this one and
|
|
128
|
+
* rebuild the receiver against the rest of the pool. Everything else stays put: a token
|
|
129
|
+
* problem reproduces on every relay, and walking the fleet would turn one client fault into
|
|
130
|
+
* what looks like a fleet-wide outage.
|
|
131
|
+
*/
|
|
132
|
+
if (refusal?.tryAnotherRelay && !this.#ctx.shuttingDown) {
|
|
133
|
+
this.#quarantineRelay(agentName, relayPeerId, refusal.reason);
|
|
134
|
+
void this.#ctx.receivers.rebuildStandingReceiver(agentName);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
this.#ctx.srRelayRefusal.delete(agentName);
|
|
139
|
+
}
|
|
140
|
+
this.#ctx.logger.info("session.standing_receiver.relay_auth.result", {
|
|
141
|
+
agentName,
|
|
142
|
+
relayPeerId,
|
|
143
|
+
// The node that actually proved itself — without this, HIGH-1 was invisible in the logs: the
|
|
144
|
+
// line said `connected: true` for a receiver that had sent nothing.
|
|
145
|
+
nodePeerId: node.getPeerId(),
|
|
146
|
+
proven,
|
|
147
|
+
...(proven ? {} : {
|
|
148
|
+
refusalReason: client.getLastAuthRefusal()?.reason ?? "no_relay_verdict",
|
|
149
|
+
tryAnotherRelay: client.getLastAuthRefusal()?.tryAnotherRelay ?? false,
|
|
150
|
+
}),
|
|
151
|
+
correlationId,
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* RELAYSIG-1: the durably-stored, signature-verified relay ordering-record receipts for an agent
|
|
156
|
+
* (optionally a single session). Empty when no receipts have been recorded yet. Read-only.
|
|
157
|
+
*/
|
|
158
|
+
getRelayReceipts(agentPubkeyHex, sessionIdHex) {
|
|
159
|
+
if (!this.#ctx.relayReceiptStore && this.#db) {
|
|
160
|
+
this.#ctx.relayReceiptStore = new RelayReceiptStore(this.#db, this.#ctx.logger);
|
|
161
|
+
}
|
|
162
|
+
return this.#ctx.relayReceiptStore?.getAll(agentPubkeyHex, sessionIdHex) ?? [];
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* DOD-M15-REFUSEDEVIDENCE-1 — retain a message refused OUTSIDE `ingestReceivedContent`
|
|
166
|
+
* (`session-content-ingest.ts`).
|
|
167
|
+
*
|
|
168
|
+
* Review F6. The park drain terminally-blocks a message that arrived for an already-committed
|
|
169
|
+
* session, then confirm-deletes the relay copy — the one other route in the tree that discarded
|
|
170
|
+
* refused content, and the highest-suspicion combination in the product: hostile bytes aimed at a
|
|
171
|
+
* conversation somebody has already sealed. Shipped guidance now tells every operator that
|
|
172
|
+
* refused messages are kept, so this is made true rather than the promise narrowed.
|
|
173
|
+
*
|
|
174
|
+
* A thin delegate, not a second implementation: the bound, the dedup, the sequence allocation and
|
|
175
|
+
* the logging are the ones every other refusal uses.
|
|
176
|
+
*/
|
|
177
|
+
quarantineRefusedInbound(agentName, sessionId, reason, content, contentHashHex, senderPubkeyHex, correlationId) {
|
|
178
|
+
return this.#ctx.refusals.quarantineRefusedContent(agentName, sessionId, reason, content, contentHashHex, {
|
|
179
|
+
senderPubkeyHex, correlationId,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
/** The metadata half of a framed quarantine read — everything known ABOUT the message, none of it
|
|
183
|
+
* taken from the message. Split out so the framing module never touches the database. */
|
|
184
|
+
quarantineFrameMeta(agentName, sessionId, rec) {
|
|
185
|
+
return {
|
|
186
|
+
reason: rec.reason,
|
|
187
|
+
senderPubkeyHex: rec.senderPubkeyHex,
|
|
188
|
+
senderLabel: rec.senderPubkeyHex === null ? null : this.#ctx.records.getContactMoniker(agentName, rec.senderPubkeyHex),
|
|
189
|
+
// `attribution` is the column that exists to answer exactly this, so it is read rather than
|
|
190
|
+
// inferred from `sender_sig` being non-null — a stored signature that was never checked
|
|
191
|
+
// against the sender's key would otherwise be reported as VERIFIED.
|
|
192
|
+
signature: rec.attribution === "verified_signature" ? "VERIFIED" : "NOT SIGNED",
|
|
193
|
+
sessionId,
|
|
194
|
+
position: rec.sequence,
|
|
195
|
+
arrivedAtMs: rec.createdAt,
|
|
196
|
+
/**
|
|
197
|
+
* The hash OF THE RETAINED BYTES, recomputed here — not the hash the sender committed to.
|
|
198
|
+
*
|
|
199
|
+
* On the highest-value case in the whole unit those two differ ON PURPOSE:
|
|
200
|
+
* `content_hash_mismatch` means the sender's committed hash does not describe these bytes.
|
|
201
|
+
* Printing their claim over our bytes would label the payload with a hash it does not have,
|
|
202
|
+
* which is the one thing a reader would use this line to check.
|
|
203
|
+
*/
|
|
204
|
+
contentHashHex: Buffer.from(contentHashFor(rec.content, { alg: "sha256", salt: null })).toString("hex"),
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* The current standing receiver node's session-transport coordinates (peer id +
|
|
209
|
+
* listen multiaddrs), or null if it is not ready. These are the addresses a local
|
|
210
|
+
* SessionNegotiator advertises as this node's counterparty endpoint so the initiator
|
|
211
|
+
* can dial it, and the value an inbound session_assignment carries in its
|
|
212
|
+
* counterparty_session_* fields. Read-only — does NOT consume the standing receiver
|
|
213
|
+
* (unlike acceptSession, which hands it off).
|
|
214
|
+
*/
|
|
215
|
+
/**
|
|
216
|
+
* 032-RELAYSPREAD — would this receiver ADMIT an inbound dial from this relay?
|
|
217
|
+
*
|
|
218
|
+
* The gater's inbound carve-out is the security-sensitive half of the spread: only relays whose
|
|
219
|
+
* own reservation is confirmed held earn it, so a directory that merely NAMES a relay cannot dial
|
|
220
|
+
* in behind the gate. Nothing could observe that from outside the manager, and the review found
|
|
221
|
+
* the consequence: substituting the CANDIDATE list for the held list at the `setReservedRelayPeers`
|
|
222
|
+
* call kept every test in the unit green while shipping exactly that hole. A guard whose wiring
|
|
223
|
+
* cannot be observed is a guard nothing can test.
|
|
224
|
+
*
|
|
225
|
+
* Reads the live gater rather than a copy, so it cannot drift from what the gate actually does.
|
|
226
|
+
*/
|
|
227
|
+
isRelayCarvedOutInbound(agentName, relayPeerId) {
|
|
228
|
+
return this.#ctx.standingReceivers.get(agentName)?.gater.holdsInboundCarveOut(relayPeerId) ?? false;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* M7 DOD-SPINE-6 / MSG-001-3b: connect a session node to the relay witness and
|
|
232
|
+
* store the client on the active entry. Best-effort: a connect/auth failure logs
|
|
233
|
+
* and leaves relayClient undefined — the session is NOT destroyed and the direct
|
|
234
|
+
* content path keeps working (the relay-park/recovery path is MSG-001-3b's domain).
|
|
235
|
+
*
|
|
236
|
+
* ⚠️ THIS BLOCK SPENT TWO MILESTONES ABOVE THE WRONG METHOD. It sat stacked on top of
|
|
237
|
+
* `relayLeafHandler`'s own docblock, so the file showed two descriptions in a row and the first
|
|
238
|
+
* one described a method further down the page. The content split then carried it verbatim into
|
|
239
|
+
* `session-content-ingest.ts`, where `connectSessionRelay` does not exist at all and the
|
|
240
|
+
* misattribution could no longer be worked out from context. Returned to the method it describes.
|
|
241
|
+
*/
|
|
242
|
+
async connectSessionRelay(sessionId, node, agentName, relay, correlationId) {
|
|
243
|
+
try {
|
|
244
|
+
// The session node's gater admits only the counterparty; the relay witness is a
|
|
245
|
+
// third peer. Permit it OUTBOUND so the dial isn't denied — inbound stays
|
|
246
|
+
// counterparty-only (INV-5). The relay peer id comes from the signed assignment.
|
|
247
|
+
this.#ctx.activeNodes.get(this.#ctx.sessionKey(agentName, sessionId))?.gater.setAllowedOutboundPeer(relay.relayPeerId);
|
|
248
|
+
// One relay client per (AGENT, RELAY NODE). The relay keys by agent pubkey, so the
|
|
249
|
+
// collision H1 addresses is per relay; CELLO is federated, so a different session for
|
|
250
|
+
// the same agent may be assigned a DIFFERENT relay — that needs its own client.
|
|
251
|
+
const clientKey = `${agentName}::${relay.relayPeerId}`;
|
|
252
|
+
let client = this.#ctx.relayClients.get(clientKey);
|
|
253
|
+
if (!client) {
|
|
254
|
+
// RELAYSIG-1: one shared receipt store (keyed by agent_pubkey, so a single instance serves all
|
|
255
|
+
// agents + relays). Lazy — the encrypted DB is open by the time sessions are active.
|
|
256
|
+
if (!this.#ctx.relayReceiptStore && this.#db) {
|
|
257
|
+
this.#ctx.relayReceiptStore = new RelayReceiptStore(this.#db, this.#ctx.logger);
|
|
258
|
+
}
|
|
259
|
+
// FED-OPTIONB-SEAL-001: one shared seal-leaf log (keyed by agent_pubkey), same lazy lifecycle.
|
|
260
|
+
if (!this.#ctx.sealLeafStore && this.#db) {
|
|
261
|
+
this.#ctx.sealLeafStore = new SessionSealLeafStore(this.#db, this.#ctx.logger);
|
|
262
|
+
}
|
|
263
|
+
client = new AgentRelayClient({
|
|
264
|
+
relayPeerId: relay.relayPeerId,
|
|
265
|
+
relayAddrs: relay.relayAddrs,
|
|
266
|
+
keyProvider: relay.keyProvider,
|
|
267
|
+
senderPubkey: relay.senderPubkey,
|
|
268
|
+
logger: this.#ctx.logger,
|
|
269
|
+
receiptStore: this.#ctx.relayReceiptStore ?? undefined,
|
|
270
|
+
sealLeafStore: this.#ctx.sealLeafStore ?? undefined,
|
|
271
|
+
// DOD-M15-RELAYSLOTS-1: read at each auth, never snapshotted — the token expires hourly.
|
|
272
|
+
onlineToken: () => this.#ctx.getDirectoryOnlineToken(agentName),
|
|
273
|
+
// DOD-M15-CORROBORATE-1: a relay's witness alert reaches the operator's inbox from here.
|
|
274
|
+
// The DETACHED clients get the same callback from the builder in daemon.ts.
|
|
275
|
+
onWitnessAlert: (alert) => { this.#ctx.witness.recordRelayWitnessAlert(agentName, alert); },
|
|
276
|
+
onWitnessUnreadable: (relayPeerId, why) => { this.#ctx.records.recordRelayWitnessUnreadable(agentName, relayPeerId, why); },
|
|
277
|
+
});
|
|
278
|
+
this.#ctx.relayClients.set(clientKey, client);
|
|
279
|
+
}
|
|
280
|
+
const sessionIdHexForRelay = Buffer.from(relay.sessionIdBytes).toString("hex");
|
|
281
|
+
/**
|
|
282
|
+
* ⚠️ THE GENESIS IS WRITTEN BEFORE THE REGISTRATION, NOT AFTER — review F10.
|
|
283
|
+
*
|
|
284
|
+
* `#leafRecords.sessionGenesisPrevRoot` reads the entry's assignment first and the column second,
|
|
285
|
+
* and BOTH were still unset at this line: the entry's assignment is set below and the column
|
|
286
|
+
* is written below that. So the argument was always `undefined` here on a first attach, and
|
|
287
|
+
* the seed only survived because `registerSession` falls back to deriving one from the
|
|
288
|
+
* assignment it is handed. That is a dead argument standing next to a live fallback, which
|
|
289
|
+
* reads as deliberate and is the shape a later edit removes the wrong half of.
|
|
290
|
+
*/
|
|
291
|
+
const entry = this.#ctx.activeNodes.get(this.#ctx.sessionKey(agentName, sessionId));
|
|
292
|
+
if (entry)
|
|
293
|
+
entry.relayAssignment = relay.assignment;
|
|
294
|
+
if (relay.assignment)
|
|
295
|
+
this.#ctx.leafRecords.persistGenesisPrevRoot(agentName, sessionId, relay.assignment);
|
|
296
|
+
client.registerSession(sessionIdHexForRelay, node, this.#ctx.contentIn.relayLeafHandler(agentName, sessionId, correlationId), relay.assignment, this.#ctx.leafRecords.sessionGenesisPrevRoot(agentName, sessionId));
|
|
297
|
+
if (entry) {
|
|
298
|
+
entry.relayClient = client;
|
|
299
|
+
entry.relaySessionIdBytes = relay.sessionIdBytes;
|
|
300
|
+
entry.relayClientKey = clientKey;
|
|
301
|
+
// 2b: remember the relay endpoint so the content-park backstop deposits to the SAME relay.
|
|
302
|
+
entry.relayPeerId = relay.relayPeerId;
|
|
303
|
+
entry.relayAddrs = relay.relayAddrs;
|
|
304
|
+
// Review H1: the dial path needs the credential in hand, not just the endpoint.
|
|
305
|
+
// (Set above, before `registerSession`, so the genesis lookup it does has something to
|
|
306
|
+
// find — see the note there.)
|
|
307
|
+
// MSG-2 startup-flush: also PERSIST it, so a restart's crash-backstop flush (which runs
|
|
308
|
+
// before the in-memory entry exists) can deposit un-acked content to the same relay.
|
|
309
|
+
try {
|
|
310
|
+
this.#db
|
|
311
|
+
?.prepare("UPDATE sessions SET relay_peer_id = ?, relay_addrs = ?, updated_at = ? WHERE agent_id = ? AND session_id = ?")
|
|
312
|
+
.run(relay.relayPeerId, JSON.stringify(relay.relayAddrs), Date.now(), this.#ctx.requireAgentId(agentName), sessionId);
|
|
313
|
+
}
|
|
314
|
+
catch (err) {
|
|
315
|
+
this.#ctx.logger.warn("session.relay.endpoint.persist.failed", {
|
|
316
|
+
sessionId,
|
|
317
|
+
error: err instanceof Error ? err.message : String(err),
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
// The session was torn down while we were wiring — undo the registration.
|
|
323
|
+
client.unregisterSession(sessionIdHexForRelay);
|
|
324
|
+
if (!client.hasSessions() && this.#ctx.relayClients.get(clientKey) === client) {
|
|
325
|
+
client.close();
|
|
326
|
+
this.#ctx.relayClients.delete(clientKey);
|
|
327
|
+
}
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
// Proactively connect so the relay has this agent's stream to deliver leaves to
|
|
331
|
+
// (the RECEIVER must be connected before the counterparty submits). Best-effort.
|
|
332
|
+
await client.connect(node);
|
|
333
|
+
// DOD-M15-RELAYAUTH-1 review HIGH-2 — see below. Best-effort and non-blocking: a session must
|
|
334
|
+
// never fail to come up because a SECOND relay could not be told about it.
|
|
335
|
+
//
|
|
336
|
+
// Review M1: `.catch()` is not decoration. This is an unawaited promise, so a throw it does not
|
|
337
|
+
// handle is an unhandled rejection — and this file already carries a comment elsewhere about an
|
|
338
|
+
// absent `await` that became a remote process kill. The method's own try/catch does not cover
|
|
339
|
+
// its prologue, so the catch here is the only thing standing between a torn-down node and the
|
|
340
|
+
// daemon dying.
|
|
341
|
+
void this.#presentAssignmentToReservationRelay(agentName, node, relay, sessionIdHexForRelay, correlationId, entry)
|
|
342
|
+
.catch((err) => {
|
|
343
|
+
this.#ctx.logger.warn("session.relay.assignment.reservation_relay_failed", {
|
|
344
|
+
agentName,
|
|
345
|
+
sessionId: sessionIdHexForRelay.slice(0, 16),
|
|
346
|
+
error: extractErrorMessage(err),
|
|
347
|
+
impact: "inbound relayed dials to this node may be refused by its reservation relay; the session still works over the direct path and the park backstop",
|
|
348
|
+
correlationId,
|
|
349
|
+
});
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
catch (err) {
|
|
353
|
+
this.#ctx.logger.warn("session.relay.connect.error", {
|
|
354
|
+
sessionId,
|
|
355
|
+
error: err instanceof Error ? err.message : String(err),
|
|
356
|
+
correlationId,
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* DOD-M15-RELAYAUTH-1 review HIGH-2 — **THE RELAY THAT GATES THE DIAL IS NOT ALWAYS THE RELAY
|
|
362
|
+
* THAT HOLDS THE ASSIGNMENT, AND THE GATE DENIES WHEN THEY DIFFER.**
|
|
363
|
+
*
|
|
364
|
+
* Two relays are in play for one session, chosen by unrelated rules:
|
|
365
|
+
* - the WITNESS relay, `assignment.relay_endpoint`, picked by the directory. Both parties
|
|
366
|
+
* present `client_record_assignment` to it and to nowhere else.
|
|
367
|
+
* - the RESERVATION relay, whichever one this node's circuit address is held on — the first
|
|
368
|
+
* candidate that granted when it was a standing receiver.
|
|
369
|
+
*
|
|
370
|
+
* The counterparty dials our CIRCUIT address, so it is the RESERVATION relay whose gater is asked
|
|
371
|
+
* `denyOutboundRelayedConnection(them, us)`. With no assignment recorded there it finds no binding
|
|
372
|
+
* and refuses a completely legitimate dial. Two relays run in production and the client's own
|
|
373
|
+
* logs show the fall-through to the second candidate is frequent, so this is the ordinary case,
|
|
374
|
+
* not a corner: the session still opens, but every message falls to the store-and-forward park
|
|
375
|
+
* path, and the only trace is a denial on a relay nobody is tailing.
|
|
376
|
+
*
|
|
377
|
+
* So the node that will be DIALLED presents the same assignment to the relay that will be asked
|
|
378
|
+
* to allow it. Safe by construction: the assignment is self-authenticating (a per-node directory
|
|
379
|
+
* signature the relay verifies against its consortium set), so presenting it more widely grants
|
|
380
|
+
* nothing that forging it would not already require. No directory change, no new frame.
|
|
381
|
+
*/
|
|
382
|
+
async #presentAssignmentToReservationRelay(agentName, node, relay, sessionIdHex, correlationId, entry) {
|
|
383
|
+
// Review M1: the prologue below lives INSIDE the try. `listenAddresses()` on a node torn down
|
|
384
|
+
// while we were wiring is exactly the case handled 30 lines above at the caller, and out here it
|
|
385
|
+
// would have escaped both this method's catch and (before the caller's `.catch`) the process.
|
|
386
|
+
let reservationRelayPeerId;
|
|
387
|
+
try {
|
|
388
|
+
if (!relay.assignment)
|
|
389
|
+
return; // direct/legacy/persisted-reconnect: nothing to present anywhere
|
|
390
|
+
const heldCircuitAddr = node.listenAddresses().find((a) => a.includes("/p2p-circuit"));
|
|
391
|
+
if (!heldCircuitAddr)
|
|
392
|
+
return; // no reservation held → nobody will gate a dial to us
|
|
393
|
+
reservationRelayPeerId = /\/p2p\/([^/]+)\/p2p-circuit/.exec(heldCircuitAddr)?.[1];
|
|
394
|
+
if (!reservationRelayPeerId || reservationRelayPeerId === relay.relayPeerId)
|
|
395
|
+
return; // same relay — already recorded
|
|
396
|
+
const clientKey = `${agentName}::${reservationRelayPeerId}`;
|
|
397
|
+
let client = this.#ctx.relayClients.get(clientKey);
|
|
398
|
+
if (!client) {
|
|
399
|
+
if (!this.#ctx.relayReceiptStore && this.#db)
|
|
400
|
+
this.#ctx.relayReceiptStore = new RelayReceiptStore(this.#db, this.#ctx.logger);
|
|
401
|
+
if (!this.#ctx.sealLeafStore && this.#db)
|
|
402
|
+
this.#ctx.sealLeafStore = new SessionSealLeafStore(this.#db, this.#ctx.logger);
|
|
403
|
+
// Split on the marker, not an anchored strip: the held address is
|
|
404
|
+
// `…/p2p/<relay>/p2p-circuit/p2p/<self>`, so the marker is in the MIDDLE.
|
|
405
|
+
const baseRelayAddr = heldCircuitAddr.split("/p2p-circuit")[0] ?? heldCircuitAddr;
|
|
406
|
+
client = this.#ctx.detachedRelayClientBuilder?.(agentName, reservationRelayPeerId, [baseRelayAddr], {
|
|
407
|
+
receiptStore: this.#ctx.relayReceiptStore ?? undefined,
|
|
408
|
+
sealLeafStore: this.#ctx.sealLeafStore ?? undefined,
|
|
409
|
+
// DOD-M15-RELAYSLOTS-1: read at each auth, never snapshotted — the token expires hourly.
|
|
410
|
+
onlineToken: () => this.#ctx.getDirectoryOnlineToken(agentName),
|
|
411
|
+
});
|
|
412
|
+
if (!client)
|
|
413
|
+
return;
|
|
414
|
+
this.#ctx.relayClients.set(clientKey, client);
|
|
415
|
+
// Review M4: record the key so teardown releases this client too — `relayClientKey` names
|
|
416
|
+
// only the witness relay, so before this the client and its session registration leaked.
|
|
417
|
+
if (entry)
|
|
418
|
+
entry.extraRelayClientKeys = [...(entry.extraRelayClientKeys ?? []), clientKey];
|
|
419
|
+
}
|
|
420
|
+
// registerSession presents the assignment eagerly (see its own comment). No leaf handler: this
|
|
421
|
+
// relay is not witnessing the session, it only needs the binding that authorizes the dial.
|
|
422
|
+
// 033-ACKEMIT: no genesis is passed and none is needed. This client is not witnessing the
|
|
423
|
+
// session — it never submits — and `registerSession` derives a seed from the assignment
|
|
424
|
+
// anyway. Reaching into the session record for one here would also be reaching with the RELAY
|
|
425
|
+
// session id, which is not the key that record is stored under.
|
|
426
|
+
client.registerSession(sessionIdHex, node, undefined, relay.assignment);
|
|
427
|
+
this.#ctx.logger.info("session.relay.assignment.presented_to_reservation_relay", {
|
|
428
|
+
agentName,
|
|
429
|
+
sessionId: sessionIdHex.slice(0, 16),
|
|
430
|
+
witnessRelayPeerId: relay.relayPeerId,
|
|
431
|
+
reservationRelayPeerId,
|
|
432
|
+
impact: "the relay that will be asked to allow inbound circuit dials to this node now holds the assignment authorizing them",
|
|
433
|
+
correlationId,
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
catch (err) {
|
|
437
|
+
this.#ctx.logger.warn("session.relay.assignment.reservation_relay_failed", {
|
|
438
|
+
agentName,
|
|
439
|
+
sessionId: sessionIdHex.slice(0, 16),
|
|
440
|
+
reservationRelayPeerId,
|
|
441
|
+
error: extractErrorMessage(err),
|
|
442
|
+
impact: "inbound relayed dials to this node may be refused by its reservation relay; the session still works over the direct path and the park backstop",
|
|
443
|
+
correlationId,
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* M7 DOD-SPINE-6 / MSG-001-3b: detach a session from its (agent, relay) client and
|
|
449
|
+
* close the client when it has no remaining sessions. Idempotent and identity-guarded:
|
|
450
|
+
* the map delete only fires if the map still holds THIS client (a racing teardown of a
|
|
451
|
+
* sibling session must not close a freshly-created replacement client for the same key).
|
|
452
|
+
*/
|
|
453
|
+
detachSessionRelay(entry) {
|
|
454
|
+
const client = entry.relayClient;
|
|
455
|
+
const key = entry.relayClientKey;
|
|
456
|
+
if (!entry.relaySessionIdBytes)
|
|
457
|
+
return;
|
|
458
|
+
const sidHex = Buffer.from(entry.relaySessionIdBytes).toString("hex");
|
|
459
|
+
/**
|
|
460
|
+
* Review M4: release the EXTRA relay clients first — the ones opened to relays that gate circuit
|
|
461
|
+
* dials. `relayClientKey` above names only the witness relay, so these were registered and never
|
|
462
|
+
* unregistered: an authenticated relay stream and a `#sessions` entry leaked per session, and
|
|
463
|
+
* relay-side the dial-through binding they hold is then cleared only by the idle timer, which is
|
|
464
|
+
* now 24h. Runs before the early return below so it happens even for a session that never got a
|
|
465
|
+
* witness client.
|
|
466
|
+
*/
|
|
467
|
+
for (const extraKey of entry.extraRelayClientKeys ?? []) {
|
|
468
|
+
const extra = this.#ctx.relayClients.get(extraKey);
|
|
469
|
+
if (!extra)
|
|
470
|
+
continue;
|
|
471
|
+
extra.unregisterSession(sidHex);
|
|
472
|
+
if (!extra.hasSessions()) {
|
|
473
|
+
extra.close();
|
|
474
|
+
this.#ctx.relayClients.delete(extraKey);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
entry.extraRelayClientKeys = undefined;
|
|
478
|
+
if (!client)
|
|
479
|
+
return;
|
|
480
|
+
// Idempotent: clear the entry's reference so a second teardown of the same entry no-ops.
|
|
481
|
+
entry.relayClient = undefined;
|
|
482
|
+
client.unregisterSession(sidHex);
|
|
483
|
+
if (!client.hasSessions() && key && this.#ctx.relayClients.get(key) === client) {
|
|
484
|
+
client.close();
|
|
485
|
+
this.#ctx.relayClients.delete(key);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
/**
|
|
489
|
+
* DOD-M15-RELAYAUTH-1 review H1 — **THE GATE WAS DENYING THE LEGITIMATE DIAL, AND USUALLY.**
|
|
490
|
+
*
|
|
491
|
+
* With the gater installed, a relay refuses a circuit dial unless it already holds a
|
|
492
|
+
* directory-signed assignment naming both transport peer ids. Both parties get that assignment
|
|
493
|
+
* from the directory independently, and until this method existed, each only presented it to
|
|
494
|
+
* relays IT had chosen — so whether a dial was allowed came down to which of two independent
|
|
495
|
+
* network races finished first:
|
|
496
|
+
*
|
|
497
|
+
* 1. WE connect to the witness relay, then dial the counterparty's circuit address. ~2 RTT.
|
|
498
|
+
* 2. THEY connect to their witness relay, then — unawaited, on a fresh dial + auth + record —
|
|
499
|
+
* tell their RESERVATION relay about the session. ~3–4 RTT.
|
|
500
|
+
*
|
|
501
|
+
* Nothing sequenced (1) against (2), and (1) is shorter, so we usually arrived first and were
|
|
502
|
+
* refused. The session still opened and reported `transportMode: "relay"`, so the failure was
|
|
503
|
+
* invisible: every message for the life of that conversation quietly took the store-and-forward
|
|
504
|
+
* park path, and the only trace was a denial logged on a third machine nobody tails.
|
|
505
|
+
*
|
|
506
|
+
* The fix is to stop racing. Whoever is about to dial presents the assignment to the relay that
|
|
507
|
+
* will gate that dial, and WAITS for the relay to confirm it recorded it. The ordering becomes
|
|
508
|
+
* local to one thread of execution, so there is nothing left to lose.
|
|
509
|
+
*
|
|
510
|
+
* Safe by construction: we are a participant the assignment names, so the relay's own participant
|
|
511
|
+
* check passes; and the assignment is self-authenticating (a directory signature the relay
|
|
512
|
+
* verifies against its consortium set), so presenting it more widely grants nothing that forging
|
|
513
|
+
* it would not already require.
|
|
514
|
+
*
|
|
515
|
+
* Best-effort by design — a relay we cannot reach must not stop us from dialling. If the record
|
|
516
|
+
* fails we dial anyway: a dial that might be refused is strictly better than no dial.
|
|
517
|
+
*/
|
|
518
|
+
async authorizeCircuitDialsToCounterparty(agentName, sessionId, entry, addrs) {
|
|
519
|
+
const assignment = entry.relayAssignment;
|
|
520
|
+
if (!assignment || !entry.relaySessionIdBytes)
|
|
521
|
+
return; // direct/legacy/persisted: no credential to present
|
|
522
|
+
const sessionIdHex = Buffer.from(entry.relaySessionIdBytes).toString("hex");
|
|
523
|
+
// One presentation per distinct relay, not per address: a counterparty commonly advertises
|
|
524
|
+
// several circuit addresses on the SAME relay.
|
|
525
|
+
const seen = new Set();
|
|
526
|
+
for (const addr of addrs) {
|
|
527
|
+
const relayPeerId = /\/p2p\/([^/]+)\/p2p-circuit/.exec(addr)?.[1];
|
|
528
|
+
if (!relayPeerId || seen.has(relayPeerId))
|
|
529
|
+
continue;
|
|
530
|
+
seen.add(relayPeerId);
|
|
531
|
+
// The witness relay already has it — registerSession presented it when the session was wired.
|
|
532
|
+
if (relayPeerId === entry.relayPeerId)
|
|
533
|
+
continue;
|
|
534
|
+
const baseRelayAddr = addr.split("/p2p-circuit")[0] ?? addr;
|
|
535
|
+
try {
|
|
536
|
+
const clientKey = `${agentName}::${relayPeerId}`;
|
|
537
|
+
let client = this.#ctx.relayClients.get(clientKey);
|
|
538
|
+
if (!client) {
|
|
539
|
+
if (!this.#ctx.relayReceiptStore && this.#db)
|
|
540
|
+
this.#ctx.relayReceiptStore = new RelayReceiptStore(this.#db, this.#ctx.logger);
|
|
541
|
+
if (!this.#ctx.sealLeafStore && this.#db)
|
|
542
|
+
this.#ctx.sealLeafStore = new SessionSealLeafStore(this.#db, this.#ctx.logger);
|
|
543
|
+
client = this.#ctx.detachedRelayClientBuilder?.(agentName, relayPeerId, [baseRelayAddr], {
|
|
544
|
+
receiptStore: this.#ctx.relayReceiptStore ?? undefined,
|
|
545
|
+
sealLeafStore: this.#ctx.sealLeafStore ?? undefined,
|
|
546
|
+
// DOD-M15-RELAYSLOTS-1: read at each auth, never snapshotted — the token expires hourly.
|
|
547
|
+
onlineToken: () => this.#ctx.getDirectoryOnlineToken(agentName),
|
|
548
|
+
});
|
|
549
|
+
if (!client) {
|
|
550
|
+
this.#ctx.logger.warn("session.transport.dial_authorization.no_builder", {
|
|
551
|
+
sessionId,
|
|
552
|
+
relayPeerId,
|
|
553
|
+
impact: "cannot present the assignment to the relay that gates this dial; if the counterparty has not presented it either, the dial will be refused and every message will fall to the park path",
|
|
554
|
+
correlationId: entry.correlationId,
|
|
555
|
+
});
|
|
556
|
+
continue;
|
|
557
|
+
}
|
|
558
|
+
this.#ctx.relayClients.set(clientKey, client);
|
|
559
|
+
// Review M4: remember it so teardown releases it — this is a SECOND client for the
|
|
560
|
+
// session, and detach only knows about the witness one.
|
|
561
|
+
entry.extraRelayClientKeys = [...(entry.extraRelayClientKeys ?? []), clientKey];
|
|
562
|
+
}
|
|
563
|
+
// No leaf handler: this relay is not witnessing the session, it only needs the binding.
|
|
564
|
+
client.registerSession(sessionIdHex, entry.node, undefined, assignment, this.#ctx.leafRecords.sessionGenesisPrevRoot(agentName, sessionId));
|
|
565
|
+
const recorded = await client.recordAssignmentAndWait(entry.node, sessionIdHex);
|
|
566
|
+
if (recorded) {
|
|
567
|
+
this.#ctx.logger.info("session.transport.dial_authorized", {
|
|
568
|
+
sessionId,
|
|
569
|
+
relayPeerId,
|
|
570
|
+
impact: "the relay that gates this circuit dial now holds the assignment authorizing it",
|
|
571
|
+
correlationId: entry.correlationId,
|
|
572
|
+
});
|
|
573
|
+
}
|
|
574
|
+
else {
|
|
575
|
+
this.#ctx.logger.warn("session.transport.dial_authorization.not_recorded", {
|
|
576
|
+
sessionId,
|
|
577
|
+
relayPeerId,
|
|
578
|
+
impact: "the relay did not confirm the assignment; the dial below may be refused and messages would fall to the park path",
|
|
579
|
+
correlationId: entry.correlationId,
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
catch (err) {
|
|
584
|
+
this.#ctx.logger.warn("session.transport.dial_authorization.failed", {
|
|
585
|
+
sessionId,
|
|
586
|
+
relayPeerId,
|
|
587
|
+
error: extractErrorMessage(err),
|
|
588
|
+
impact: "could not tell the relay that gates this dial about the session; the dial is attempted anyway",
|
|
589
|
+
correlationId: entry.correlationId,
|
|
590
|
+
});
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* TEST-ONLY (M8C-INBOX-1 reviewer F1): buffer a received message + persist its transcript row,
|
|
596
|
+
* exactly as the real inbound path (appendVerifiedContent, in `session-content-ingest.ts`) does,
|
|
597
|
+
* WITHOUT standing up a session
|
|
598
|
+
* tree — so a test can drive a live cello_receive that advances the read watermark (the N3
|
|
599
|
+
* "delivery marks read" coupling). Only reachable via the CELLO_ENV=test IPC hook.
|
|
600
|
+
*/
|
|
601
|
+
/** CELLO_ENV=test only: patch a relay client and session-id bytes onto an existing active node entry
|
|
602
|
+
* so submitSealLeaf succeeds without a real relay handshake (used by the oneshot relay-path test). */
|
|
603
|
+
patchRelayClientForTest(agentName, sessionId, relayClient, relaySessionIdBytes) {
|
|
604
|
+
const entry = this.#ctx.activeNodes.get(this.#ctx.sessionKey(agentName, sessionId));
|
|
605
|
+
if (!entry)
|
|
606
|
+
throw new Error(`patchRelayClientForTest: no active node for (${agentName}, ${sessionId})`);
|
|
607
|
+
entry.relayClient = relayClient;
|
|
608
|
+
entry.relaySessionIdBytes = relaySessionIdBytes;
|
|
609
|
+
/**
|
|
610
|
+
* ⚠️ REGISTER THE SESSION TOO — the seam must leave the state production leaves.
|
|
611
|
+
*
|
|
612
|
+
* A relay client that has never been told about a session holds no starting point for it, and
|
|
613
|
+
* since `DOD-M15-SELFCHAIN-1` every submit on such a session is refused: there is nothing for
|
|
614
|
+
* the chain links to anchor to. Production always registers, because attaching a relay is what
|
|
615
|
+
* registration IS. A seam that attached the client and skipped the registration left fixtures
|
|
616
|
+
* exercising a refusal path, and the failure surfaced as "the seal never happened" in a test
|
|
617
|
+
* about away-mode replies.
|
|
618
|
+
*/
|
|
619
|
+
relayClient.registerSession(Buffer.from(relaySessionIdBytes).toString("hex"), entry.node, undefined, entry.relayAssignment, this.#ctx.leafRecords.sessionGenesisPrevRoot(agentName, sessionId));
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* M7-SESSION-001 AC-004/AC-005: Register a relay stream for an active session.
|
|
623
|
+
* Starts a background reader that watches for session_interrupted frames and
|
|
624
|
+
* stream close events. Both detection paths call markInterruptedWithDetails().
|
|
625
|
+
*
|
|
626
|
+
* The reader runs for the lifetime of the relay stream. If the stream closes
|
|
627
|
+
* without delivering a session_interrupted frame (AC-005 / 'stream_close' path),
|
|
628
|
+
* the session is still marked interrupted.
|
|
629
|
+
*
|
|
630
|
+
* @param sessionId The hex session ID
|
|
631
|
+
* @param stream The relay stream to monitor
|
|
632
|
+
* @param messageCount Number of message leaves at the time of registration
|
|
633
|
+
* (used as the count at interruption — best effort since exact count at frame
|
|
634
|
+
* receipt may differ, but this is the value available at stream setup time)
|
|
635
|
+
*/
|
|
636
|
+
registerRelayStream(agentName, sessionId, stream, messageCount = 0) {
|
|
637
|
+
void this.#watchRelayStream(agentName, sessionId, stream, messageCount);
|
|
638
|
+
}
|
|
639
|
+
/**
|
|
640
|
+
* Background relay stream watcher.
|
|
641
|
+
* Pseudocode:
|
|
642
|
+
* 1. Create LP-framed iterator over the stream
|
|
643
|
+
* 2. For each frame:
|
|
644
|
+
* a. If type === 'session_interrupted':
|
|
645
|
+
* - Record receivedInterruptFrame = true
|
|
646
|
+
* - Call markInterruptedWithDetails(sessionId, messageCount, 'relay_frame')
|
|
647
|
+
* - Break (no more frames expected)
|
|
648
|
+
* 3. On stream close (loop ends normally or with error):
|
|
649
|
+
* a. If !receivedInterruptFrame:
|
|
650
|
+
* - Call markInterruptedWithDetails(sessionId, messageCount, 'stream_close')
|
|
651
|
+
*/
|
|
652
|
+
async #watchRelayStream(agentName, sessionId, stream, messageCount) {
|
|
653
|
+
let receivedInterruptFrame = false;
|
|
654
|
+
// CELLO-M7-TRANSPORT-001: cast the stream input to lp.decode. Adding the
|
|
655
|
+
// @libp2p/autonat service (interface@3.2.2 / uint8arraylist v2) to the
|
|
656
|
+
// transport package surfaced a benign mixed-version split between the Stream
|
|
657
|
+
// type (now v2) and it-length-prefixed's expected Uint8ArrayList (v3). The two
|
|
658
|
+
// are structurally identical at runtime — this is a build-time-only artifact.
|
|
659
|
+
const lpSource = stream;
|
|
660
|
+
const source = lp.decode(lpSource)[Symbol.asyncIterator]();
|
|
661
|
+
try {
|
|
662
|
+
while (true) {
|
|
663
|
+
let result;
|
|
664
|
+
try {
|
|
665
|
+
result = await source.next();
|
|
666
|
+
}
|
|
667
|
+
catch {
|
|
668
|
+
// Stream error (e.g. stream aborted) — treat as stream close
|
|
669
|
+
break;
|
|
670
|
+
}
|
|
671
|
+
if (result.done || result.value === undefined)
|
|
672
|
+
break;
|
|
673
|
+
let frame;
|
|
674
|
+
try {
|
|
675
|
+
const bytes = result.value instanceof Uint8Array ? result.value
|
|
676
|
+
: Buffer.isBuffer(result.value) ? new Uint8Array(result.value)
|
|
677
|
+
: result.value.slice();
|
|
678
|
+
frame = decode(bytes);
|
|
679
|
+
}
|
|
680
|
+
catch {
|
|
681
|
+
continue;
|
|
682
|
+
}
|
|
683
|
+
if (frame["type"] === "session_interrupted") {
|
|
684
|
+
// H-3 SECURITY: this stream is registered (bound) to a specific
|
|
685
|
+
// sessionId. A malicious or buggy relay could put a DIFFERENT session_id
|
|
686
|
+
// in the frame body to target a session this stream is not authorized
|
|
687
|
+
// for (cross-session targeting). Never trust the frame's id: if the frame
|
|
688
|
+
// names a different session, reject it and keep watching the bound one.
|
|
689
|
+
const frameSessionId = typeof frame["session_id"] === "string"
|
|
690
|
+
? frame["session_id"]
|
|
691
|
+
: (frame["session_id"] instanceof Uint8Array
|
|
692
|
+
? Buffer.from(frame["session_id"]).toString("hex")
|
|
693
|
+
: null);
|
|
694
|
+
if (frameSessionId !== null && frameSessionId !== sessionId) {
|
|
695
|
+
this.#ctx.logger.warn("session.interrupt.frame.session_mismatch", {
|
|
696
|
+
boundSessionId: sessionId,
|
|
697
|
+
frameSessionId,
|
|
698
|
+
reason: "cross_session_frame_rejected",
|
|
699
|
+
});
|
|
700
|
+
continue; // ignore the hostile/mismatched frame; keep reading
|
|
701
|
+
}
|
|
702
|
+
receivedInterruptFrame = true;
|
|
703
|
+
// Always mark the BOUND sessionId — never the id carried in the frame.
|
|
704
|
+
await this.#ctx.markInterruptedWithDetails(agentName, sessionId, messageCount, "relay_frame");
|
|
705
|
+
break; // No more relay frames expected after session_interrupted
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
catch {
|
|
710
|
+
// Stream read loop ended — fall through to stream_close check
|
|
711
|
+
}
|
|
712
|
+
// AC-005: stream closed without a session_interrupted frame
|
|
713
|
+
if (!receivedInterruptFrame) {
|
|
714
|
+
// Only mark interrupted if this session is still active in SQLite
|
|
715
|
+
const record = this.#ctx.queries.getSessionRecord(agentName, sessionId);
|
|
716
|
+
if (record && record.status === "active") {
|
|
717
|
+
await this.#ctx.markInterruptedWithDetails(agentName, sessionId, messageCount, "stream_close");
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
/**
|
|
722
|
+
* DOD-NAT-REACHABILITY-1 (Phase 2): accept the directory's relay-pool endpoints
|
|
723
|
+
* for an agent (arrives with signaling_auth_ok, i.e. on every connect AND every
|
|
724
|
+
* reconnect). If the agent's standing receiver is up but holds NO reservation —
|
|
725
|
+
* the agent-online ensure raced ahead of auth_ok, or every relay was down at
|
|
726
|
+
* create time — rebuild it now so the agent becomes dialable without waiting
|
|
727
|
+
* for a session handoff that (being unreachable) would never come.
|
|
728
|
+
*/
|
|
729
|
+
setDirectoryRelayEndpoints(agentName, endpoints) {
|
|
730
|
+
// DOD-M12B-RESERVATION-RETRY-1: a DIFFERENT relay pool re-arms the retry budget; the same one
|
|
731
|
+
// does not. This fires on every signaling connect AND reconnect, so clearing unconditionally
|
|
732
|
+
// would reset the bound on a short grid and defeat the whole point of having one. But once the
|
|
733
|
+
// budget is spent, `getStandingReceiverReachability` reports `unreachable` for the rest of the
|
|
734
|
+
// online episode — including while this path is actively re-attempting against relays we have
|
|
735
|
+
// never tried. A new relay is new information; a repeat of the same list is not.
|
|
736
|
+
const previous = this.#ctx.directoryRelayEndpoints.get(agentName);
|
|
737
|
+
const poolChanged = previous === undefined ||
|
|
738
|
+
previous.length !== endpoints.length ||
|
|
739
|
+
endpoints.some((e, i) => e.relayPeerId !== previous[i]?.relayPeerId);
|
|
740
|
+
this.#ctx.directoryRelayEndpoints.set(agentName, endpoints);
|
|
741
|
+
if (poolChanged) {
|
|
742
|
+
this.#ctx.srReservationRetry.delete(agentName);
|
|
743
|
+
this.#ctx.srLastRejectionReason.delete(agentName);
|
|
744
|
+
}
|
|
745
|
+
if (endpoints.length === 0 || this.#ctx.shuttingDown)
|
|
746
|
+
return;
|
|
747
|
+
const sr = this.#ctx.standingReceivers.get(agentName);
|
|
748
|
+
if (!sr)
|
|
749
|
+
return; // not ensured yet — the coming ensure reads the map
|
|
750
|
+
if (sr.node.listenAddresses().some((a) => a.includes("/p2p-circuit")))
|
|
751
|
+
return; // already reserved
|
|
752
|
+
this.#ctx.logger.info("session.standing_receiver.reservation.rebuild", {
|
|
753
|
+
agentName,
|
|
754
|
+
relayPeerIds: endpoints.map((e) => e.relayPeerId),
|
|
755
|
+
});
|
|
756
|
+
void this.#ctx.receivers.rebuildStandingReceiver(agentName);
|
|
757
|
+
}
|
|
758
|
+
/**
|
|
759
|
+
* Is this agent currently skipping this relay? The observable half of the failover decision — a
|
|
760
|
+
* test that asserts only on the classifier's boolean proves nothing about what the daemon does.
|
|
761
|
+
*/
|
|
762
|
+
isRelayQuarantined(agentName, relayPeerId) {
|
|
763
|
+
return this.#relayQuarantineFor(agentName).has(relayPeerId);
|
|
764
|
+
}
|
|
765
|
+
/** Live quarantine entries for an agent, expired ones swept on read. */
|
|
766
|
+
#relayQuarantineFor(agentName) {
|
|
767
|
+
const byRelay = this.#ctx.relayQuarantine.get(agentName);
|
|
768
|
+
if (!byRelay)
|
|
769
|
+
return new Set();
|
|
770
|
+
const now = Date.now();
|
|
771
|
+
for (const [relayPeerId, expiresAt] of byRelay) {
|
|
772
|
+
if (now >= expiresAt)
|
|
773
|
+
byRelay.delete(relayPeerId);
|
|
774
|
+
}
|
|
775
|
+
if (byRelay.size === 0)
|
|
776
|
+
this.#ctx.relayQuarantine.delete(agentName);
|
|
777
|
+
return new Set(byRelay.keys());
|
|
778
|
+
}
|
|
779
|
+
/**
|
|
780
|
+
* Skip this relay for this agent for a while. Called only for refusals the classifier marks
|
|
781
|
+
* `tryAnotherRelay` — a fault of the relay's, not one that would follow us to the next one.
|
|
782
|
+
*/
|
|
783
|
+
#quarantineRelay(agentName, relayPeerId, reason) {
|
|
784
|
+
let byRelay = this.#ctx.relayQuarantine.get(agentName);
|
|
785
|
+
if (!byRelay) {
|
|
786
|
+
byRelay = new Map();
|
|
787
|
+
this.#ctx.relayQuarantine.set(agentName, byRelay);
|
|
788
|
+
}
|
|
789
|
+
byRelay.set(relayPeerId, Date.now() + RELAY_QUARANTINE_MS);
|
|
790
|
+
this.#ctx.logger.warn("session.standing_receiver.relay_quarantined", {
|
|
791
|
+
agentName,
|
|
792
|
+
relayPeerId,
|
|
793
|
+
reason,
|
|
794
|
+
forMs: RELAY_QUARANTINE_MS,
|
|
795
|
+
impact: "this relay refused this agent for a fault of its own, so the agent will ask a " +
|
|
796
|
+
"different relay for its reservation until the quarantine lapses. Its inbound reachability " +
|
|
797
|
+
"is restored by moving, not by waiting for someone to fix that relay.",
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
reservationCircuitAddrs(agentName) {
|
|
801
|
+
let persisted;
|
|
802
|
+
try {
|
|
803
|
+
persisted = this.#ctx.queries.getAgentRelayEndpoints(agentName);
|
|
804
|
+
}
|
|
805
|
+
catch (err) {
|
|
806
|
+
// No DB / unknown agent — persisted source unavailable. The directory
|
|
807
|
+
// source may still serve; reachability degrades only if both are empty.
|
|
808
|
+
// Logged (not swallowed): a genuine DB failure must be distinguishable
|
|
809
|
+
// from "fresh agent, no history" in the reachability trail.
|
|
810
|
+
this.#ctx.logger.debug("session.standing_receiver.persisted_relays.unavailable", {
|
|
811
|
+
agentName,
|
|
812
|
+
error: extractErrorMessage(err),
|
|
813
|
+
});
|
|
814
|
+
persisted = [];
|
|
815
|
+
}
|
|
816
|
+
const merged = new Map();
|
|
817
|
+
for (const ep of [...(this.#ctx.directoryRelayEndpoints.get(agentName) ?? []), ...persisted]) {
|
|
818
|
+
if (!merged.has(ep.relayPeerId))
|
|
819
|
+
merged.set(ep.relayPeerId, ep);
|
|
820
|
+
}
|
|
821
|
+
/**
|
|
822
|
+
* DOD-M15-RELAYSLOTS-1 — **THE FAILOVER.** Skip relays that refused this agent for a fault of
|
|
823
|
+
* their own (today: a relay holding no directory public key, which can verify nobody and is
|
|
824
|
+
* refusing everyone). We run several relays precisely so one being broken is survivable.
|
|
825
|
+
*
|
|
826
|
+
* ⚠️ NEVER TO THE POINT OF HAVING NO RELAY AT ALL. If the quarantine would empty the candidate
|
|
827
|
+
* list it is ignored wholesale: a relay that refuses is strictly better than no relay, because
|
|
828
|
+
* the refusal at least has a cause the operator can read, while an agent with no candidates is
|
|
829
|
+
* simply unreachable with nothing to show for it. This is the same "refusing too eagerly is the
|
|
830
|
+
* failure mode" rule, applied to the client's own choice of where to ask.
|
|
831
|
+
*/
|
|
832
|
+
const quarantined = this.#relayQuarantineFor(agentName);
|
|
833
|
+
const eligible = [...merged.values()].filter((ep) => !quarantined.has(ep.relayPeerId));
|
|
834
|
+
const usable = eligible.length > 0 ? eligible : [...merged.values()];
|
|
835
|
+
if (eligible.length === 0 && quarantined.size > 0 && merged.size > 0) {
|
|
836
|
+
this.#ctx.logger.warn("session.standing_receiver.relay_quarantine.ignored", {
|
|
837
|
+
agentName,
|
|
838
|
+
quarantined: [...quarantined],
|
|
839
|
+
impact: "every known relay has refused this agent for a relay-side fault, so the quarantine " +
|
|
840
|
+
"is being ignored and they are all being tried again. A relay that refuses with a cause " +
|
|
841
|
+
"is better than no relay at all — but if this persists, every relay this agent knows " +
|
|
842
|
+
"about is misconfigured, and that is the thing to look at.",
|
|
843
|
+
});
|
|
844
|
+
}
|
|
845
|
+
const addrs = [];
|
|
846
|
+
const relayPeerIds = [];
|
|
847
|
+
for (const ep of usable) {
|
|
848
|
+
const base = ep.relayAddrs[0];
|
|
849
|
+
if (!base)
|
|
850
|
+
continue;
|
|
851
|
+
const candidate = base.includes("/p2p/")
|
|
852
|
+
? `${base}/p2p-circuit`
|
|
853
|
+
: `${base}/p2p/${ep.relayPeerId}/p2p-circuit`;
|
|
854
|
+
// These addresses are built from DIRECTORY-supplied endpoints — data from off
|
|
855
|
+
// this machine. A malformed one throws inside libp2p node construction, which
|
|
856
|
+
// would take the standing receiver down entirely and leave the agent deaf to
|
|
857
|
+
// ALL inbound (worse than the defect this fixes). A bad endpoint must cost one
|
|
858
|
+
// relay, never the receiver.
|
|
859
|
+
if (!isValidMultiaddr(candidate)) {
|
|
860
|
+
this.#ctx.logger.warn("session.standing_receiver.relay_endpoint.invalid", {
|
|
861
|
+
agentName,
|
|
862
|
+
relayPeerId: ep.relayPeerId,
|
|
863
|
+
addr: candidate,
|
|
864
|
+
});
|
|
865
|
+
continue;
|
|
866
|
+
}
|
|
867
|
+
addrs.push(candidate);
|
|
868
|
+
relayPeerIds.push(ep.relayPeerId);
|
|
869
|
+
}
|
|
870
|
+
return { addrs, relayPeerIds };
|
|
871
|
+
}
|
|
872
|
+
/**
|
|
873
|
+
* DOD-NAT-REACHABILITY-1: notice when a standing receiver has SILENTLY LOST its
|
|
874
|
+
* reservation, and get it another one.
|
|
875
|
+
*
|
|
876
|
+
* libp2p refreshes a circuit reservation before it expires. If the relay has died,
|
|
877
|
+
* that refresh fails and the /p2p-circuit address simply DISAPPEARS from the node's
|
|
878
|
+
* addresses. Nothing throws. The receiver is still up, still directly dialable, and
|
|
879
|
+
* still looks perfectly healthy — but no NAT'd peer can reach the agent any more.
|
|
880
|
+
* That is precisely the silent-loss-of-inbound failure this whole story exists to
|
|
881
|
+
* kill, so it cannot be left to chance: we watch for it and re-pick a relay.
|
|
882
|
+
*
|
|
883
|
+
* Only receivers that HAD a reservation are watched. One that never got one is
|
|
884
|
+
* already degraded and already loud (reservation.none / reservation.timeout);
|
|
885
|
+
* rebuilding it on a timer would just thrash against relays we know are refusing.
|
|
886
|
+
*/
|
|
887
|
+
/**
|
|
888
|
+
* DOD-M12B-RESERVATION-RETRY-1 — ask again for a reservation the relay refused.
|
|
889
|
+
*
|
|
890
|
+
* The rebuild is the re-attempt: a circuit listener is fixed at node creation, so the only way to
|
|
891
|
+
* obtain a reservation is to build a new node asking for one.
|
|
892
|
+
*/
|
|
893
|
+
#retryReservationIfDue(agentName) {
|
|
894
|
+
const now = Date.now();
|
|
895
|
+
const state = this.#ctx.srReservationRetry.get(agentName)
|
|
896
|
+
?? { attempts: 0, nextAt: now + this.#ctx.srReservationRetryMs, correlationId: randomUUID() };
|
|
897
|
+
// The reason the LAST attempt was refused, captured where it is actually known.
|
|
898
|
+
const lastReason = this.#ctx.srLastRejectionReason.get(agentName);
|
|
899
|
+
if (lastReason !== undefined)
|
|
900
|
+
state.lastReason = lastReason;
|
|
901
|
+
if (state.attempts === 0 && !this.#ctx.srReservationRetry.has(agentName)) {
|
|
902
|
+
// First sighting — schedule, do not fire. The creation attempt just happened.
|
|
903
|
+
this.#ctx.srReservationRetry.set(agentName, state);
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
if (now < state.nextAt)
|
|
907
|
+
return;
|
|
908
|
+
if (state.attempts >= SR_RESERVATION_MAX_RETRIES) {
|
|
909
|
+
if (state.attempts === SR_RESERVATION_MAX_RETRIES) {
|
|
910
|
+
state.attempts += 1; // mark as reported, so this fires exactly once
|
|
911
|
+
this.#ctx.srReservationRetry.set(agentName, state);
|
|
912
|
+
this.#ctx.logger.error("session.standing_receiver.reservation.gave_up", {
|
|
913
|
+
agentName,
|
|
914
|
+
attempts: SR_RESERVATION_MAX_RETRIES,
|
|
915
|
+
correlationId: state.correlationId,
|
|
916
|
+
// WHY, not just the consequence. Three different problems reach this one message and they
|
|
917
|
+
// need three different responses: `relay_granted_no_reservation` is relay CAPACITY (and a
|
|
918
|
+
// trustless-cello problem), `relay_unreachable` is the NETWORK, and
|
|
919
|
+
// `reservation_did_not_complete_in_time` is LATENCY — and the only one of the three that
|
|
920
|
+
// can pin a slot it never uses, so its appearance is also the signal that this retry
|
|
921
|
+
// budget needs tightening.
|
|
922
|
+
...(state.lastReason !== undefined ? { lastRejectionReason: state.lastReason } : {}),
|
|
923
|
+
// "No relay would grant" and "there was no relay to ask" are different facts and lead to
|
|
924
|
+
// different places — the first at relay capacity, the second at this agent's directory
|
|
925
|
+
// connection. Without this they are the same sentence.
|
|
926
|
+
//
|
|
927
|
+
// 032-RELAYSPREAD: this was also called `reservationsRequested` — the same mis-naming as
|
|
928
|
+
// the reachability events, in its worst form, because here the value is a BOOLEAN under a
|
|
929
|
+
// name that reads as a count. It is NOT `relaysOffered`: that field counts the merged
|
|
930
|
+
// candidate list the walk actually asks (directory pool + persisted endpoints, minus
|
|
931
|
+
// quarantine), and this reads the directory pool alone. Two populations must not share
|
|
932
|
+
// one field name, so this one is named for what it measures.
|
|
933
|
+
hadRelayToAsk: (this.#ctx.directoryRelayEndpoints.get(agentName)?.length ?? 0) > 0,
|
|
934
|
+
// …and HOW MANY the walk actually asks, so this event stands on its own instead of
|
|
935
|
+
// needing the last reachability line to be read beside it. Same population and same
|
|
936
|
+
// meaning as `relaysOffered` everywhere else: the merged, quarantine-filtered candidate
|
|
937
|
+
// list.
|
|
938
|
+
relaysOffered: this.reservationCircuitAddrs(agentName).addrs.length,
|
|
939
|
+
impact: "no relay would grant this agent a circuit reservation, so anyone behind NAT cannot reach or dial it — inbound sessions will only arrive from peers that can connect directly, and everything else falls back to the relay's store-and-forward",
|
|
940
|
+
});
|
|
941
|
+
}
|
|
942
|
+
return;
|
|
943
|
+
}
|
|
944
|
+
state.attempts += 1;
|
|
945
|
+
// The FINAL attempt gets a fixed settle window rather than another doubled wait: at the top of
|
|
946
|
+
// the ladder that would be 80 minutes of silence after the last thing we did, which is a long
|
|
947
|
+
// time to tell an operator nothing. Every earlier attempt doubles, which is what keeps a fleet
|
|
948
|
+
// off a scarce relay.
|
|
949
|
+
state.nextAt = now + (state.attempts >= SR_RESERVATION_MAX_RETRIES
|
|
950
|
+
? this.#ctx.srReservationRetryMs
|
|
951
|
+
: this.#ctx.srReservationRetryMs * 2 ** (state.attempts - 1));
|
|
952
|
+
this.#ctx.srReservationRetry.set(agentName, state);
|
|
953
|
+
this.#ctx.logger.warn("session.standing_receiver.reservation.retry", {
|
|
954
|
+
agentName,
|
|
955
|
+
attempt: state.attempts,
|
|
956
|
+
maxAttempts: SR_RESERVATION_MAX_RETRIES,
|
|
957
|
+
correlationId: state.correlationId,
|
|
958
|
+
...(state.lastReason !== undefined ? { lastRejectionReason: state.lastReason } : {}),
|
|
959
|
+
impact: "this agent currently holds no circuit reservation, so a NAT'd peer cannot dial it",
|
|
960
|
+
});
|
|
961
|
+
void this.#ctx.receivers.rebuildStandingReceiver(agentName);
|
|
962
|
+
}
|
|
963
|
+
#reservationWatchdogTick() {
|
|
964
|
+
if (this.#ctx.shuttingDown)
|
|
965
|
+
return;
|
|
966
|
+
for (const [agentName, sr] of this.#ctx.standingReceivers) {
|
|
967
|
+
if (!this.#ctx.agentsWantingReceiver.has(agentName))
|
|
968
|
+
continue; // agent went offline
|
|
969
|
+
// DOD-M12B-RESERVATION-RETRY-1 — NEVER HAD ONE IS NOT "NOTHING TO DO".
|
|
970
|
+
//
|
|
971
|
+
// This used to `continue` unconditionally, on the grounds that a receiver with no reservation
|
|
972
|
+
// is "already degraded and already loud". Measured over 17 days: `reservation.none` fired 481
|
|
973
|
+
// times and `relay.rejected` 2,215 — every one of the latter `relay_granted_no_reservation`,
|
|
974
|
+
// a relay out of slots completing the handshake and granting nothing. Nothing ever acted on
|
|
975
|
+
// the noise, and each of those receivers is a plain TCP node with no circuit address: behind
|
|
976
|
+
// NAT, dialable by NOBODY, for its whole life. That is the silent loss of inbound this file
|
|
977
|
+
// says three lines below it exists to kill.
|
|
978
|
+
//
|
|
979
|
+
// A relay out of slots at boot may have one minutes later, so re-attempt — on a BACKOFF and
|
|
980
|
+
// BOUNDED, never on this 30-second grid. A reservation is scarce: the relay holds it for its
|
|
981
|
+
// full TTL even after the client disconnects, and churning attempts across a fleet is how a
|
|
982
|
+
// relay is exhausted (`#startReceiverNode` records that hazard).
|
|
983
|
+
if (sr.relayPeerIds.length === 0) {
|
|
984
|
+
// …unless one has arrived since. Review F4, same class as the recompute below: the
|
|
985
|
+
// slow-start path installs a receiver before every circuit has bound, so "held nothing at
|
|
986
|
+
// install" is not the same fact as "holds nothing now". Adopting it here is what stops the
|
|
987
|
+
// retry ladder rebuilding a receiver that is already reachable.
|
|
988
|
+
const arrived = heldRelayIdsOf(sr.node)
|
|
989
|
+
.filter((id) => sr.node.getConnections().some((c) => c.peerId === id && c.status === "open"));
|
|
990
|
+
if (arrived.length === 0) {
|
|
991
|
+
this.#retryReservationIfDue(agentName);
|
|
992
|
+
continue;
|
|
993
|
+
}
|
|
994
|
+
sr.relayPeerIds = arrived;
|
|
995
|
+
sr.gater.setReservedRelayPeers(arrived);
|
|
996
|
+
this.#ctx.logger.info("session.standing_receiver.reservation.gained", {
|
|
997
|
+
agentName,
|
|
998
|
+
relayPeerIds: arrived,
|
|
999
|
+
reservationsHeld: arrived.length,
|
|
1000
|
+
});
|
|
1001
|
+
}
|
|
1002
|
+
// It has at least one — any earlier retry budget, and the reason the last attempt failed, are
|
|
1003
|
+
// stale.
|
|
1004
|
+
this.#ctx.srReservationRetry.delete(agentName);
|
|
1005
|
+
this.#ctx.srLastRejectionReason.delete(agentName);
|
|
1006
|
+
// Watch the CONNECTION to the relay, not the circuit address.
|
|
1007
|
+
//
|
|
1008
|
+
// Killing the relay does NOT make the /p2p-circuit address disappear: libp2p
|
|
1009
|
+
// keeps the listen address until the reservation's own refresh, up to two hours
|
|
1010
|
+
// away. Watching the address would therefore miss a dead relay for hours — the
|
|
1011
|
+
// agent would advertise a circuit address that routes through a relay that no
|
|
1012
|
+
// longer exists, which is exactly the silent unreachability we are hunting. The
|
|
1013
|
+
// live connection to the relay is the honest signal: no connection, no relay,
|
|
1014
|
+
// no reservation.
|
|
1015
|
+
//
|
|
1016
|
+
// AND IT MUST BE OPEN. `getConnections()` returns libp2p's registry, which keeps a connection
|
|
1017
|
+
// listed while it is closing and after its muxer has died — the state the whole M12 Tier P5
|
|
1018
|
+
// investigation turned on. Without the status check a registered corpse reads as "still
|
|
1019
|
+
// connected", the rebuild never fires, and the agent silently stops being reachable while
|
|
1020
|
+
// this loop reports it healthy. The comment above claimed liveness; only this tests it.
|
|
1021
|
+
// 032-RELAYSPREAD — PER RELAY, and the health question is now a COUNT.
|
|
1022
|
+
//
|
|
1023
|
+
// This used to evaluate one peer id and rebuild the entire standing receiver when it went
|
|
1024
|
+
// false. With a pool of one that was the only thing it could do; with a pool of three it is
|
|
1025
|
+
// the churn engine — every relay is another watchdog subject, and a full rebuild per loss
|
|
1026
|
+
// multiplies the 30-second grid by the size of the pool while throwing away reservations that
|
|
1027
|
+
// are perfectly healthy.
|
|
1028
|
+
/**
|
|
1029
|
+
* RECOMPUTED FROM THE NODE, never filtered down from the stored list. Review F4: filtering
|
|
1030
|
+
* `sr.relayPeerIds` makes it SHRINK-ONLY, and a list that can only shrink cannot see a
|
|
1031
|
+
* circuit arrive. Three things went wrong with that, and the first one happens routinely:
|
|
1032
|
+
* - the slow-start path installs the receiver before every circuit has bound, so a relay
|
|
1033
|
+
* that binds four seconds later was invisible to this watchdog and absent from the
|
|
1034
|
+
* gater's carve-out set FOREVER — its AutoNAT probe reply refused by our own gate;
|
|
1035
|
+
* - shrinking to zero then rebuilt a receiver that was announcing live circuits, which is
|
|
1036
|
+
* the exact defect this unit is against;
|
|
1037
|
+
* - anything that ever restores a circuit could not be counted.
|
|
1038
|
+
* Reading the node's own addresses costs the same and has none of that.
|
|
1039
|
+
*/
|
|
1040
|
+
const open = sr.node.getConnections().filter((c) => c.status === "open").map((c) => c.peerId);
|
|
1041
|
+
const stillHeld = heldRelayIdsOf(sr.node).filter((id) => open.includes(id));
|
|
1042
|
+
const lost = sr.relayPeerIds.filter((id) => !stillHeld.includes(id));
|
|
1043
|
+
const gained = stillHeld.filter((id) => !sr.relayPeerIds.includes(id));
|
|
1044
|
+
sr.relayPeerIds = stillHeld;
|
|
1045
|
+
if (gained.length > 0) {
|
|
1046
|
+
// A circuit this receiver did not have at install. Said out loud because it is the visible
|
|
1047
|
+
// half of the slow-start case, and because it is the moment that relay earns its inbound
|
|
1048
|
+
// carve-out — a silent widening of the gate is not something to do without a line.
|
|
1049
|
+
this.#ctx.logger.info("session.standing_receiver.reservation.gained", {
|
|
1050
|
+
agentName,
|
|
1051
|
+
relayPeerIds: gained,
|
|
1052
|
+
reservationsHeld: stillHeld.length,
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
if (lost.length === 0) {
|
|
1056
|
+
// Nothing lost. The gater still gets the current set, because `gained` may have widened it.
|
|
1057
|
+
if (gained.length > 0)
|
|
1058
|
+
sr.gater.setReservedRelayPeers(stillHeld);
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1061
|
+
// REVOKE FIRST. A relay whose reservation is gone must lose its inbound carve-out in the same
|
|
1062
|
+
// breath as the loss is noticed, or the gater's bound quietly becomes "granted one once".
|
|
1063
|
+
sr.gater.setReservedRelayPeers(stillHeld);
|
|
1064
|
+
for (const relayPeerId of lost) {
|
|
1065
|
+
// DOD-RELAY-KEEPALIVE-1 (review F4): carry the CAUSE, not just the exit point.
|
|
1066
|
+
// `relay_connection_gone` says where this was noticed — a poll of getConnections() — by
|
|
1067
|
+
// which time the abort reason that actually killed the link is long discarded. The relay
|
|
1068
|
+
// client for this (agent, relay) pair kept the error that ended its reader; that is the
|
|
1069
|
+
// nearest thing to an upstream cause available here, and its absence is how 2,061 of these
|
|
1070
|
+
// went untraced.
|
|
1071
|
+
const upstreamReason = this.#ctx.relayClients.get(`${agentName}::${relayPeerId}`)?.getLastReaderError();
|
|
1072
|
+
this.#ctx.logger.warn("session.standing_receiver.reservation.lost", {
|
|
1073
|
+
agentName,
|
|
1074
|
+
relayPeerId,
|
|
1075
|
+
reason: open.includes(relayPeerId) ? "circuit_address_vanished" : "relay_connection_gone",
|
|
1076
|
+
...(upstreamReason ? { upstreamReason } : {}),
|
|
1077
|
+
reservationsHeld: stillHeld.length,
|
|
1078
|
+
// The line an operator reads, and the two cases are not the same event at all.
|
|
1079
|
+
impact: stillHeld.length > 0
|
|
1080
|
+
? "this agent still holds " + stillHeld.length + " other circuit reservation(s), so it "
|
|
1081
|
+
+ "stays dialable from behind NAT and the receiver is NOT rebuilt. Losing one relay "
|
|
1082
|
+
+ "costs this agent nothing it can feel."
|
|
1083
|
+
: "this agent now holds NO circuit reservation, so nobody behind a home router can "
|
|
1084
|
+
+ "reach it. The receiver is being rebuilt against the rest of the pool.",
|
|
1085
|
+
});
|
|
1086
|
+
}
|
|
1087
|
+
if (stillHeld.length === 0) {
|
|
1088
|
+
// ZERO HELD IS STILL THE LOUD, STRUCTURAL CASE — the agent is unreachable behind NAT and
|
|
1089
|
+
// only a new node can take a new reservation, because a circuit listener is fixed at node
|
|
1090
|
+
// creation.
|
|
1091
|
+
void this.#ctx.receivers.rebuildStandingReceiver(agentName);
|
|
1092
|
+
continue;
|
|
1093
|
+
}
|
|
1094
|
+
/**
|
|
1095
|
+
* STILL REACHABLE, SO THE RECEIVER STANDS, AND NOTHING ELSE HAPPENS HERE. That second half is
|
|
1096
|
+
* the part worth reading, because the obvious next line is wrong twice over.
|
|
1097
|
+
*
|
|
1098
|
+
* **A LOST CONFIGURED CIRCUIT CANNOT BE RETAKEN BY THIS NODE.** Read out of
|
|
1099
|
+
* `@libp2p/circuit-relay-v2@4.2.5`, not assumed: for an explicit relay address
|
|
1100
|
+
* `transport/listener.js#listen()` is a ONE-SHOT — it reserves once and nothing calls it
|
|
1101
|
+
* again; `reservation-store.js#removeReservation()` clears the refresh timeout and deletes
|
|
1102
|
+
* the entry; and the listener's `_onAddRelayPeer` returns early for `type === 'configured'`,
|
|
1103
|
+
* so even a later reservation would not be announced. A circuit listener is fixed at node
|
|
1104
|
+
* creation, and the only thing that takes a new one is a NEW NODE — which is exactly the
|
|
1105
|
+
* rebuild this branch exists to refuse.
|
|
1106
|
+
*
|
|
1107
|
+
* **AND RE-PROVING TO THE LOST RELAY WOULD REBUILD THE RECEIVER ANYWAY.** Review F3: an
|
|
1108
|
+
* earlier version called `authenticateStandingReceiver` here to "remove the relay-side
|
|
1109
|
+
* reason for the revocation". That function ends with `if (refusal?.tryAnotherRelay) { …
|
|
1110
|
+
* void this.#ctx.receivers.rebuildStandingReceiver(agentName); }` — and a dead or misconfigured relay is
|
|
1111
|
+
* precisely the one that answers that way. So the common case was: lose relay A while
|
|
1112
|
+
* holding B, decline to rebuild, prove to A, A refuses, rebuild the whole receiver and throw
|
|
1113
|
+
* B's healthy reservation away. The churn engine, re-entered through the back door.
|
|
1114
|
+
*
|
|
1115
|
+
* **THE BOUND, STATED PLAINLY BECAUSE IT IS A REAL SHORTFALL AGAINST THE DoD:** a lost
|
|
1116
|
+
* circuit is gone until the receiver is next rebuilt for another reason. What the agent buys
|
|
1117
|
+
* is that it never STOPS BEING REACHABLE while that is true — the surviving relays carry it,
|
|
1118
|
+
* the loss is named in the log with its cause, and the lost relay's inbound carve-out is
|
|
1119
|
+
* revoked above. That is availability, not restoration in place.
|
|
1120
|
+
*
|
|
1121
|
+
* WHICH LEAVES A RATCHET, and `#respreadIfDecayed` below is what stops it: relays are only
|
|
1122
|
+
* ever lost between rebuilds, never regained, so an agent nobody talks to walks itself back
|
|
1123
|
+
* down to one relay — the exact state this unit exists to get it out of.
|
|
1124
|
+
*/
|
|
1125
|
+
}
|
|
1126
|
+
for (const agentName of this.#ctx.standingReceivers.keys()) {
|
|
1127
|
+
if (this.#ctx.agentsWantingReceiver.has(agentName))
|
|
1128
|
+
this.#respreadIfDecayed(agentName);
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
/**
|
|
1132
|
+
* 032-RELAYSPREAD — **AN IDLE AGENT MUST NOT RATCHET ITSELF BACK DOWN TO ONE RELAY.**
|
|
1133
|
+
*
|
|
1134
|
+
* Spreading happens when a receiver is BUILT, and between builds the count only falls: a lost
|
|
1135
|
+
* circuit cannot be retaken by a running node (a circuit listener is fixed at node creation), and
|
|
1136
|
+
* a relay the directory announces later is skipped while any circuit is held. An agent in
|
|
1137
|
+
* conversation re-spreads constantly — the receiver is handed into each session and a fresh one
|
|
1138
|
+
* is built behind it — so this is about the agent nobody has talked to for a day. It loses relays
|
|
1139
|
+
* one at a time, nothing pulls it back up, and it ends up exactly where this unit found it:
|
|
1140
|
+
* reachable through one relay, one relay away from being reachable through none.
|
|
1141
|
+
*
|
|
1142
|
+
* **THE COST OF FIXING IT IS A NEW PEER ID**, which is why it is fenced three ways rather than
|
|
1143
|
+
* simply rebuilding on sight:
|
|
1144
|
+
* - **ONLY WHEN IDLE.** A rebuild replaces the receiver's transport identity, and a counterparty
|
|
1145
|
+
* may be holding the old one from a `session_offer_accept`. With a live session for this agent
|
|
1146
|
+
* we leave it alone — a degraded spread costs redundancy, a changed peer id mid-conversation
|
|
1147
|
+
* costs the conversation.
|
|
1148
|
+
* - **ONLY WHEN THERE IS SOMETHING TO GAIN.** Holding every relay that was offered is not decay.
|
|
1149
|
+
* - **ON ITS OWN SLOW CLOCK**, never the watchdog's 30-second grid. A reservation is scarce —
|
|
1150
|
+
* the relay holds it for its full TTL even after we disconnect — so this reuses the
|
|
1151
|
+
* reservation retry interval rather than inventing a faster one.
|
|
1152
|
+
*/
|
|
1153
|
+
#respreadIfDecayed(agentName) {
|
|
1154
|
+
if (this.#ctx.shuttingDown)
|
|
1155
|
+
return;
|
|
1156
|
+
const sr = this.#ctx.standingReceivers.get(agentName);
|
|
1157
|
+
if (!sr || sr.relayPeerIds.length === 0)
|
|
1158
|
+
return; // zero held is the loud path
|
|
1159
|
+
for (const entry of this.#ctx.activeNodes.values()) {
|
|
1160
|
+
if (entry.agentName === agentName)
|
|
1161
|
+
return; // in conversation — hands off
|
|
1162
|
+
}
|
|
1163
|
+
const offered = this.reservationCircuitAddrs(agentName).addrs.length;
|
|
1164
|
+
if (sr.relayPeerIds.length >= offered)
|
|
1165
|
+
return; // nothing to gain
|
|
1166
|
+
const now = Date.now();
|
|
1167
|
+
const last = this.#ctx.srLastRespreadAt.get(agentName) ?? 0;
|
|
1168
|
+
if (now - last < this.#ctx.srReservationRetryMs)
|
|
1169
|
+
return;
|
|
1170
|
+
this.#ctx.srLastRespreadAt.set(agentName, now);
|
|
1171
|
+
this.#ctx.logger.info("session.standing_receiver.respread", {
|
|
1172
|
+
agentName,
|
|
1173
|
+
reservationsHeld: sr.relayPeerIds.length,
|
|
1174
|
+
relaysOffered: offered,
|
|
1175
|
+
impact: "this agent is idle and holds fewer relay reservations than it was offered, so its " +
|
|
1176
|
+
"receiver is being rebuilt to take the rest. Without this it can only lose relays between " +
|
|
1177
|
+
"rebuilds, and an agent nobody talks to drifts back down to a single relay — one relay " +
|
|
1178
|
+
"away from being unreachable behind NAT, which is the state this whole mechanism exists " +
|
|
1179
|
+
"to keep it out of.",
|
|
1180
|
+
});
|
|
1181
|
+
void this.#ctx.receivers.rebuildStandingReceiver(agentName);
|
|
1182
|
+
}
|
|
1183
|
+
/** Start the reservation watchdog (idempotent). Stopped by gracefulShutdown. */
|
|
1184
|
+
startReservationWatchdog() {
|
|
1185
|
+
if (this.#ctx.reservationWatchdog !== null)
|
|
1186
|
+
return;
|
|
1187
|
+
// Arm the backstop clock from the START of watching, not from the epoch — otherwise the first
|
|
1188
|
+
// tick always fires a sweep on top of the install drain that just ran.
|
|
1189
|
+
this.#ctx.park.armBackstopClock(Date.now());
|
|
1190
|
+
this.#ctx.reservationWatchdog = setInterval(() => {
|
|
1191
|
+
try {
|
|
1192
|
+
this.#reservationWatchdogTick();
|
|
1193
|
+
this.#ctx.park.parkedDrainBackstopTick(Date.now());
|
|
1194
|
+
}
|
|
1195
|
+
catch (err) {
|
|
1196
|
+
this.#ctx.logger.warn("session.standing_receiver.watchdog.failed", { error: extractErrorMessage(err) });
|
|
1197
|
+
}
|
|
1198
|
+
}, this.#ctx.srWatchdogIntervalMs);
|
|
1199
|
+
// Never hold the process open on account of the watchdog.
|
|
1200
|
+
this.#ctx.reservationWatchdog.unref?.();
|
|
1201
|
+
}
|
|
1202
|
+
/**
|
|
1203
|
+
* Start the standing receiver's libp2p node, holding a circuit-relay reservation
|
|
1204
|
+
* if one can be had — and WITHOUT one if it cannot.
|
|
1205
|
+
*
|
|
1206
|
+
* THE INVARIANT, learned live: **standing-receiver creation must NEVER be gated on
|
|
1207
|
+
* a relay.** libp2p's circuit listener awaits a live connection to its relay before
|
|
1208
|
+
* start() resolves, and it does not time out. A relay that does not answer parks
|
|
1209
|
+
* start() forever: no created event, no failure, no retry, no alarm — the agent
|
|
1210
|
+
* simply has no receiver and is deaf to ALL inbound, including the direct path that
|
|
1211
|
+
* worked before reservations existed. Strictly worse than the NAT defect this whole
|
|
1212
|
+
* line exists to fix. So every attempt is raced against a deadline, and failure
|
|
1213
|
+
* ALWAYS falls through to a plain TCP receiver.
|
|
1214
|
+
*
|
|
1215
|
+
* ONE relay, tried on the REAL node — not a probe.
|
|
1216
|
+
*
|
|
1217
|
+
* A relay reservation is a scarce resource: the relay holds it for its full TTL
|
|
1218
|
+
* even after the client disconnects, and it has a finite number of slots. An
|
|
1219
|
+
* earlier design probed each relay on a throwaway node and then reserved AGAIN on
|
|
1220
|
+
* the receiver — burning TWO slots per agent to get one, and leaving the throwaway's
|
|
1221
|
+
* slot pinned for hours. That is how a fleet exhausts a relay. Here the receiver
|
|
1222
|
+
* itself makes the attempt: if the relay grants the reservation, we KEEP that node.
|
|
1223
|
+
* One slot per agent, which is the true cost.
|
|
1224
|
+
*
|
|
1225
|
+
* Candidates are tried in order (directory pool first). The first that actually
|
|
1226
|
+
* grants a reservation wins; the rest are never touched.
|
|
1227
|
+
*/
|
|
1228
|
+
/**
|
|
1229
|
+
* DOD-M15-RELAYSLOTS-1 — tell the relay this transport identity belongs to a registered agent, so
|
|
1230
|
+
* the reservation it just refused is granted on the next attempt.
|
|
1231
|
+
*
|
|
1232
|
+
* Returns `"proven"` on success, or the shape of the failure so the candidate loop can act on it.
|
|
1233
|
+
* It never throws: throwing would turn one unreachable relay into a failure to build a receiver
|
|
1234
|
+
* at all.
|
|
1235
|
+
*
|
|
1236
|
+
* ─── Why this returns a verdict instead of a boolean ──────────────────────────────────────────
|
|
1237
|
+
*
|
|
1238
|
+
* Review HIGH-1/HIGH-2. Putting the reservation behind a proof MOVED THE FIRST REFUSAL onto this
|
|
1239
|
+
* path. `authenticateStandingReceiver` does everything right with a refusal — records it where
|
|
1240
|
+
* `cello_status` can read it, and quarantines a relay whose fault is its own — but it runs only
|
|
1241
|
+
* on a receiver that ALREADY HAS a reservation, so under the new gate a total failure never
|
|
1242
|
+
* reaches it. This method was logging `proven: false` and returning.
|
|
1243
|
+
*
|
|
1244
|
+
* What that cost the operator: an expired token, or an agent at its slot cap, refused by every
|
|
1245
|
+
* relay in the pool. `cello_status` shows an agent that is online and reachable by nobody, with
|
|
1246
|
+
* no cause anywhere the person will look — while the relay had computed the cause, the count, and
|
|
1247
|
+
* the next step, and put them on the wire. And `slot_cap_exceeded` is classified
|
|
1248
|
+
* `tryAnotherRelay: false` precisely so the client STOPS walking the fleet; without the verdict,
|
|
1249
|
+
* the loop walked it anyway, turning one client-side fault into what reads as a fleet outage.
|
|
1250
|
+
*/
|
|
1251
|
+
async proveToRelay(agentName, circuitAddr, node, correlationId,
|
|
1252
|
+
/**
|
|
1253
|
+
* Whether this proof is the STANDING RECEIVER's, and may therefore write the surface
|
|
1254
|
+
* `cello_status` reads as "your standing receiver was refused".
|
|
1255
|
+
*
|
|
1256
|
+
* A revival proves itself too, and its refusal is real — but it is not evidence about the
|
|
1257
|
+
* receiver. A receiver that proved thirty seconds ago and holds a slot, plus a revival refused
|
|
1258
|
+
* with `slot_cap_exceeded`, would otherwise have `cello_status` report a front door as refused
|
|
1259
|
+
* while it is open. The refusal is still logged and still steers the candidate loop; what it
|
|
1260
|
+
* does not do is claim to be about something it did not measure.
|
|
1261
|
+
*/
|
|
1262
|
+
surfaceAsReceiverRefusal) {
|
|
1263
|
+
const relayPeerId = relayPeerIdOf(circuitAddr);
|
|
1264
|
+
const baseRelayAddr = circuitAddr.split("/p2p-circuit")[0];
|
|
1265
|
+
if (!relayPeerId || !baseRelayAddr) {
|
|
1266
|
+
this.#ctx.logger.warn("session.standing_receiver.prove.address_unreadable", {
|
|
1267
|
+
agentName,
|
|
1268
|
+
circuitAddr,
|
|
1269
|
+
correlationId,
|
|
1270
|
+
impact: "this candidate's circuit address does not name a relay peer, so there is nothing " +
|
|
1271
|
+
"to prove to and its reservation will stay refused. Skipped silently before — which made " +
|
|
1272
|
+
"a malformed address look identical to a relay that simply said no.",
|
|
1273
|
+
});
|
|
1274
|
+
return "unavailable";
|
|
1275
|
+
}
|
|
1276
|
+
/**
|
|
1277
|
+
* Declared out here so the `finally` can close it. Scoped inside the `try` before, so a throw
|
|
1278
|
+
* from `proveReservation` skipped the close and left the stream and its pending settles behind.
|
|
1279
|
+
*/
|
|
1280
|
+
let client;
|
|
1281
|
+
try {
|
|
1282
|
+
client = this.#ctx.detachedRelayClientBuilder?.(agentName, relayPeerId, [baseRelayAddr], {
|
|
1283
|
+
receiptStore: this.#ctx.relayReceiptStore ?? undefined,
|
|
1284
|
+
sealLeafStore: this.#ctx.sealLeafStore ?? undefined,
|
|
1285
|
+
onlineToken: () => this.#ctx.getDirectoryOnlineToken(agentName),
|
|
1286
|
+
});
|
|
1287
|
+
if (!client) {
|
|
1288
|
+
this.#ctx.logger.warn("session.standing_receiver.prove.no_builder", {
|
|
1289
|
+
agentName,
|
|
1290
|
+
relayPeerId,
|
|
1291
|
+
correlationId,
|
|
1292
|
+
impact: "no relay client could be built, so this receiver cannot prove itself and the " +
|
|
1293
|
+
"relay will refuse its reservation again. The agent is reachable only over a direct " +
|
|
1294
|
+
"connection until this is wired.",
|
|
1295
|
+
});
|
|
1296
|
+
return "unavailable";
|
|
1297
|
+
}
|
|
1298
|
+
const proven = await client.proveReservation(node);
|
|
1299
|
+
if (proven) {
|
|
1300
|
+
if (surfaceAsReceiverRefusal)
|
|
1301
|
+
this.#ctx.srRelayRefusal.delete(agentName);
|
|
1302
|
+
this.#ctx.logger.info("session.standing_receiver.prove.result", {
|
|
1303
|
+
agentName, relayPeerId, peerId: node.getPeerId(), proven: true, correlationId,
|
|
1304
|
+
});
|
|
1305
|
+
return "proven";
|
|
1306
|
+
}
|
|
1307
|
+
/**
|
|
1308
|
+
* The same two lines `authenticateStandingReceiver` runs, for the same reason. The `else`
|
|
1309
|
+
* matters as much as the `if`: `proveReservation` also fails for transport reasons, which
|
|
1310
|
+
* leave `getLastAuthRefusal()` null, and leaving a PREVIOUS refusal in the map would have
|
|
1311
|
+
* `cello_status` explaining a cause that is no longer what is wrong.
|
|
1312
|
+
*/
|
|
1313
|
+
const refusal = client.getLastAuthRefusal();
|
|
1314
|
+
if (surfaceAsReceiverRefusal) {
|
|
1315
|
+
if (refusal) {
|
|
1316
|
+
this.#ctx.srRelayRefusal.set(agentName, { ...this.#ctx.withDirectoryCause(agentName, refusal), relayPeerId });
|
|
1317
|
+
}
|
|
1318
|
+
else {
|
|
1319
|
+
this.#ctx.srRelayRefusal.delete(agentName);
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
this.#ctx.logger.warn("session.standing_receiver.prove.result", {
|
|
1323
|
+
agentName,
|
|
1324
|
+
relayPeerId,
|
|
1325
|
+
peerId: node.getPeerId(),
|
|
1326
|
+
proven: false,
|
|
1327
|
+
refusalReason: refusal?.reason ?? "no_relay_verdict",
|
|
1328
|
+
tryAnotherRelay: refusal?.tryAnotherRelay ?? true,
|
|
1329
|
+
correlationId,
|
|
1330
|
+
impact: refusal?.advice ??
|
|
1331
|
+
"the relay would not accept this agent's proof and said nothing about why, which is what " +
|
|
1332
|
+
"a transport failure mid-handshake looks like. The candidate loop moves on to the next relay.",
|
|
1333
|
+
});
|
|
1334
|
+
if (refusal && !refusal.tryAnotherRelay)
|
|
1335
|
+
return "refused_this_agent";
|
|
1336
|
+
if (refusal?.tryAnotherRelay && !this.#ctx.shuttingDown) {
|
|
1337
|
+
this.#quarantineRelay(agentName, relayPeerId, refusal.reason);
|
|
1338
|
+
}
|
|
1339
|
+
return "refused_try_another_relay";
|
|
1340
|
+
}
|
|
1341
|
+
catch (err) {
|
|
1342
|
+
this.#ctx.logger.warn("session.standing_receiver.prove.failed", {
|
|
1343
|
+
agentName,
|
|
1344
|
+
correlationId,
|
|
1345
|
+
error: extractErrorMessage(err),
|
|
1346
|
+
impact: "this receiver could not prove itself, so its retry will be refused and the " +
|
|
1347
|
+
"candidate loop will try the next relay.",
|
|
1348
|
+
});
|
|
1349
|
+
return "unavailable";
|
|
1350
|
+
}
|
|
1351
|
+
finally {
|
|
1352
|
+
client?.close();
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
/**
|
|
1356
|
+
* DOD-M12B-REVIVE-RELAY-1 — reconnect the session's relay WITNESS, which revival never did.
|
|
1357
|
+
*
|
|
1358
|
+
* THE FIRST-PRINCIPLES DEFECT, and the one that explains every symptom chased separately before
|
|
1359
|
+
* it. Establishment does five things: build the node, register the content handler, wire liveness,
|
|
1360
|
+
* **connect the relay**, and dial the counterparty. Revival did the first three. A revived session
|
|
1361
|
+
* was therefore not a session — it looked live, reported `active`, and had no live inbound path at
|
|
1362
|
+
* all.
|
|
1363
|
+
*
|
|
1364
|
+
* MEASURED 2026-08-18 with two real agents: a message on a reconnected session took **three
|
|
1365
|
+
* minutes**, against seconds on a fresh one, because only the five-minute mailbox backstop ever
|
|
1366
|
+
* found it. Doorbells stopped firing for the same reason — the relay stream is what rings them.
|
|
1367
|
+
* And `#parkContent` refuses without `entry.relayClient`, so sends could not park either.
|
|
1368
|
+
*
|
|
1369
|
+
* NO ASSIGNMENT IS PRESENTED, and that is by design rather than omission: `RelayConnectParams`
|
|
1370
|
+
* documents the reconnect mode itself — *"absent … on the restart/persisted reconnect path (the
|
|
1371
|
+
* relay already recorded the session at first establishment) — the client then just reconnects
|
|
1372
|
+
* without re-recording."* A revival is exactly that path.
|
|
1373
|
+
*
|
|
1374
|
+
* Best-effort and non-fatal: a session that comes back without its witness is still better than
|
|
1375
|
+
* one that does not come back, and the failure is named rather than silent.
|
|
1376
|
+
*/
|
|
1377
|
+
async reconnectRevivedSessionRelay(agentName, sessionId, node, gater, correlationId, ep) {
|
|
1378
|
+
if (!ep) {
|
|
1379
|
+
this.#ctx.logger.warn("session.revive.relay.absent", {
|
|
1380
|
+
agentName,
|
|
1381
|
+
sessionId,
|
|
1382
|
+
impact: "no relay is recorded for this session, so it comes back with no live inbound path — "
|
|
1383
|
+
+ "messages arrive only on the periodic mailbox poll, and a failed send cannot park and is "
|
|
1384
|
+
+ "reported lost",
|
|
1385
|
+
});
|
|
1386
|
+
return;
|
|
1387
|
+
}
|
|
1388
|
+
try {
|
|
1389
|
+
// The gater admits only the counterparty inbound; the relay is a third peer and must be
|
|
1390
|
+
// permitted OUTBOUND or our own gate refuses the dial (INV-5 keeps inbound counterparty-only).
|
|
1391
|
+
gater.setAllowedOutboundPeer(ep.relayPeerId);
|
|
1392
|
+
const clientKey = `${agentName}::${ep.relayPeerId}`;
|
|
1393
|
+
let client = this.#ctx.relayClients.get(clientKey);
|
|
1394
|
+
if (!client) {
|
|
1395
|
+
if (!this.#ctx.relayReceiptStore && this.#db)
|
|
1396
|
+
this.#ctx.relayReceiptStore = new RelayReceiptStore(this.#db, this.#ctx.logger);
|
|
1397
|
+
if (!this.#ctx.sealLeafStore && this.#db)
|
|
1398
|
+
this.#ctx.sealLeafStore = new SessionSealLeafStore(this.#db, this.#ctx.logger);
|
|
1399
|
+
client = this.#ctx.detachedRelayClientBuilder?.(agentName, ep.relayPeerId, [...ep.relayAddrs], {
|
|
1400
|
+
receiptStore: this.#ctx.relayReceiptStore ?? undefined,
|
|
1401
|
+
sealLeafStore: this.#ctx.sealLeafStore ?? undefined,
|
|
1402
|
+
// DOD-M15-RELAYSLOTS-1: read at each auth, never snapshotted — the token expires hourly.
|
|
1403
|
+
onlineToken: () => this.#ctx.getDirectoryOnlineToken(agentName),
|
|
1404
|
+
});
|
|
1405
|
+
if (!client) {
|
|
1406
|
+
this.#ctx.logger.warn("session.revive.relay.builder_absent", {
|
|
1407
|
+
agentName,
|
|
1408
|
+
sessionId,
|
|
1409
|
+
impact: "no relay client could be built, so this revived session has no live inbound path",
|
|
1410
|
+
});
|
|
1411
|
+
return;
|
|
1412
|
+
}
|
|
1413
|
+
this.#ctx.relayClients.set(clientKey, client);
|
|
1414
|
+
}
|
|
1415
|
+
// 033-ACKEMIT: a revived session re-registers with no assignment in hand, so the genesis comes
|
|
1416
|
+
// from the entry that was just restored above.
|
|
1417
|
+
client.registerSession(sessionId, node, this.#ctx.contentIn.relayLeafHandler(agentName, sessionId, correlationId), undefined, this.#ctx.leafRecords.sessionGenesisPrevRoot(agentName, sessionId));
|
|
1418
|
+
const entry = this.#ctx.activeNodes.get(this.#ctx.sessionKey(agentName, sessionId));
|
|
1419
|
+
if (entry) {
|
|
1420
|
+
entry.relayClient = client;
|
|
1421
|
+
entry.relaySessionIdBytes = Uint8Array.from(Buffer.from(sessionId, "hex"));
|
|
1422
|
+
entry.relayClientKey = clientKey;
|
|
1423
|
+
}
|
|
1424
|
+
/**
|
|
1425
|
+
* THE STEP THIS METHOD IS NAMED AFTER, and the first version did not take it (review HIGH-2).
|
|
1426
|
+
*
|
|
1427
|
+
* `registerSession` files a handler in a Map. It opens nothing — no dial, no auth, no reader
|
|
1428
|
+
* loop. `connectSessionRelay` ends with exactly this line and the reconnect ended without it,
|
|
1429
|
+
* so a revived session registered a handler on a client whose stream was `null` and then
|
|
1430
|
+
* logged that its live inbound path was back. It was not: the counterparty's leaves queued at
|
|
1431
|
+
* the relay, no doorbell fired, and delivery fell back to the five-minute mailbox poll — the
|
|
1432
|
+
* three-minutes-versus-seconds symptom the whole unit exists to remove.
|
|
1433
|
+
*
|
|
1434
|
+
* Worse, the client is usually BRAND NEW here: `markInterruptedWithDetails` closes and drops
|
|
1435
|
+
* the client for the last session on a relay, so a single-session agent always lands in the
|
|
1436
|
+
* build branch above with a fresh, unconnected client.
|
|
1437
|
+
*
|
|
1438
|
+
* `#ensureConnected` is idempotent, so this also repairs the cached-but-dead-stream case.
|
|
1439
|
+
*/
|
|
1440
|
+
await client.connect(node);
|
|
1441
|
+
this.#ctx.logger.info("session.revive.relay.connected", {
|
|
1442
|
+
agentName,
|
|
1443
|
+
sessionId,
|
|
1444
|
+
relayPeerId: ep.relayPeerId,
|
|
1445
|
+
impact: "the revived session has its live inbound path back — messages arrive promptly "
|
|
1446
|
+
+ "instead of waiting for the periodic mailbox poll",
|
|
1447
|
+
});
|
|
1448
|
+
}
|
|
1449
|
+
catch (err) {
|
|
1450
|
+
this.#ctx.logger.warn("session.revive.relay.failed", {
|
|
1451
|
+
agentName,
|
|
1452
|
+
sessionId,
|
|
1453
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1454
|
+
impact: "the session is back but without its witness — delivery falls back to the periodic poll",
|
|
1455
|
+
});
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
/** DOD-M12B-REVIVE-PARK-1 test seam: the relay the live entry will park to. Not otherwise
|
|
1459
|
+
* observable — `#activeNodes` is private and the park's own refusal is silent about which of its
|
|
1460
|
+
* four preconditions was missing. */
|
|
1461
|
+
getSessionRelayForTest(agentName, sessionId) {
|
|
1462
|
+
const entry = this.#ctx.activeNodes.get(this.#ctx.sessionKey(agentName, sessionId));
|
|
1463
|
+
if (!entry)
|
|
1464
|
+
return null;
|
|
1465
|
+
return {
|
|
1466
|
+
...(entry.relayPeerId !== undefined ? { relayPeerId: entry.relayPeerId } : {}),
|
|
1467
|
+
...(entry.relayAddrs !== undefined ? { relayAddrs: entry.relayAddrs } : {}),
|
|
1468
|
+
};
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
//# sourceMappingURL=session-relay.js.map
|