@cello-protocol/daemon 0.0.207 → 0.0.208
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-handlers.d.ts +12 -0
- package/dist/agent-handlers.d.ts.map +1 -1
- package/dist/agent-handlers.js +4 -1
- package/dist/agent-handlers.js.map +1 -1
- package/dist/daemon.js +5 -4
- package/dist/daemon.js.map +1 -1
- package/dist/outbound-sessions.d.ts +27 -2
- package/dist/outbound-sessions.d.ts.map +1 -1
- package/dist/outbound-sessions.js +25 -3
- package/dist/outbound-sessions.js.map +1 -1
- package/dist/session-node-types.d.ts +39 -0
- package/dist/session-node-types.d.ts.map +1 -1
- package/dist/session-node-types.js +49 -0
- package/dist/session-node-types.js.map +1 -1
- package/dist/session-relay.d.ts.map +1 -1
- package/dist/session-relay.js +12 -12
- package/dist/session-relay.js.map +1 -1
- package/dist/standing-receivers.d.ts.map +1 -1
- package/dist/standing-receivers.js +378 -190
- package/dist/standing-receivers.js.map +1 -1
- package/dist/trust-signal-sweep-tick.d.ts +45 -0
- package/dist/trust-signal-sweep-tick.d.ts.map +1 -0
- package/dist/trust-signal-sweep-tick.js +84 -0
- package/dist/trust-signal-sweep-tick.js.map +1 -0
- package/dist/trust-signal-sweep.d.ts.map +1 -1
- package/dist/trust-signal-sweep.js +32 -10
- package/dist/trust-signal-sweep.js.map +1 -1
- package/package.json +5 -5
|
@@ -3,7 +3,7 @@ import { SessionConnectionGater } from "./session-connection-gater.js";
|
|
|
3
3
|
import { relayOnlyState, publishableEndpoint } from "./relay-only.js";
|
|
4
4
|
import { extractErrorMessage } from "./error-message.js";
|
|
5
5
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
6
|
-
import { relayPeerIdOf, heldRelayIdsOf, CIRCUIT_RELAY_ID, SR_RESERVATION_MAX_RETRIES, REVIVE_RESERVATION_CANDIDATES, REVIVE_RESERVATION_TIMEOUT_MS, } from "./session-node-types.js";
|
|
6
|
+
import { relayPeerIdOf, heldRelayIdsOf, CIRCUIT_RELAY_ID, SR_RESERVATION_MAX_RETRIES, REVIVE_RESERVATION_CANDIDATES, REVIVE_RESERVATION_TIMEOUT_MS, clientSideAskFault, holdsCircuit, stopWhenSettled, } from "./session-node-types.js";
|
|
7
7
|
import { STANDING_RECEIVER_AGENT_NAME } from "./types.js";
|
|
8
8
|
export class StandingReceivers {
|
|
9
9
|
#ctx;
|
|
@@ -61,152 +61,252 @@ export class StandingReceivers {
|
|
|
61
61
|
for (const circuitAddr of candidateCircuitAddrs) {
|
|
62
62
|
const candidateSeed = receiverSeed;
|
|
63
63
|
/**
|
|
64
|
-
* DOD-M15-
|
|
64
|
+
* DOD-M15-RELAYPROVE-ORDER-1 — **ONE NODE, ONE ASK: prove, then reserve.**
|
|
65
65
|
*
|
|
66
|
-
* The relay
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
* relay
|
|
70
|
-
*
|
|
66
|
+
* The relay refuses a reservation from a peer that has not shown it belongs to a registered
|
|
67
|
+
* agent. This candidate therefore comes up with **no circuit address in its listen set** —
|
|
68
|
+
* TCP/WS only — so libp2p asks nobody for anything at start. We dial the relay, prove over
|
|
69
|
+
* `/cello/relay/1.0.0`, and only then ask libp2p's own transport manager to listen on the
|
|
70
|
+
* circuit. The relay grants on that first ask because it sets `slot.provenForReservation` per
|
|
71
|
+
* CONNECTION at auth time, and libp2p's reservation store reuses the connection we proved on.
|
|
71
72
|
*
|
|
72
|
-
* ⚠️
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
73
|
+
* ⚠️ **THIS REPLACES "ask, be refused, prove, ask again", AND THE CLAIM THAT FORCED IT WAS
|
|
74
|
+
* FALSE.** The old comment here read: *"It has to be two connections, and that was measured
|
|
75
|
+
* rather than chosen. Taking the reservation by hand on the same connection as the proof DOES
|
|
76
|
+
* get a slot — and libp2p then announces no circuit address for it, because it only announces
|
|
77
|
+
* addresses for reservations its own relay-discovery made."* Its first clause described
|
|
78
|
+
* taking the slot BY HAND over a raw HOP stream. Asking libp2p's transport manager is a
|
|
79
|
+
* different act: the reservation is then libp2p's own, and it announces the address. Measured
|
|
80
|
+
* live 2026-09-08 against the Virginia relay (libp2p 3.3.11, circuit-relay-v2 4.2.13) —
|
|
81
|
+
* granted on the first ask in 708ms, circuit address announced, no new connection opened.
|
|
82
|
+
*
|
|
83
|
+
* What the old shape cost: libp2p's answer to the refused first ask is to restart its
|
|
84
|
+
* connection manager, closing every connection — including the one the proof was riding — so
|
|
85
|
+
* the proof had to be retried on a fresh node. Two node builds and four dials per relay,
|
|
86
|
+
* 5.4–9.3s to a first reservation, against 3.1s dial-to-reachable here.
|
|
76
87
|
*/
|
|
77
88
|
let candidateGranted = false;
|
|
78
89
|
// Set when the relay refused the AGENT rather than being unwilling itself: every other relay
|
|
79
90
|
// in the pool answers identically, so the walk ends here rather than reproducing it N times.
|
|
80
91
|
let candidateRefusedAgent = false;
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
92
|
+
/**
|
|
93
|
+
* Whether the proof branch already named this candidate's failure. Without it the generic
|
|
94
|
+
* rejection below fires a SECOND `relay.rejected` for the same candidate, with a reason
|
|
95
|
+
* derived from a connection state rather than from the verdict the relay actually gave — the
|
|
96
|
+
* specific cause overwritten by a generic one, one frame later, which is the exact shape
|
|
97
|
+
* Invariant 3 forbids.
|
|
98
|
+
*/
|
|
99
|
+
let rejectionNamed = false;
|
|
100
|
+
/**
|
|
101
|
+
* The ask's own promise, when one was made. The abandonment below has to wait on THIS as well
|
|
102
|
+
* as on `start()`: the reservation is taken here now, so a node torn down while its ask is
|
|
103
|
+
* still in flight can be granted late and come up on the receiver's advertised peer id with
|
|
104
|
+
* nothing holding a reference to kill it.
|
|
105
|
+
*/
|
|
106
|
+
let listenP;
|
|
107
|
+
/** Set when the ask failed for a fault of OURS, so it is not re-described as the relay's. */
|
|
108
|
+
let askFault;
|
|
109
|
+
const candidate = await this.createAgentNode(agentName, {
|
|
110
|
+
sessionId,
|
|
111
|
+
connectionGater: gater,
|
|
112
|
+
nodeType: "standing_receiver",
|
|
113
|
+
// NO `circuitRelayListenAddrs` — that is the whole change. A circuit address here is
|
|
114
|
+
// libp2p asking the relay before anything has proved, which is the refusal this removes.
|
|
115
|
+
transportPrivateKey: candidateSeed,
|
|
116
|
+
});
|
|
117
|
+
let timer;
|
|
118
|
+
const timedOut = Symbol("reservation_timeout");
|
|
119
|
+
let outcome = "failed";
|
|
120
|
+
let error = "";
|
|
121
|
+
// KEEP THE START PROMISE. Every candidate now carries the receiver's identity, so an
|
|
122
|
+
// abandoned one must be reliably torn down rather than best-effort — and only its own start
|
|
123
|
+
// promise says when it is stoppable (see the seed note above).
|
|
124
|
+
const startP = candidate.start();
|
|
125
|
+
try {
|
|
126
|
+
outcome = await Promise.race([
|
|
127
|
+
startP.then(() => "started"),
|
|
128
|
+
new Promise((resolve) => {
|
|
129
|
+
timer = setTimeout(() => resolve(timedOut), this.#ctx.srReservationTimeoutMs);
|
|
130
|
+
}),
|
|
131
|
+
]);
|
|
132
|
+
}
|
|
133
|
+
catch (err) {
|
|
134
|
+
error = extractErrorMessage(err);
|
|
135
|
+
}
|
|
136
|
+
finally {
|
|
137
|
+
if (timer !== undefined)
|
|
138
|
+
clearTimeout(timer);
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* The node is up on TCP/WS. Everything below is the relay handshake, in the order that works:
|
|
142
|
+
* prove, then ask. A failure at any step abandons this candidate and moves to the next relay
|
|
143
|
+
* — there is no second attempt, because there is no refusal left for one to recover from.
|
|
144
|
+
*/
|
|
145
|
+
if (outcome === "started") {
|
|
128
146
|
/**
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
147
|
+
* PROVE. `proveToRelay` dials the relay's base address from THIS node and authenticates
|
|
148
|
+
* over `/cello/relay/1.0.0`, which is what marks this connection proven at the relay.
|
|
149
|
+
*
|
|
150
|
+
* ⚠️ THE NODE STAYS UP. The old shape stopped it here, because the reservation was about to
|
|
151
|
+
* be asked for by a rebuilt node. Stopping it now would close the very connection the relay
|
|
152
|
+
* marked proven, and the ask below would be refused — the connection is the thing carrying
|
|
153
|
+
* the property, not the peer id.
|
|
133
154
|
*/
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
155
|
+
const verdict = await this.#ctx.proveToRelay(agentName, circuitAddr, candidate, correlationId, true);
|
|
156
|
+
/**
|
|
157
|
+
* ⚠️ **A PROOF THAT NEVER REACHED A VERDICT IS NOT A REFUSAL** — and the ask goes ahead.
|
|
158
|
+
*
|
|
159
|
+
* `unavailable` means no relay answer was obtained at all: no relay client is wired, the
|
|
160
|
+
* dial failed, the stream threw. `proveReservation` already draws exactly this line for its
|
|
161
|
+
* own retry (*"`true`/`false` are VERDICTS; `transport_failed` means no verdict was reached
|
|
162
|
+
* and the question is still open"*), and it is the line that matters here for a second
|
|
163
|
+
* reason: **not every relay gates reservations.** A relay that never asks for a proof
|
|
164
|
+
* grants on the first ask, and refusing to ask because our own proof machinery was
|
|
165
|
+
* unavailable would make this client unable to reserve with it at all — a capability lost
|
|
166
|
+
* to a precaution.
|
|
167
|
+
*
|
|
168
|
+
* The cost of asking anyway is one refused ask against a gated relay we could not prove to
|
|
169
|
+
* — which is what the whole walk used to do on every relay, every time.
|
|
170
|
+
*/
|
|
171
|
+
if (verdict === "refused_this_agent" || verdict === "refused_try_another_relay") {
|
|
142
172
|
/**
|
|
143
173
|
* DOD-M15-RELAYSLOTS-1 clause 9 — **A CLIENT-SIDE REFUSAL ENDS THE WALK.**
|
|
144
174
|
*
|
|
145
175
|
* `slot_cap_exceeded` and an expired or missing token are classified `tryAnotherRelay:
|
|
146
|
-
* false` because they reproduce on every relay in the pool: the cap is per AGENT, and
|
|
147
|
-
* token comes from the directory, not from here. Walking on costs a node build and
|
|
148
|
-
*
|
|
176
|
+
* false` because they reproduce on every relay in the pool: the cap is per AGENT, and
|
|
177
|
+
* the token comes from the directory, not from here. Walking on costs a node build and
|
|
178
|
+
* a dial per remaining relay to arrive at the same answer, and it makes one client-side
|
|
149
179
|
* fault look like a fleet-wide outage in the logs. The refusal is already recorded where
|
|
150
180
|
* `cello_status` reads it, so stopping is not silence.
|
|
181
|
+
*
|
|
182
|
+
* ⚠️ **THE BOUNDARY THIS BRANCH RESTS ON IS ENFORCED IN `proveToRelay`, NOT HERE.** It
|
|
183
|
+
* returns `unavailable` whenever no relay verdict was reached — including a proof that
|
|
184
|
+
* failed for a transport reason, which it used to label `refused_try_another_relay`
|
|
185
|
+
* (review HIGH-1). Stated as where the property lives rather than asserted as a fact
|
|
186
|
+
* about this branch: a comment claiming "only a verdict reaches here" is true only for
|
|
187
|
+
* as long as that producer keeps its side, and the producer is in another file.
|
|
151
188
|
*/
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
189
|
+
const proofReason = verdict === "refused_this_agent" ? "relay_refused_this_agent" : "relay_proof_refused";
|
|
190
|
+
this.#ctx.srLastRejectionReason.set(agentName, proofReason);
|
|
191
|
+
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
192
|
+
agentName,
|
|
193
|
+
circuitAddr,
|
|
194
|
+
reason: proofReason,
|
|
195
|
+
correlationId,
|
|
196
|
+
impact: proofReason === "relay_refused_this_agent"
|
|
197
|
+
? "the relay refused this AGENT rather than this relay being unwilling or " +
|
|
161
198
|
"unwell, so every other relay would refuse it identically. Stopped here; " +
|
|
162
|
-
"cello_status carries the cause and what to do about it."
|
|
163
|
-
|
|
199
|
+
"cello_status carries the cause and what to do about it."
|
|
200
|
+
: "this relay would not take the agent's proof. Moving to the next relay.",
|
|
201
|
+
});
|
|
202
|
+
if (verdict === "refused_this_agent")
|
|
164
203
|
candidateRefusedAgent = true;
|
|
165
|
-
|
|
204
|
+
rejectionNamed = true;
|
|
205
|
+
try {
|
|
206
|
+
await candidate.stop();
|
|
166
207
|
}
|
|
208
|
+
catch { /* it may never have finished starting */ }
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
167
211
|
/**
|
|
168
|
-
* ⚠️
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
212
|
+
* ⚠️ **A PROOF THAT NEVER REACHED A VERDICT IS NOT A REFUSAL** — and the ask goes ahead.
|
|
213
|
+
*
|
|
214
|
+
* `unavailable` means no relay answer was obtained at all: no relay client is wired, the
|
|
215
|
+
* dial failed, the stream threw. `proveReservation` already draws exactly this line for its
|
|
216
|
+
* own retry (*"`true`/`false` are VERDICTS; `transport_failed` means no verdict was reached
|
|
217
|
+
* and the question is still open"*), and it matters here for a second reason: **not every
|
|
218
|
+
* relay gates reservations.** One that never asks for a proof grants on the first ask, and
|
|
219
|
+
* refusing to ask because our own proof machinery was unavailable would make this client
|
|
220
|
+
* unable to reserve with it at all — a capability lost to a precaution.
|
|
221
|
+
*
|
|
222
|
+
* The cost of asking anyway is one refused ask against a gated relay we could not prove to,
|
|
223
|
+
* which is what the walk used to spend on every relay, every time.
|
|
172
224
|
*/
|
|
173
|
-
if (verdict
|
|
174
|
-
this.#ctx.
|
|
175
|
-
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
225
|
+
if (verdict === "unavailable") {
|
|
226
|
+
this.#ctx.logger.warn("session.standing_receiver.prove.no_verdict", {
|
|
176
227
|
agentName,
|
|
177
228
|
circuitAddr,
|
|
178
|
-
reason: "relay_proof_refused",
|
|
179
|
-
attempts: attempt + 1,
|
|
180
229
|
correlationId,
|
|
181
|
-
impact: "
|
|
182
|
-
"
|
|
230
|
+
impact: "no proof verdict was obtained from this relay — no relay client is wired, or " +
|
|
231
|
+
"it could not be reached. Asking for the reservation anyway: a relay that does not " +
|
|
232
|
+
"gate them grants it, and one that does refuses an ask that cost a single dial.",
|
|
183
233
|
});
|
|
184
|
-
break;
|
|
185
234
|
}
|
|
186
|
-
|
|
235
|
+
/**
|
|
236
|
+
* ASK — ONCE, on the connection we just proved on. Raced against the same budget the whole
|
|
237
|
+
* candidate used to get, because this is now the only part that talks to the relay.
|
|
238
|
+
*/
|
|
239
|
+
let listenTimer;
|
|
240
|
+
const listenTimedOut = Symbol("listen_timeout");
|
|
241
|
+
let listenOutcome = listenTimedOut;
|
|
242
|
+
// INSIDE the try. A node that cannot take the ask at all throws SYNCHRONOUSLY, and outside
|
|
243
|
+
// the try that escapes the whole walk instead of failing this one candidate.
|
|
244
|
+
try {
|
|
245
|
+
listenP = candidate.listenOnCircuit(circuitAddr);
|
|
246
|
+
listenOutcome = await Promise.race([
|
|
247
|
+
listenP.then(() => "asked"),
|
|
248
|
+
new Promise((resolve) => {
|
|
249
|
+
listenTimer = setTimeout(() => resolve(listenTimedOut), this.#ctx.srReservationTimeoutMs);
|
|
250
|
+
}),
|
|
251
|
+
]);
|
|
252
|
+
}
|
|
253
|
+
catch (err) {
|
|
254
|
+
error = extractErrorMessage(err);
|
|
255
|
+
// Review HIGH-2: a fault of OURS keeps its own name all the way to the operator, instead
|
|
256
|
+
// of being re-described from the connection state as something the relay did.
|
|
257
|
+
askFault = clientSideAskFault(err);
|
|
258
|
+
listenOutcome = listenTimedOut;
|
|
259
|
+
}
|
|
260
|
+
finally {
|
|
261
|
+
if (listenTimer !== undefined)
|
|
262
|
+
clearTimeout(listenTimer);
|
|
263
|
+
}
|
|
264
|
+
// The only proof that counts: the relay actually GRANTED the reservation. `listen()`
|
|
265
|
+
// resolving is not enough — a relay that is out of reservation slots completes the
|
|
266
|
+
// handshake and simply grants nothing, leaving a node that looks started and is reachable
|
|
267
|
+
// by nobody.
|
|
268
|
+
if (listenOutcome === "asked" && holdsCircuit(candidate)) {
|
|
269
|
+
candidateGranted = true;
|
|
270
|
+
// The probe has done its job: this relay grants THIS identity. Tear it down and ask the
|
|
271
|
+
// next relay — the reservation is re-taken by the final node below, which is the only one
|
|
272
|
+
// that can listen on every granted address at once. AWAITED, because the next probe comes
|
|
273
|
+
// up on this same peer id.
|
|
274
|
+
try {
|
|
275
|
+
await candidate.stop();
|
|
276
|
+
}
|
|
277
|
+
catch { /* it may never have finished starting */ }
|
|
278
|
+
}
|
|
187
279
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
280
|
+
}
|
|
281
|
+
if (!candidateGranted && !rejectionNamed) {
|
|
282
|
+
const rejectionReason =
|
|
283
|
+
// A CLIENT-SIDE FAULT KEEPS ITS OWN NAME, and it is checked FIRST because everything below
|
|
284
|
+
// infers a cause from the relay connection — which is intact and irrelevant when the ask
|
|
285
|
+
// never left this process (review HIGH-2).
|
|
286
|
+
askFault !== undefined
|
|
287
|
+
? askFault
|
|
288
|
+
: outcome === "started"
|
|
289
|
+
? /**
|
|
290
|
+
* ⚠️ Review MEDIUM-7 — **"ASKED" DOES NOT MEAN THE RELAY ANSWERED.** A relay that is
|
|
291
|
+
* simply DOWN yields no circuit address — indistinguishable, here, from a relay that
|
|
292
|
+
* answered and granted nothing. Reporting that as `relay_granted_no_reservation` sends
|
|
293
|
+
* the operator to look at relay capacity for what is a network fault. An open
|
|
294
|
+
* connection to the relay peer is the thing that separates them, and we have one to
|
|
295
|
+
* ask. (Reachable now only in the narrow case where the proof landed and the
|
|
296
|
+
* connection died before the ask — the proof branch above owns every other route to a
|
|
297
|
+
* dead relay, and names it.)
|
|
298
|
+
*/
|
|
299
|
+
(candidate.getConnections().some((c) => c.peerId === relayPeerIdOf(circuitAddr))
|
|
300
|
+
? "relay_granted_no_reservation"
|
|
301
|
+
: "relay_unreachable")
|
|
302
|
+
: outcome === "failed"
|
|
303
|
+
? "relay_unreachable"
|
|
304
|
+
: "reservation_did_not_complete_in_time";
|
|
204
305
|
this.#ctx.srLastRejectionReason.set(agentName, rejectionReason);
|
|
205
306
|
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
206
307
|
agentName,
|
|
207
308
|
circuitAddr,
|
|
208
309
|
reason: rejectionReason,
|
|
209
|
-
attempts: attempt + 1,
|
|
210
310
|
...(error !== "" ? { error } : {}),
|
|
211
311
|
correlationId,
|
|
212
312
|
});
|
|
@@ -214,8 +314,10 @@ export class StandingReceivers {
|
|
|
214
314
|
// a dial, and this candidate carries the receiver's identity: an unawaited `stop()` on a node
|
|
215
315
|
// whose status is still `starting` returns without stopping anything, and the node then goes
|
|
216
316
|
// live on our peer id with nothing left holding a reference to kill it.
|
|
217
|
-
|
|
218
|
-
|
|
317
|
+
//
|
|
318
|
+
// BOTH promises: the reservation is taken by the ASK now, so a candidate abandoned on the
|
|
319
|
+
// ask's deadline still has work in flight that `start()` settling says nothing about.
|
|
320
|
+
stopWhenSettled(candidate, [startP, listenP], this.#ctx.srReservationTimeoutMs * 2);
|
|
219
321
|
}
|
|
220
322
|
if (candidateGranted)
|
|
221
323
|
grantedAddrs.push(circuitAddr);
|
|
@@ -289,10 +391,24 @@ export class StandingReceivers {
|
|
|
289
391
|
*
|
|
290
392
|
* ⚠️ IT HAS A KNOWN CAUSE AND A CROSS-REPO CLOCK. The walk stops the granted candidate and the
|
|
291
393
|
* node below RE-ASKS, which works because the relay remembers the proof — for
|
|
292
|
-
* `PROVEN_PEER_MEMORY_MS = 2 minutes` (`relay-connection-gater.ts`, trustless-cello).
|
|
293
|
-
*
|
|
294
|
-
*
|
|
295
|
-
*
|
|
394
|
+
* `PROVEN_PEER_MEMORY_MS = 2 minutes` (`relay-connection-gater.ts`, trustless-cello).
|
|
395
|
+
*
|
|
396
|
+
* ⚠️ **AND THIS IS WHERE THE ORIGINAL COLLISION STILL LIVES** (review MEDIUM-3). The node built
|
|
397
|
+
* below carries `circuitRelayListenAddrs`, so libp2p asks at start, on a FRESH connection that
|
|
398
|
+
* has proved nothing on itself. `DOD-M15-RELAYPROVE-ORDER-1` removed the refused first ask from
|
|
399
|
+
* the WALK; the installed receiver still depends on the relay's two-minute peer memory, and when
|
|
400
|
+
* that memory has expired the whole original loop returns for it — refused ask, connection
|
|
401
|
+
* manager restart, connections closed. **It is mitigated by a cross-repo TTL, not removed.** The
|
|
402
|
+
* probe/final split is what forces it, and that split is what unit 2 of
|
|
403
|
+
* `M15-STORY-RESERVATIONS-ON-DEMAND` (splitting the standing receiver) deletes; the fix belongs
|
|
404
|
+
* there rather than in a second pass here.
|
|
405
|
+
*
|
|
406
|
+
* ⚠️ **THE BUDGET ARITHMETIC BELOW IS NOT WHAT IT WAS.** It used to read *"`#srReservationTimeoutMs`
|
|
407
|
+
* × 2 attempts per relay"* — there is one attempt now, but the wall clock did NOT halve: a
|
|
408
|
+
* candidate costs `start` + `proveToRelay` + `ask`, and **`proveToRelay` carries no deadline of
|
|
409
|
+
* its own at either call site**. So a pool of three can still exceed the two-minute memory, and
|
|
410
|
+
* the ceiling is now harder to state than it was, not easier. That missing deadline is recorded
|
|
411
|
+
* in `053-FIRSTASK`'s *Newly discovered*.
|
|
296
412
|
*/
|
|
297
413
|
const boundRelays = new Set(heldRelayIdsOf(node));
|
|
298
414
|
const grantedButUnbound = grantedAddrs
|
|
@@ -685,21 +801,27 @@ export class StandingReceivers {
|
|
|
685
801
|
* candidate and came up on the plain floor: alive, `active`, and dialable by nobody, with
|
|
686
802
|
* every message in both directions forced through the relay park route.
|
|
687
803
|
*
|
|
688
|
-
*
|
|
689
|
-
*
|
|
690
|
-
*
|
|
691
|
-
*
|
|
804
|
+
* ⚠️ DOD-M15-RELAYPROVE-ORDER-1 — **ONE ATTEMPT NOW, exactly as `#startReceiverNode` does
|
|
805
|
+
* it.** This used to be two: ask, be refused, prove, ask again, justified by *"a reservation
|
|
806
|
+
* taken by hand on the same connection as the proof yields no dialable address."* That
|
|
807
|
+
* described taking the slot over a raw HOP stream; asking libp2p's own transport manager
|
|
808
|
+
* after the proof makes the reservation libp2p's own, and it announces the address. Measured
|
|
809
|
+
* live 2026-09-08. So the candidate comes up with no circuit address, proves, and asks once.
|
|
810
|
+
*
|
|
811
|
+
* The seed is fixed here — that is what a revival IS — so this node carries the identity the
|
|
812
|
+
* relay records, and it must STAY UP between the proof and the ask: the relay marks the
|
|
813
|
+
* CONNECTION proven, and stopping the node closes it.
|
|
692
814
|
*/
|
|
693
815
|
let revivedNode;
|
|
694
816
|
let terminalRefusal = false;
|
|
695
|
-
|
|
817
|
+
{
|
|
696
818
|
const candidate = await this.createAgentNode(agentName, {
|
|
697
819
|
sessionId,
|
|
698
820
|
connectionGater: gater,
|
|
699
821
|
nodeType: "session",
|
|
700
822
|
inboundReachable: true,
|
|
701
823
|
transportPrivateKey: seed,
|
|
702
|
-
circuitRelayListenAddrs
|
|
824
|
+
// NO `circuitRelayListenAddrs` — libp2p must not ask before the proof below has landed.
|
|
703
825
|
});
|
|
704
826
|
// KEEP THE START PROMISE. Review HIGH-3: `libp2p.stop()` opens with
|
|
705
827
|
// `if (this.status !== 'started') return`, and during the whole timeout window the status is
|
|
@@ -714,82 +836,148 @@ export class StandingReceivers {
|
|
|
714
836
|
startP.then(() => true),
|
|
715
837
|
new Promise((res) => setTimeout(() => res(false), REVIVE_RESERVATION_TIMEOUT_MS).unref?.()),
|
|
716
838
|
]).catch((err) => { startError = err; return false; });
|
|
717
|
-
if (started && candidate.listenAddresses().some((a) => a.includes("/p2p-circuit"))) {
|
|
718
|
-
this.#ctx.logger.info("session.revive.reservation.granted", { agentName, sessionId, attempts: attempt + 1 });
|
|
719
|
-
revivedNode = candidate;
|
|
720
|
-
break;
|
|
721
|
-
}
|
|
722
839
|
/**
|
|
723
|
-
*
|
|
724
|
-
* aged out. Prove and go round once more.
|
|
840
|
+
* PROVE, THEN ASK — the same order as `#startReceiverNode`, for the same reason.
|
|
725
841
|
*
|
|
726
|
-
*
|
|
727
|
-
* return`, so a timed-out candidate cannot be torn down here
|
|
728
|
-
*
|
|
729
|
-
*
|
|
842
|
+
* `started` gates it because `libp2p.stop()` opens with `if (this.status !== 'started')
|
|
843
|
+
* return`, so a timed-out candidate cannot be torn down here; that case falls through to the
|
|
844
|
+
* settlement-chained teardown below, which is the only thing that reliably kills a
|
|
845
|
+
* still-starting node.
|
|
846
|
+
*
|
|
847
|
+
* ⚠️ THE CANDIDATE IS NOT STOPPED BETWEEN THE PROOF AND THE ASK. It used to be, because the
|
|
848
|
+
* ask came from a rebuilt node. The relay marks the CONNECTION proven, so stopping here
|
|
849
|
+
* would throw away the very thing that makes the next line succeed.
|
|
850
|
+
*/
|
|
851
|
+
let proofDeclined = false;
|
|
852
|
+
/**
|
|
853
|
+
* The ask's own promise, when one was made. The teardown below has to wait on THIS as well as
|
|
854
|
+
* on `start()`: the node whose reservation is still in flight is the one that can come up
|
|
855
|
+
* late holding this session's peer id, and `stop()` on a node mid-ask is the same no-op the
|
|
856
|
+
* start-promise note describes.
|
|
730
857
|
*/
|
|
731
|
-
|
|
858
|
+
let listenP;
|
|
859
|
+
/** Set when the ask failed for a fault of OURS, so it is not re-described as the relay's. */
|
|
860
|
+
let askFault;
|
|
861
|
+
if (started) {
|
|
732
862
|
const verdict = await this.#ctx.proveToRelay(agentName, circuitAddr, candidate, sessionId, false);
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
// The refusal is about this AGENT, so the remaining candidates
|
|
739
|
-
|
|
863
|
+
// A VERDICT DECLINES; NO VERDICT DOES NOT. `unavailable` means the relay never answered —
|
|
864
|
+
// no client wired, or unreachable — and not every relay gates reservations, so the ask
|
|
865
|
+
// still goes ahead. Same rule and same reasoning as `#startReceiverNode`.
|
|
866
|
+
if (verdict === "refused_this_agent" || verdict === "refused_try_another_relay") {
|
|
867
|
+
proofDeclined = true;
|
|
868
|
+
// The agent-level refusal is about this AGENT, so the remaining candidates answer
|
|
869
|
+
// identically.
|
|
870
|
+
if (verdict === "refused_this_agent")
|
|
871
|
+
terminalRefusal = true;
|
|
740
872
|
this.#ctx.logger.warn("session.revive.reservation.declined", {
|
|
741
873
|
agentName,
|
|
742
874
|
sessionId,
|
|
743
875
|
circuitAddr,
|
|
744
|
-
reason: "relay_refused_this_agent",
|
|
745
|
-
impact:
|
|
746
|
-
"
|
|
747
|
-
|
|
876
|
+
reason: verdict === "refused_this_agent" ? "relay_refused_this_agent" : "relay_proof_refused",
|
|
877
|
+
impact: verdict === "refused_this_agent"
|
|
878
|
+
? "the relay refused this agent rather than being unwilling or unwell, so every " +
|
|
879
|
+
"other relay refuses it the same way. The session comes up reachable only via " +
|
|
880
|
+
"the relay park route; cello_status carries the cause."
|
|
881
|
+
: "this relay would not take the agent's proof. Trying the next relay.",
|
|
748
882
|
});
|
|
749
|
-
|
|
883
|
+
try {
|
|
884
|
+
await candidate.stop();
|
|
885
|
+
}
|
|
886
|
+
catch { /* best-effort */ }
|
|
750
887
|
}
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
888
|
+
else {
|
|
889
|
+
/**
|
|
890
|
+
* ASK — once, on the connection the proof was made on.
|
|
891
|
+
*
|
|
892
|
+
* ⚠️ **RACED AGAINST THE SAME DEADLINE `start()` USED TO CARRY, AND IT HAS TO BE.** The
|
|
893
|
+
* measured production failure this whole loop exists for — 10,002ms and still waiting —
|
|
894
|
+
* was a relay that never answered a reservation. That used to park `start()`, because a
|
|
895
|
+
* circuit address in the constructor made start the moment libp2p asked. The ask is here
|
|
896
|
+
* now, so a bare await here is the same hang with a new address: the revival never
|
|
897
|
+
* returns and every send on that session is refused forever.
|
|
898
|
+
*
|
|
899
|
+
* A throw is not fatal — the grant check below is the only thing that decides, and it
|
|
900
|
+
* reads the announced addresses.
|
|
901
|
+
*/
|
|
902
|
+
// Wrapped, not bare: a node that cannot take the ask at all throws SYNCHRONOUSLY, and
|
|
903
|
+
// `.catch()` on the race never sees that — it would escape the revival entirely.
|
|
904
|
+
const asked = await (async () => {
|
|
905
|
+
listenP = candidate.listenOnCircuit(circuitAddr);
|
|
906
|
+
return Promise.race([
|
|
907
|
+
listenP.then(() => true),
|
|
908
|
+
new Promise((res) => setTimeout(() => res(false), REVIVE_RESERVATION_TIMEOUT_MS).unref?.()),
|
|
909
|
+
]);
|
|
910
|
+
})().catch((err) => {
|
|
911
|
+
startError = err;
|
|
912
|
+
// Review HIGH-2, same rule as the receiver walk: a fault of OURS keeps its own name.
|
|
913
|
+
askFault = clientSideAskFault(err);
|
|
914
|
+
return false;
|
|
760
915
|
});
|
|
761
|
-
|
|
916
|
+
if (!asked && askFault === undefined) {
|
|
917
|
+
this.#ctx.logger.warn("session.revive.reservation.ask_timeout", {
|
|
918
|
+
agentName,
|
|
919
|
+
sessionId,
|
|
920
|
+
circuitAddr,
|
|
921
|
+
budgetMs: REVIVE_RESERVATION_TIMEOUT_MS,
|
|
922
|
+
impact: "this relay took the proof and then never answered the reservation. Abandoned " +
|
|
923
|
+
"on the deadline and trying the next relay — a relay that does not answer must not " +
|
|
924
|
+
"be able to hold a session down.",
|
|
925
|
+
});
|
|
926
|
+
}
|
|
762
927
|
}
|
|
763
|
-
continue;
|
|
764
928
|
}
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
:
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
929
|
+
if (!proofDeclined && started && holdsCircuit(candidate)) {
|
|
930
|
+
this.#ctx.logger.info("session.revive.reservation.granted", { agentName, sessionId });
|
|
931
|
+
revivedNode = candidate;
|
|
932
|
+
break;
|
|
933
|
+
}
|
|
934
|
+
if (!proofDeclined) {
|
|
935
|
+
// Started but granted nothing, or never started. Either way this node is not the one.
|
|
936
|
+
//
|
|
937
|
+
// Review MEDIUM-5: name WHICH of the three causes this was, the way `#startReceiverNode` does.
|
|
938
|
+
// "declined" alone stood for a relay that is full, a relay that is unreachable, and a relay
|
|
939
|
+
// that is merely slow — three different problems with three different responses, and the
|
|
940
|
+
// thrown error was discarded entirely.
|
|
941
|
+
const declineReason = askFault !== undefined
|
|
942
|
+
// A client-side fault, checked FIRST: everything below infers a cause from the relay, and
|
|
943
|
+
// the relay had nothing to do with an ask that never left this process (review HIGH-2).
|
|
944
|
+
? askFault
|
|
945
|
+
: !started
|
|
946
|
+
? startError !== undefined
|
|
947
|
+
? "relay_unreachable"
|
|
948
|
+
: "reservation_did_not_complete_in_time"
|
|
949
|
+
: // Started, proved, asked — and the ask is where a slow relay now shows up. An ask still in
|
|
950
|
+
// flight is "did not complete in time"; one that returned with nothing is a relay that
|
|
951
|
+
// answered and granted nothing.
|
|
952
|
+
listenP !== undefined && !holdsCircuit(candidate)
|
|
953
|
+
? "relay_granted_no_reservation"
|
|
954
|
+
: "reservation_did_not_complete_in_time";
|
|
955
|
+
const isLast = circuitAddr === candidateAddrs.slice(0, REVIVE_RESERVATION_CANDIDATES).at(-1);
|
|
956
|
+
this.#ctx.logger.warn("session.revive.reservation.declined", {
|
|
957
|
+
agentName,
|
|
958
|
+
sessionId,
|
|
959
|
+
circuitAddr,
|
|
960
|
+
reason: declineReason,
|
|
961
|
+
...(startError !== undefined ? { error: extractErrorMessage(startError) } : {}),
|
|
962
|
+
impact: isLast
|
|
963
|
+
? "no relay granted; the session comes up reachable only via the relay park route"
|
|
964
|
+
: "trying the next relay",
|
|
965
|
+
});
|
|
966
|
+
/**
|
|
967
|
+
* Teardown at SETTLEMENT, not now: a `stop()` issued while the node is still starting is a
|
|
968
|
+
* no-op (see above), so the only way to guarantee this node dies is to wait for its own work
|
|
969
|
+
* to finish first. Not awaited, so a hung relay cannot hold the revival up — the point is that
|
|
970
|
+
* the teardown eventually happens, not that it happens before the next candidate.
|
|
971
|
+
*
|
|
972
|
+
* ⚠️ **BOTH PROMISES, and the second one is new.** The abandoned candidate's outstanding work
|
|
973
|
+
* used to be `start()`, because that is where the reservation was taken. It is the ASK now, so
|
|
974
|
+
* waiting only on `start()` tears the node down while its reservation is still in flight —
|
|
975
|
+
* and a late grant then brings a node up on THIS SESSION'S peer id, sharing the gater, with no
|
|
976
|
+
* content handler and nothing holding a reference to kill it. That is the open endpoint
|
|
977
|
+
* review HIGH-3 exists to prevent, reintroduced through a different promise.
|
|
978
|
+
*/
|
|
979
|
+
stopWhenSettled(candidate, [startP, listenP], REVIVE_RESERVATION_TIMEOUT_MS * 2);
|
|
980
|
+
}
|
|
793
981
|
}
|
|
794
982
|
if (revivedNode)
|
|
795
983
|
return revivedNode;
|