@cello-protocol/daemon 0.0.207 → 0.0.209
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-handlers.d.ts +12 -0
- package/dist/agent-handlers.d.ts.map +1 -1
- package/dist/agent-handlers.js +6 -2
- package/dist/agent-handlers.js.map +1 -1
- package/dist/attendance-wiring.js +2 -2
- package/dist/attendance-wiring.js.map +1 -1
- package/dist/boot-agents.d.ts.map +1 -1
- package/dist/boot-agents.js +5 -5
- package/dist/boot-agents.js.map +1 -1
- package/dist/close-session-handler.d.ts.map +1 -1
- package/dist/close-session-handler.js +26 -3
- package/dist/close-session-handler.js.map +1 -1
- package/dist/content-park.d.ts.map +1 -1
- package/dist/content-park.js +5 -3
- package/dist/content-park.js.map +1 -1
- package/dist/daemon.js +5 -4
- package/dist/daemon.js.map +1 -1
- package/dist/outbound-sessions.d.ts +27 -2
- package/dist/outbound-sessions.d.ts.map +1 -1
- package/dist/outbound-sessions.js +53 -3
- package/dist/outbound-sessions.js.map +1 -1
- package/dist/park-recovery.d.ts +3 -2
- package/dist/park-recovery.d.ts.map +1 -1
- package/dist/park-recovery.js +3 -2
- package/dist/park-recovery.js.map +1 -1
- package/dist/refusal-reasons.d.ts +46 -0
- package/dist/refusal-reasons.d.ts.map +1 -1
- package/dist/refusal-reasons.js +44 -0
- package/dist/refusal-reasons.js.map +1 -1
- package/dist/relay-only.d.ts +9 -2
- package/dist/relay-only.d.ts.map +1 -1
- package/dist/relay-only.js +9 -2
- package/dist/relay-only.js.map +1 -1
- package/dist/restart-seal-resolver.d.ts.map +1 -1
- package/dist/restart-seal-resolver.js +13 -0
- package/dist/restart-seal-resolver.js.map +1 -1
- package/dist/session-ceremony.d.ts +17 -0
- package/dist/session-ceremony.d.ts.map +1 -1
- package/dist/session-ceremony.js +77 -1
- package/dist/session-ceremony.js.map +1 -1
- package/dist/session-content-send.js +12 -11
- package/dist/session-content-send.js.map +1 -1
- package/dist/session-lifecycle.d.ts.map +1 -1
- package/dist/session-lifecycle.js +36 -0
- package/dist/session-lifecycle.js.map +1 -1
- package/dist/session-node-factory.js +4 -4
- package/dist/session-node-factory.js.map +1 -1
- package/dist/session-node-manager.d.ts +7 -2
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +22 -6
- package/dist/session-node-manager.js.map +1 -1
- package/dist/session-node-types.d.ts +79 -13
- package/dist/session-node-types.d.ts.map +1 -1
- package/dist/session-node-types.js +67 -0
- package/dist/session-node-types.js.map +1 -1
- package/dist/session-relay-client.d.ts +21 -0
- package/dist/session-relay-client.d.ts.map +1 -1
- package/dist/session-relay-client.js +219 -3
- package/dist/session-relay-client.js.map +1 -1
- package/dist/session-relay.d.ts +15 -1
- package/dist/session-relay.d.ts.map +1 -1
- package/dist/session-relay.js +264 -168
- package/dist/session-relay.js.map +1 -1
- package/dist/signaling-connect.d.ts.map +1 -1
- package/dist/signaling-connect.js +23 -4
- package/dist/signaling-connect.js.map +1 -1
- package/dist/signaling-wiring.js +7 -7
- package/dist/signaling-wiring.js.map +1 -1
- package/dist/standing-receivers.d.ts +53 -14
- package/dist/standing-receivers.d.ts.map +1 -1
- package/dist/standing-receivers.js +695 -500
- package/dist/standing-receivers.js.map +1 -1
- package/dist/trust-signal-sweep-tick.d.ts +60 -0
- package/dist/trust-signal-sweep-tick.d.ts.map +1 -0
- package/dist/trust-signal-sweep-tick.js +101 -0
- package/dist/trust-signal-sweep-tick.js.map +1 -0
- package/dist/trust-signal-sweep.d.ts +16 -1
- package/dist/trust-signal-sweep.d.ts.map +1 -1
- package/dist/trust-signal-sweep.js +40 -16
- package/dist/trust-signal-sweep.js.map +1 -1
- package/dist/types.d.ts +18 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +5 -5
|
@@ -3,7 +3,7 @@ import { SessionConnectionGater } from "./session-connection-gater.js";
|
|
|
3
3
|
import { relayOnlyState, publishableEndpoint } from "./relay-only.js";
|
|
4
4
|
import { extractErrorMessage } from "./error-message.js";
|
|
5
5
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
6
|
-
import { relayPeerIdOf, heldRelayIdsOf,
|
|
6
|
+
import { relayPeerIdOf, heldRelayIdsOf, SR_RESERVATION_MAX_RETRIES, REVIVE_RESERVATION_CANDIDATES, REVIVE_RESERVATION_TIMEOUT_MS, RELEASE_TELL_BUDGET_MS, OFFER_RESERVATION_GRACE_MS, clientSideAskFault, holdsCircuit, stopWhenSettled, } from "./session-node-types.js";
|
|
7
7
|
import { STANDING_RECEIVER_AGENT_NAME } from "./types.js";
|
|
8
8
|
export class StandingReceivers {
|
|
9
9
|
#ctx;
|
|
@@ -25,310 +25,474 @@ export class StandingReceivers {
|
|
|
25
25
|
}
|
|
26
26
|
async #startReceiverNode(agentName, sessionId, gater, candidateCircuitAddrs, correlationId) {
|
|
27
27
|
/**
|
|
28
|
-
*
|
|
29
|
-
* DOD-M12B-SESSION-SEED-1's seed-per-candidate.
|
|
28
|
+
* 054-SRSPLIT — **ONE NODE. IT STARTS ON TCP, THEN TAKES ITS RESERVATIONS IN PLACE.**
|
|
30
29
|
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* the
|
|
30
|
+
* This used to be a walk of throwaway PROBE nodes — one built per relay to find out whether that
|
|
31
|
+
* relay would grant, torn down, and then a FINAL node rebuilt carrying every granted address in
|
|
32
|
+
* its constructor. The probes existed for exactly one reason: a circuit listener was fixed at
|
|
33
|
+
* node creation, so the only way to ask a relay anything was to build a node to ask with.
|
|
35
34
|
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
* `#buildRevivedNode` already runs a fixed identity through this same walk for the same reason.
|
|
35
|
+
* `listenOnCircuit` (unit 1) removed that constraint, and this is the simplification it unlocks:
|
|
36
|
+
* build the receiver ONCE, and for each relay prove and ask on the node that will keep it.
|
|
37
|
+
*
|
|
38
|
+
* ⚠️ **AND IT CLOSES THE DEFECT UNIT 1 LEFT STANDING, which was recorded here rather than
|
|
39
|
+
* fixed.** The final node was built with `circuitRelayListenAddrs`, so libp2p asked at start on
|
|
40
|
+
* a fresh connection that had proved nothing on itself. It worked only because the relay
|
|
41
|
+
* remembers a proof for `PROVEN_PEER_MEMORY_MS` — two minutes, in the OTHER repo — and when that
|
|
42
|
+
* memory had expired the original refused-ask collision came back for the one node that IS the
|
|
43
|
+
* agent's front door. There is no constructor-time ask left, so that dependency is gone, and
|
|
44
|
+
* `spread.grant_not_bound` — the event that existed to catch the proof expiring between the walk
|
|
45
|
+
* and the rebuild — has nothing left to report.
|
|
46
|
+
*
|
|
47
|
+
* ⚠️ **ONE SEED, and it still matters** (032-RELAYSPREAD). The agent is ONE identity and must be
|
|
48
|
+
* dialable at ONE peer id through any of its circuits. That is now structural rather than
|
|
49
|
+
* maintained: there is one node, so there is one key, and a second identity has nowhere to come
|
|
50
|
+
* from. The old hazard this note used to carry — an abandoned probe still starting on the
|
|
51
|
+
* receiver's seed — cannot occur, because no probe is built.
|
|
54
52
|
*/
|
|
55
53
|
const receiverSeed = randomBytes(32);
|
|
56
|
-
/** Circuit addresses whose relay ACTUALLY GRANTED this identity a reservation. */
|
|
57
|
-
const grantedAddrs = [];
|
|
58
|
-
// For `spread.grant_not_bound` below: the walk's own duration is measured against the relay's
|
|
59
|
-
// two-minute proof memory, so it has to be a number rather than an inference.
|
|
60
|
-
const walkStartedAt = Date.now();
|
|
61
|
-
for (const circuitAddr of candidateCircuitAddrs) {
|
|
62
|
-
const candidateSeed = receiverSeed;
|
|
63
|
-
/**
|
|
64
|
-
* DOD-M15-RELAYSLOTS-1 — **TWO ATTEMPTS PER RELAY: ask, prove, ask again.**
|
|
65
|
-
*
|
|
66
|
-
* The relay now refuses a reservation from a peer that has not shown it belongs to a
|
|
67
|
-
* registered agent. A brand-new receiver has shown nothing, so its FIRST ask is refused —
|
|
68
|
-
* expected, not a failure. It then authenticates over `/cello/relay/1.0.0`, which tells the
|
|
69
|
-
* relay this transport identity is a registered agent's, and asks again on a fresh connection
|
|
70
|
-
* carrying the SAME identity (that is what reusing `candidateSeed` buys).
|
|
71
|
-
*
|
|
72
|
-
* ⚠️ It has to be two connections, and that was measured rather than chosen. Taking the
|
|
73
|
-
* reservation by hand on the same connection as the proof DOES get a slot — and libp2p then
|
|
74
|
-
* announces no circuit address for it, because it only announces addresses for reservations
|
|
75
|
-
* its own relay-discovery made. The agent would hold a slot nobody could dial through.
|
|
76
|
-
*/
|
|
77
|
-
let candidateGranted = false;
|
|
78
|
-
// Set when the relay refused the AGENT rather than being unwilling itself: every other relay
|
|
79
|
-
// in the pool answers identically, so the walk ends here rather than reproducing it N times.
|
|
80
|
-
let candidateRefusedAgent = false;
|
|
81
|
-
for (let attempt = 0; attempt < 2; attempt++) {
|
|
82
|
-
const candidate = await this.createAgentNode(agentName, {
|
|
83
|
-
sessionId,
|
|
84
|
-
connectionGater: gater,
|
|
85
|
-
nodeType: "standing_receiver",
|
|
86
|
-
circuitRelayListenAddrs: [circuitAddr],
|
|
87
|
-
transportPrivateKey: candidateSeed,
|
|
88
|
-
});
|
|
89
|
-
let timer;
|
|
90
|
-
const timedOut = Symbol("reservation_timeout");
|
|
91
|
-
let outcome = "failed";
|
|
92
|
-
let error = "";
|
|
93
|
-
// KEEP THE START PROMISE. Every candidate now carries the receiver's identity, so an
|
|
94
|
-
// abandoned one must be reliably torn down rather than best-effort — and only its own start
|
|
95
|
-
// promise says when it is stoppable (see the seed note above).
|
|
96
|
-
const startP = candidate.start();
|
|
97
|
-
try {
|
|
98
|
-
outcome = await Promise.race([
|
|
99
|
-
startP.then(() => "started"),
|
|
100
|
-
new Promise((resolve) => {
|
|
101
|
-
timer = setTimeout(() => resolve(timedOut), this.#ctx.srReservationTimeoutMs);
|
|
102
|
-
}),
|
|
103
|
-
]);
|
|
104
|
-
}
|
|
105
|
-
catch (err) {
|
|
106
|
-
error = extractErrorMessage(err);
|
|
107
|
-
}
|
|
108
|
-
finally {
|
|
109
|
-
if (timer !== undefined)
|
|
110
|
-
clearTimeout(timer);
|
|
111
|
-
}
|
|
112
|
-
// The only proof that counts: the relay actually GRANTED the reservation.
|
|
113
|
-
// start() resolving is not enough — a relay that is out of reservation slots
|
|
114
|
-
// completes the handshake and simply grants nothing, leaving a node that looks
|
|
115
|
-
// started and is reachable by nobody.
|
|
116
|
-
if (outcome === "started" && candidate.listenAddresses().some((a) => a.includes("/p2p-circuit"))) {
|
|
117
|
-
candidateGranted = true;
|
|
118
|
-
// The probe has done its job: this relay grants THIS identity. Tear it down and ask the
|
|
119
|
-
// next relay — the reservation is re-taken by the final node below, which is the only one
|
|
120
|
-
// that can listen on every granted address at once. AWAITED, because the next probe comes
|
|
121
|
-
// up on this same peer id.
|
|
122
|
-
try {
|
|
123
|
-
await candidate.stop();
|
|
124
|
-
}
|
|
125
|
-
catch { /* it may never have finished starting */ }
|
|
126
|
-
break;
|
|
127
|
-
}
|
|
128
|
-
/**
|
|
129
|
-
* No reservation. On the FIRST attempt that is the expected answer for a receiver that has
|
|
130
|
-
* not proved itself yet, so prove and go round once more. `proveReservation` opens its own
|
|
131
|
-
* stream from this node, which is what binds this transport identity to the agent at the
|
|
132
|
-
* relay; the relay remembers it across the reconnect below.
|
|
133
|
-
*/
|
|
134
|
-
if (attempt === 0 && outcome === "started") {
|
|
135
|
-
const verdict = await this.#ctx.proveToRelay(agentName, circuitAddr, candidate, correlationId, true);
|
|
136
|
-
// AWAITED, not fire-and-forget: the retry rebuilds on this same transport identity, and two
|
|
137
|
-
// live nodes sharing one peer id is the defect DOD-M12B-SESSION-SEED-1 exists to prevent.
|
|
138
|
-
try {
|
|
139
|
-
await candidate.stop();
|
|
140
|
-
}
|
|
141
|
-
catch { /* it may never have finished starting */ }
|
|
142
|
-
/**
|
|
143
|
-
* DOD-M15-RELAYSLOTS-1 clause 9 — **A CLIENT-SIDE REFUSAL ENDS THE WALK.**
|
|
144
|
-
*
|
|
145
|
-
* `slot_cap_exceeded` and an expired or missing token are classified `tryAnotherRelay:
|
|
146
|
-
* false` because they reproduce on every relay in the pool: the cap is per AGENT, and the
|
|
147
|
-
* token comes from the directory, not from here. Walking on costs a node build and two
|
|
148
|
-
* dials per remaining relay to arrive at the same answer, and it makes one client-side
|
|
149
|
-
* fault look like a fleet-wide outage in the logs. The refusal is already recorded where
|
|
150
|
-
* `cello_status` reads it, so stopping is not silence.
|
|
151
|
-
*/
|
|
152
|
-
if (verdict === "refused_this_agent") {
|
|
153
|
-
this.#ctx.srLastRejectionReason.set(agentName, "relay_refused_this_agent");
|
|
154
|
-
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
155
|
-
agentName,
|
|
156
|
-
circuitAddr,
|
|
157
|
-
reason: "relay_refused_this_agent",
|
|
158
|
-
attempts: attempt + 1,
|
|
159
|
-
correlationId,
|
|
160
|
-
impact: "the relay refused this AGENT rather than this relay being unwilling or " +
|
|
161
|
-
"unwell, so every other relay would refuse it identically. Stopped here; " +
|
|
162
|
-
"cello_status carries the cause and what to do about it.",
|
|
163
|
-
});
|
|
164
|
-
candidateRefusedAgent = true;
|
|
165
|
-
break;
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* ⚠️ RETRY ONLY WHAT A PROOF CAN FIX. The second attempt exists because the relay now
|
|
169
|
-
* remembers this transport identity; if the proof did not land, it remembers nothing and
|
|
170
|
-
* the retry is a node build and a dial spent to be refused identically. Only `proven`
|
|
171
|
-
* earns the retry — everything else moves to the next relay.
|
|
172
|
-
*/
|
|
173
|
-
if (verdict !== "proven") {
|
|
174
|
-
this.#ctx.srLastRejectionReason.set(agentName, "relay_proof_refused");
|
|
175
|
-
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
176
|
-
agentName,
|
|
177
|
-
circuitAddr,
|
|
178
|
-
reason: "relay_proof_refused",
|
|
179
|
-
attempts: attempt + 1,
|
|
180
|
-
correlationId,
|
|
181
|
-
impact: "this relay would not take the agent's proof, so it will refuse the retry the " +
|
|
182
|
-
"same way. Moving to the next relay rather than asking this one twice.",
|
|
183
|
-
});
|
|
184
|
-
break;
|
|
185
|
-
}
|
|
186
|
-
continue;
|
|
187
|
-
}
|
|
188
|
-
const rejectionReason = outcome === "started"
|
|
189
|
-
? /**
|
|
190
|
-
* ⚠️ Review MEDIUM-7 — **"STARTED" DOES NOT MEAN THE RELAY ANSWERED.** A circuit listen
|
|
191
|
-
* entry sets `FaultTolerance.NO_FATAL`, and `start()` only throws when the DIRECT
|
|
192
|
-
* listener fails, so a relay that is simply DOWN resolves `started` with no circuit
|
|
193
|
-
* address — indistinguishable, here, from a relay that answered and granted nothing.
|
|
194
|
-
* Reporting that as `relay_granted_no_reservation` sends the operator to look at relay
|
|
195
|
-
* capacity for what is a network fault. An open connection to the relay peer is the
|
|
196
|
-
* thing that separates them, and we have one to ask.
|
|
197
|
-
*/
|
|
198
|
-
(candidate.getConnections().some((c) => c.peerId === relayPeerIdOf(circuitAddr))
|
|
199
|
-
? "relay_granted_no_reservation"
|
|
200
|
-
: "relay_unreachable")
|
|
201
|
-
: outcome === "failed"
|
|
202
|
-
? "relay_unreachable"
|
|
203
|
-
: "reservation_did_not_complete_in_time";
|
|
204
|
-
this.#ctx.srLastRejectionReason.set(agentName, rejectionReason);
|
|
205
|
-
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
206
|
-
agentName,
|
|
207
|
-
circuitAddr,
|
|
208
|
-
reason: rejectionReason,
|
|
209
|
-
attempts: attempt + 1,
|
|
210
|
-
...(error !== "" ? { error } : {}),
|
|
211
|
-
correlationId,
|
|
212
|
-
});
|
|
213
|
-
// Abandon it — but on its OWN settlement, never best-effort. `start()` may still be parked on
|
|
214
|
-
// a dial, and this candidate carries the receiver's identity: an unawaited `stop()` on a node
|
|
215
|
-
// whose status is still `starting` returns without stopping anything, and the node then goes
|
|
216
|
-
// live on our peer id with nothing left holding a reference to kill it.
|
|
217
|
-
void startP.then(() => candidate.stop().catch(() => { }), () => { });
|
|
218
|
-
break;
|
|
219
|
-
}
|
|
220
|
-
if (candidateGranted)
|
|
221
|
-
grantedAddrs.push(circuitAddr);
|
|
222
|
-
// 032-RELAYSPREAD: DO NOT BREAK ON THE FIRST GRANT. The walk used to stop here, which is why
|
|
223
|
-
// an agent held exactly one reservation and losing that relay cost it every NAT'd caller for
|
|
224
|
-
// however long detection happened to take. It now asks every remaining relay.
|
|
225
|
-
if (candidateRefusedAgent)
|
|
226
|
-
break;
|
|
227
|
-
}
|
|
228
54
|
/**
|
|
229
|
-
*
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
* identity, so the final node's first ask is the one that succeeds; the two-attempt dance was
|
|
234
|
-
* already paid per relay in the walk.
|
|
235
|
-
*
|
|
236
|
-
* ⚠️ RACED AGAINST A DEADLINE, and that is measured rather than cautious: `#buildRevivedNode`
|
|
237
|
-
* records a live 2026-08-18 result where a node handed two relay addresses at once with no
|
|
238
|
-
* deadline never finished starting at all (10,002ms and counting). Its identity was unproven at
|
|
239
|
-
* both relays, which is not this case — but "not this case" is a prediction, and the standing
|
|
240
|
-
* receiver is the thing that makes an agent reachable, so it does not wait on one.
|
|
241
|
-
*
|
|
242
|
-
* An empty `grantedAddrs` yields the plain TCP floor, exactly as before: reachable by peers
|
|
243
|
-
* that can dial directly, and loud about it (`session.standing_receiver.reservation.none`).
|
|
55
|
+
* Relay peers are allowed OUTBOUND before the node starts. Unchanged and load-bearing: our own
|
|
56
|
+
* gater would otherwise refuse our own dial, and refreshes must keep working after
|
|
57
|
+
* `setAllowedPeer()` narrows the inbound gate to a session counterparty (DOD-M15-ASSIGN-1).
|
|
58
|
+
* The gater still admits NOBODY inbound here.
|
|
244
59
|
*/
|
|
60
|
+
for (const addr of candidateCircuitAddrs) {
|
|
61
|
+
const relayPeerId = relayPeerIdOf(addr);
|
|
62
|
+
if (relayPeerId)
|
|
63
|
+
gater.setAllowedOutboundPeer(relayPeerId);
|
|
64
|
+
}
|
|
245
65
|
const node = await this.createAgentNode(agentName, {
|
|
246
66
|
sessionId,
|
|
247
67
|
connectionGater: gater,
|
|
248
68
|
nodeType: "standing_receiver",
|
|
249
|
-
|
|
69
|
+
// NO `circuitRelayListenAddrs` — nothing is asked for at start. See the note above.
|
|
250
70
|
transportPrivateKey: receiverSeed,
|
|
251
71
|
});
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
72
|
+
await node.start();
|
|
73
|
+
/**
|
|
74
|
+
* ⚠️ **THE WALK IS GONE — 055-ONDEMAND, and this is the capacity change itself.**
|
|
75
|
+
*
|
|
76
|
+
* The receiver used to visit every relay it had ever heard of and hold a slot on each, for the
|
|
77
|
+
* life of the login, against the chance that somebody called. Demand was `agents × relays`, so
|
|
78
|
+
* the tenth relay added a tenth of the fleet's demand and one relay's worth of capacity, and
|
|
79
|
+
* the ratio `agents / slots-per-relay` never improved however many relays were run.
|
|
80
|
+
*
|
|
81
|
+
* **An idle agent now holds ZERO.** A reservation is taken when an offer arrives, on the relay
|
|
82
|
+
* the directory names (`takeReservationForSession`), and given back at the seal. Demand becomes
|
|
83
|
+
* `live sessions × 1`.
|
|
84
|
+
*
|
|
85
|
+
* **What makes that safe, and it is the reason this is not a reachability regression:**
|
|
86
|
+
* - The relay's MAILBOX consults no reservation. Deposit authenticates the depositor by their
|
|
87
|
+
* Noise peer id; pull is a signature challenge on the recipient and is an OUTBOUND dial. So
|
|
88
|
+
* store-and-forward keeps working with nothing held.
|
|
89
|
+
* - The WITNESS is a separate dial on the relay's own protocol. Nothing about the seal, the
|
|
90
|
+
* hash chain or the transcript depended on a reservation.
|
|
91
|
+
* - A cold call to a LOGGED-OUT agent was already refused with nothing queued, so no
|
|
92
|
+
* capability is lost — there was never an answering machine to lose.
|
|
93
|
+
*
|
|
94
|
+
* The candidate list is still passed in and still used: it is what the offer path reserves
|
|
95
|
+
* against when it needs to, and what `cello_status` reports as relays this agent could use.
|
|
96
|
+
*/
|
|
97
|
+
this.#ctx.logger.info("session.standing_receiver.idle", {
|
|
98
|
+
agentName,
|
|
99
|
+
relaysAvailable: candidateCircuitAddrs.length,
|
|
100
|
+
correlationId,
|
|
101
|
+
impact: "this agent holds no relay reservation while idle, by design. One is taken on the " +
|
|
102
|
+
"relay the directory names when a session is offered, and given back at the seal.",
|
|
103
|
+
});
|
|
104
|
+
return { node, seed: receiverSeed };
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* 054-SRSPLIT — **TAKE ONE RESERVATION, ON A NODE THAT IS ALREADY RUNNING.**
|
|
108
|
+
*
|
|
109
|
+
* The whole relay handshake for one relay: prove over `/cello/relay/1.0.0` on a connection we
|
|
110
|
+
* open, then ask libp2p's own transport manager to listen on the circuit. The relay grants on the
|
|
111
|
+
* first ask because it marks `slot.provenForReservation` per CONNECTION at auth time and libp2p's
|
|
112
|
+
* reservation store reuses the open connection rather than dialling a new one (measured live
|
|
113
|
+
* 2026-09-08; `DOD-M15-RELAYPROVE-ORDER-1`).
|
|
114
|
+
*
|
|
115
|
+
* ⚠️ **THE NODE IS NOT STOPPED BETWEEN THE PROOF AND THE ASK.** The relay marks the CONNECTION,
|
|
116
|
+
* not the peer id, so closing it throws away the very thing that makes the ask succeed.
|
|
117
|
+
*/
|
|
118
|
+
/**
|
|
119
|
+
* 055-ONDEMAND — **take a reservation for a SESSION, on the relay the directory named.**
|
|
120
|
+
*
|
|
121
|
+
* The public face of `#takeReservation` for the offer path. An idle agent holds nothing, so this
|
|
122
|
+
* is what makes it dialable, and it lasts only as long as the session that asked for it.
|
|
123
|
+
*
|
|
124
|
+
* Returns whether one was granted. A `false` is a degradation, not a failure: the counterparty
|
|
125
|
+
* can still reach this agent directly, or through the relay's store-and-forward. The one caller
|
|
126
|
+
* that must treat it as fatal is relay-only mode, which does so at its own guard.
|
|
127
|
+
*/
|
|
128
|
+
/**
|
|
129
|
+
* 055-ONDEMAND — the abandoned-offer timers, keyed `agent::sessionIdHex`. See `#armOfferRelease`.
|
|
130
|
+
*/
|
|
131
|
+
#offerReleaseTimers = new Map();
|
|
132
|
+
/**
|
|
133
|
+
* ⚠️ **AN OFFER THAT GOES QUIET MUST NOT KEEP A SLOT — `DOD-M15-OFFER-EXPIRY-1`, RELOCATED.**
|
|
134
|
+
*
|
|
135
|
+
* The story predicted this exactly: the permanently-open door units 2 and 3 removed does not take
|
|
136
|
+
* the defect with it, it moves it somewhere more expensive. The responder reserves the moment an
|
|
137
|
+
* offer arrives; an initiator that never dials — it aborted, the directory faulted — would leave a
|
|
138
|
+
* slot held on a SHARED relay until its TTL, two hours by default. That is a cost on the exact
|
|
139
|
+
* resource this whole story exists to conserve, and "released at seal" cannot cover it because
|
|
140
|
+
* there is no seal.
|
|
141
|
+
*
|
|
142
|
+
* ⚠️ **THE BUDGET IS NOT THE DIRECTORY'S 2-SECOND ACCEPT CLOCK, and picking that would be worse
|
|
143
|
+
* than not doing this at all.** The accept is only the start: the assignment still has to be
|
|
144
|
+
* FROST-signed by a threshold of directory nodes and delivered to both parties before either
|
|
145
|
+
* builds a session. Releasing on 2 s would take the slot out from under a session that was about
|
|
146
|
+
* to begin — turning a rare abandoned offer into a common broken one.
|
|
147
|
+
*
|
|
148
|
+
* No cancellation plumbing: the timer ASKS whether the session started. A session that began and
|
|
149
|
+
* has already sealed is also not live, and that case has already released through the seal path,
|
|
150
|
+
* where the teardown releases from the session's own node.
|
|
151
|
+
*/
|
|
152
|
+
#armOfferRelease(agentName, sessionIdHex, circuitAddr, correlationId) {
|
|
153
|
+
const key = `${agentName}::${sessionIdHex}`;
|
|
154
|
+
const existing = this.#offerReleaseTimers.get(key);
|
|
155
|
+
if (existing !== undefined)
|
|
156
|
+
clearTimeout(existing);
|
|
157
|
+
const timer = setTimeout(() => {
|
|
158
|
+
this.#offerReleaseTimers.delete(key);
|
|
159
|
+
if (this.#ctx.shuttingDown())
|
|
160
|
+
return;
|
|
161
|
+
if (this.#ctx.sessionIsLive(agentName, sessionIdHex))
|
|
162
|
+
return; // it started; the seal owns it now
|
|
163
|
+
this.#ctx.logger.warn("session.reservation.offer_abandoned", {
|
|
164
|
+
agentName,
|
|
165
|
+
sessionIdHex,
|
|
166
|
+
circuitAddr,
|
|
167
|
+
heldForMs: OFFER_RESERVATION_GRACE_MS,
|
|
168
|
+
correlationId,
|
|
169
|
+
impact: "a relay slot was taken to answer an offer that never became a session — the " +
|
|
170
|
+
"initiator did not dial. Given back rather than held until the relay's TTL, which is " +
|
|
171
|
+
"two hours and is a cost on every other agent that relay serves.",
|
|
172
|
+
});
|
|
173
|
+
/**
|
|
174
|
+
* ⚠️ THE RECEIVER'S OWN NODE, and at this point that is the right one: the offer never became a
|
|
175
|
+
* session, so nothing was promoted and the circuit is still on the standing receiver. (Had it
|
|
176
|
+
* been promoted, `sessionIsLive` above would have returned and we would not be here.)
|
|
177
|
+
*/
|
|
178
|
+
const sr = this.#ctx.standingReceivers.get(agentName);
|
|
179
|
+
if (sr) {
|
|
180
|
+
void this.releaseSessionReservation(agentName, sr.node, sessionIdHex, correlationId)
|
|
181
|
+
.catch(() => { });
|
|
182
|
+
}
|
|
183
|
+
}, OFFER_RESERVATION_GRACE_MS);
|
|
184
|
+
timer.unref?.();
|
|
185
|
+
this.#offerReleaseTimers.set(key, timer);
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* 055-ONDEMAND — re-take a circuit on a node that already exists, for a live session that lost it.
|
|
189
|
+
*
|
|
190
|
+
* ⚠️ **IT PROVES OVER THE DELIVERY PATH TOO.** `DOD-M15-RELAYAUTH-1` applies identically here: a
|
|
191
|
+
* holder that has not proven key possession has its reservation revoked inside the relay's grace
|
|
192
|
+
* window, about fifteen seconds. The first version of the re-take called the low-level ask
|
|
193
|
+
* directly and skipped it, so a re-taken circuit would have been revoked almost immediately while
|
|
194
|
+
* the log said "granted" — the same defect this unit already fixed one function up.
|
|
195
|
+
*/
|
|
196
|
+
async retakeReservationOn(agentName, node, circuitAddr, correlationId) {
|
|
197
|
+
const relayPeerId = relayPeerIdOf(circuitAddr);
|
|
198
|
+
const outcome = await this.#takeReservation(agentName, node, circuitAddr, correlationId);
|
|
199
|
+
if (outcome !== "granted")
|
|
200
|
+
return false;
|
|
201
|
+
/**
|
|
202
|
+
* ⚠️ **THE OTHER HALF OF `reservation_lost` — 056-SLOTDEAD, review F4.**
|
|
203
|
+
*
|
|
204
|
+
* The loss trigger fires the moment a reservation goes, which is exactly when the relay link is
|
|
205
|
+
* down, so the pull it starts is aimed at a relay that cannot answer. Without something on the
|
|
206
|
+
* recovery, content the counterparty parked during the outage waits for the slow periodic
|
|
207
|
+
* backstop while the relay is healthy and this agent is connected to it again.
|
|
208
|
+
*
|
|
209
|
+
* **This path, and not the two that look like it.** The watchdog's `gained` branch was the
|
|
210
|
+
* first attempt and never fired — a take records the new circuit on the receiver itself, so by
|
|
211
|
+
* the next tick there is nothing left for the watchdog to see as gained; a test caught it, which
|
|
212
|
+
* is the only reason it is not still in the tree looking correct. `takeReservationForSession`
|
|
213
|
+
* was the second, and it double-drains: on a first login it fires moments after the install
|
|
214
|
+
* drain, for the same empty mailbox.
|
|
215
|
+
*
|
|
216
|
+
* A re-take is unambiguous. It happens only when a session that HAD a circuit lost it, which is
|
|
217
|
+
* exactly the outage whose recovery this is.
|
|
218
|
+
*/
|
|
219
|
+
this.#ctx.park.fireParkedDrain(agentName, "reservation_regained");
|
|
220
|
+
if (relayPeerId) {
|
|
221
|
+
void this.#ctx.authenticateStandingReceiver(agentName, node, relayPeerId, circuitAddr, correlationId)
|
|
222
|
+
.catch((err) => {
|
|
223
|
+
this.#ctx.logger.warn("session.standing_receiver.relay_auth.failed", {
|
|
224
|
+
agentName, relayPeerId, correlationId,
|
|
225
|
+
error: extractErrorMessage(err),
|
|
226
|
+
impact: "a live session re-took a circuit but did not prove key possession over the " +
|
|
227
|
+
"delivery path, so the relay may revoke it inside its grace window.",
|
|
228
|
+
});
|
|
229
|
+
});
|
|
255
230
|
}
|
|
231
|
+
return true;
|
|
232
|
+
}
|
|
233
|
+
async takeReservationForSession(agentName, circuitAddr, correlationId,
|
|
234
|
+
/**
|
|
235
|
+
* ⚠️ **PRESENT ONLY ON THE OFFER PATH, AND THAT IS WHAT ARMS THE ABANDONED-OFFER RELEASE.**
|
|
236
|
+
*
|
|
237
|
+
* The first version reused `correlationId` for this, because the offer handler happens to pass
|
|
238
|
+
* the session id as its correlation id. Two meanings in one parameter is how a value ends up
|
|
239
|
+
* being trusted for something it was never chosen for: the watchdog's re-take passes a genuine
|
|
240
|
+
* correlation id, and it would have armed a timer keyed on a string that is not a session.
|
|
241
|
+
*
|
|
242
|
+
* Absent for the re-take path, which is by definition for a session that is already live.
|
|
243
|
+
*/
|
|
244
|
+
offerSessionIdHex) {
|
|
245
|
+
const sr = this.#ctx.standingReceivers.get(agentName);
|
|
246
|
+
if (!sr) {
|
|
247
|
+
this.#ctx.logger.warn("session.reservation.on_demand.no_receiver", {
|
|
248
|
+
agentName,
|
|
249
|
+
circuitAddr,
|
|
250
|
+
correlationId,
|
|
251
|
+
impact: "there is no standing receiver to hold a reservation, so this agent cannot be " +
|
|
252
|
+
"dialled for this session; it is reachable through the relay's store-and-forward only.",
|
|
253
|
+
});
|
|
254
|
+
return false;
|
|
255
|
+
}
|
|
256
|
+
// The relay must be dialable BEFORE we dial it — our own gater refuses otherwise, which is the
|
|
257
|
+
// same ordering the login walk uses and the one that cost a whole debugging session when it
|
|
258
|
+
// was missing.
|
|
259
|
+
const relayPeerId = relayPeerIdOf(circuitAddr);
|
|
260
|
+
if (relayPeerId)
|
|
261
|
+
sr.gater.setAllowedOutboundPeer(relayPeerId);
|
|
262
|
+
const outcome = await this.#takeReservation(agentName, sr.node, circuitAddr, correlationId);
|
|
256
263
|
/**
|
|
257
|
-
* ⚠️
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
*
|
|
264
|
+
* ⚠️ **THE RECEIVER'S RECORD OF WHAT IT HOLDS MUST FOLLOW, or two things go quietly wrong.**
|
|
265
|
+
* `relayPeerIds` is what the reservation watchdog compares against to decide a reservation was
|
|
266
|
+
* LOST, and what `cello_status` reports as reachability. Left at its build-time value — empty,
|
|
267
|
+
* now that nothing is taken at login — the watchdog would see a held circuit it never recorded
|
|
268
|
+
* and `cello_status` would call a reachable agent unreachable.
|
|
262
269
|
*
|
|
263
|
-
*
|
|
264
|
-
*
|
|
265
|
-
* operator was told the receiver "did not finish binding every circuit inside the deadline" and
|
|
266
|
-
* "is reachable through those" — sending them to the relay fleet for a port held by an orphan
|
|
267
|
-
* daemon on their own machine. The rejection is rethrown so it reaches
|
|
268
|
-
* `session.node.create.failed` with its own cause, exactly as it does on the no-relay path.
|
|
270
|
+
* Read from the NODE rather than appended to, and deduped by relay: libp2p announces one
|
|
271
|
+
* address per relay listen address, so a five-address relay would otherwise count five times.
|
|
269
272
|
*/
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
const started = node.start().then(() => "ok", (err) => { startError = err; return "failed"; });
|
|
273
|
-
const outcome = await Promise.race([
|
|
274
|
-
started,
|
|
275
|
-
new Promise((resolve) => {
|
|
276
|
-
// Per granted relay: each circuit listener is its own dial and its own reservation, so a
|
|
277
|
-
// pool of three must not be judged on a budget sized for one.
|
|
278
|
-
deadline = setTimeout(() => resolve("slow"), this.#ctx.srReservationTimeoutMs * grantedAddrs.length);
|
|
279
|
-
}),
|
|
280
|
-
]);
|
|
281
|
-
if (deadline !== undefined)
|
|
282
|
-
clearTimeout(deadline);
|
|
283
|
-
if (outcome === "failed")
|
|
284
|
-
throw startError;
|
|
273
|
+
const heldNow = heldRelayIdsOf(sr.node);
|
|
274
|
+
this.#ctx.standingReceivers.set(agentName, { ...sr, relayPeerIds: heldNow });
|
|
285
275
|
/**
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
276
|
+
* ⚠️ **A RELAY THAT GRANTED MUST BE ADMITTED INBOUND, OR THE RESERVATION BUYS NOTHING.**
|
|
277
|
+
*
|
|
278
|
+
* The gater's inbound carve-out is the security-sensitive half of a reservation: only relays
|
|
279
|
+
* whose own grant is confirmed earn it, so a directory that merely NAMES a relay cannot dial us
|
|
280
|
+
* through it. That set was built once from what the login walk held — empty now — so an
|
|
281
|
+
* on-demand reservation would have been taken, announced, and then refused by OUR OWN gater
|
|
282
|
+
* when the counterparty dialled through it.
|
|
283
|
+
*
|
|
284
|
+
* Recomputed from what the node HOLDS, never from what was asked: being named by the directory
|
|
285
|
+
* must not buy a foothold, and under on-demand the directory is what names the relay.
|
|
286
|
+
*/
|
|
287
|
+
sr.gater.setReservedRelayPeers(heldNow);
|
|
288
|
+
/**
|
|
289
|
+
* ⚠️ **DOD-M15-RELAYAUTH-1 STILL APPLIES, AND IT NO LONGER FIRES ON ITS OWN.**
|
|
290
|
+
*
|
|
291
|
+
* That auth is a SECOND proof, over the DELIVERY path, and it is what keeps the relay from
|
|
292
|
+
* revoking a reservation whose holder has not proven key possession to it — the grace window is
|
|
293
|
+
* about fifteen seconds. It used to run in the receiver build, over the circuits the login walk
|
|
294
|
+
* had just collected. Nothing is collected there any more, so without this line an on-demand
|
|
295
|
+
* reservation would be taken and then quietly revoked mid-session, and the agent would go
|
|
296
|
+
* unreachable while every log said the reservation was granted.
|
|
289
297
|
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
-
* `PROVEN_PEER_MEMORY_MS = 2 minutes` (`relay-connection-gater.ts`, trustless-cello). The walk
|
|
293
|
-
* costs up to `#srReservationTimeoutMs` × 2 attempts per relay, so a pool of three at the
|
|
294
|
-
* 15s default can spend 90 seconds before the final node asks relay 1 again. The earliest
|
|
295
|
-
* proof can expire before it is used, and that is what this event catches.
|
|
298
|
+
* Best-effort and unawaited, exactly as it is in the build path: a failure here costs the
|
|
299
|
+
* relay's own grace-window revoke, which the watchdog already treats as an ordinary loss.
|
|
296
300
|
*/
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
301
|
+
if (outcome === "granted" && offerSessionIdHex !== undefined) {
|
|
302
|
+
this.#armOfferRelease(agentName, offerSessionIdHex, circuitAddr, correlationId);
|
|
303
|
+
}
|
|
304
|
+
if (outcome === "granted" && relayPeerId) {
|
|
305
|
+
void this.#ctx.authenticateStandingReceiver(agentName, sr.node, relayPeerId, circuitAddr, correlationId)
|
|
306
|
+
.catch((err) => {
|
|
307
|
+
this.#ctx.logger.warn("session.standing_receiver.relay_auth.failed", {
|
|
308
|
+
agentName, relayPeerId, correlationId,
|
|
309
|
+
error: extractErrorMessage(err),
|
|
310
|
+
impact: "this agent has a reservation the relay may revoke within its grace window, " +
|
|
311
|
+
"because key possession was not proven over the delivery path.",
|
|
312
|
+
});
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
return outcome === "granted";
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* 055-ONDEMAND — **GIVE THIS SESSION'S SLOT BACK, FROM THE NODE THAT ACTUALLY HOLDS IT.**
|
|
319
|
+
*
|
|
320
|
+
* ⚠️ **THE FIRST VERSION LOOKED THE NODE UP BY AGENT NAME AND ALWAYS FOUND THE WRONG ONE.** When a
|
|
321
|
+
* session opens, the standing receiver is PROMOTED: `session-lifecycle` deletes it from
|
|
322
|
+
* `standingReceivers`, moves that exact node into `activeNodes`, and builds a fresh empty receiver
|
|
323
|
+
* behind it. So by seal time `standingReceivers.get(agentName)` is the new idle node, holding
|
|
324
|
+
* nothing — the release read `[]`, returned immediately, and told the relay nothing. The slot the
|
|
325
|
+
* offer took lived on the promoted node and died with it, invisible to the relay, held for the
|
|
326
|
+
* full TTL. **A release that always no-ops is worse than none: it logs success.**
|
|
327
|
+
*
|
|
328
|
+
* ⚠️ **AND THE CROSS-SESSION RECOMPUTE THE ORDER ASKED FOR IS UNNECESSARY, which is the good news
|
|
329
|
+
* in the correction.** libp2p's shared `reservationStore` is shared *within one node*. Each live
|
|
330
|
+
* session owns its OWN node, so sealing one cannot clear another session's refresh timers. There
|
|
331
|
+
* is nothing to recompute: this node is being torn down anyway, so tell its relays and drop its
|
|
332
|
+
* circuits. Re-deriving that removed a whole class of drift rather than managing it.
|
|
333
|
+
*/
|
|
334
|
+
async releaseSessionReservation(agentName, node, sessionId, correlationId) {
|
|
335
|
+
const held = heldRelayIdsOf(node);
|
|
336
|
+
if (held.length === 0)
|
|
337
|
+
return;
|
|
338
|
+
for (const relayPeerId of held) {
|
|
339
|
+
/**
|
|
340
|
+
* ⚠️ BOUNDED, BECAUSE THIS RUNS INSIDE A TEARDOWN. Telling a relay means dialling it, and an
|
|
341
|
+
* unreachable one would otherwise hold the seal open for as long as its dial takes. Measured
|
|
342
|
+
* the hard way: unbounded, this hung fourteen unrelated suites at 237s each. A seal waiting on
|
|
343
|
+
* a courtesy is worse than a slot held until its TTL.
|
|
344
|
+
*/
|
|
345
|
+
await Promise.race([
|
|
346
|
+
this.#ctx.tellRelayReleased(agentName, relayPeerId, node, correlationId),
|
|
347
|
+
new Promise((r) => setTimeout(r, RELEASE_TELL_BUDGET_MS).unref?.()),
|
|
348
|
+
]);
|
|
349
|
+
}
|
|
350
|
+
// Local half. The node is going away regardless; this stops it announcing a route it no longer
|
|
351
|
+
// holds for whatever is left of its life.
|
|
352
|
+
try {
|
|
353
|
+
await node.releaseAllCircuits();
|
|
354
|
+
}
|
|
355
|
+
catch (err) {
|
|
356
|
+
this.#ctx.logger.debug("session.reservation.release.local_failed", {
|
|
357
|
+
agentName, sessionId, correlationId, error: extractErrorMessage(err),
|
|
358
|
+
// The node is being torn down either way, so this costs nothing beyond a few more seconds
|
|
359
|
+
// of announcing a route that is already gone at the relay. Deliberately NOT claiming the
|
|
360
|
+
// watchdog repairs it (review MEDIUM-11): this unit removed the rebuild that used to.
|
|
361
|
+
impact: "the session's node keeps announcing a circuit it has released, until it stops.",
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
this.#ctx.logger.info("session.reservation.released", {
|
|
365
|
+
agentName,
|
|
366
|
+
sessionId,
|
|
367
|
+
releasedRelays: held,
|
|
368
|
+
correlationId,
|
|
369
|
+
impact: "the slot this session borrowed is back in the relay's table, rather than held until " +
|
|
370
|
+
"its two-hour TTL.",
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
async #takeReservation(agentName, node, circuitAddr, correlationId) {
|
|
374
|
+
const verdict = await this.#ctx.proveToRelay(agentName, circuitAddr, node, correlationId, true);
|
|
375
|
+
if (verdict === "refused_this_agent" || verdict === "refused_try_another_relay") {
|
|
376
|
+
const reason = verdict === "refused_this_agent" ? "relay_refused_this_agent" : "relay_proof_refused";
|
|
377
|
+
this.#ctx.srLastRejectionReason.set(agentName, reason);
|
|
378
|
+
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
303
379
|
agentName,
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
reservationsHeld: boundRelays.size,
|
|
307
|
-
walkMs: Date.now() - walkStartedAt,
|
|
380
|
+
circuitAddr,
|
|
381
|
+
reason,
|
|
308
382
|
correlationId,
|
|
309
|
-
impact:
|
|
310
|
-
"
|
|
311
|
-
|
|
312
|
-
"
|
|
313
|
-
"not the relay fleet.",
|
|
383
|
+
impact: verdict === "refused_this_agent"
|
|
384
|
+
? "the relay refused this AGENT rather than being unwilling or unwell, so every other " +
|
|
385
|
+
"relay would refuse it identically. Stopped here; cello_status carries the cause."
|
|
386
|
+
: "this relay would not take the agent's proof. Moving to the next relay.",
|
|
314
387
|
});
|
|
388
|
+
return verdict === "refused_this_agent" ? "refused_this_agent" : "declined";
|
|
315
389
|
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
390
|
+
/**
|
|
391
|
+
* ⚠️ **A PROOF THAT REACHED NO VERDICT IS NOT A REFUSAL** — `unavailable` means the relay never
|
|
392
|
+
* answered (no client wired, or unreachable), and NOT every relay gates reservations. Refusing
|
|
393
|
+
* to ask because our own proof path was unavailable would lose the ability to reserve with an
|
|
394
|
+
* ungated relay entirely. The boundary is enforced in `proveToRelay`, which returns
|
|
395
|
+
* `unavailable` for exactly this (review HIGH-1 on unit 1).
|
|
396
|
+
*/
|
|
397
|
+
if (verdict === "unavailable") {
|
|
398
|
+
this.#ctx.logger.warn("session.standing_receiver.prove.no_verdict", {
|
|
321
399
|
agentName,
|
|
322
|
-
|
|
323
|
-
circuitAddrs: node.listenAddresses().filter((a) => a.includes("/p2p-circuit")).length,
|
|
324
|
-
budgetMs: this.#ctx.srReservationTimeoutMs * grantedAddrs.length,
|
|
400
|
+
circuitAddr,
|
|
325
401
|
correlationId,
|
|
326
|
-
impact: "
|
|
327
|
-
"
|
|
328
|
-
"
|
|
402
|
+
impact: "no proof verdict was obtained from this relay. Asking for the reservation anyway: " +
|
|
403
|
+
"a relay that does not gate them grants it, and one that does refuses an ask that cost a " +
|
|
404
|
+
"single dial.",
|
|
329
405
|
});
|
|
330
406
|
}
|
|
331
|
-
|
|
407
|
+
let askFault;
|
|
408
|
+
let error = "";
|
|
409
|
+
let timer;
|
|
410
|
+
const timedOut = Symbol("listen_timeout");
|
|
411
|
+
let outcome = timedOut;
|
|
412
|
+
try {
|
|
413
|
+
/**
|
|
414
|
+
* ⚠️ **AN ABANDONED ASK NOW LANDS ON A NODE THAT LIVES ON** — review MEDIUM-5, and it is the
|
|
415
|
+
* mirror image of the hazard the old probe teardown existed for.
|
|
416
|
+
*
|
|
417
|
+
* When a probe timed out it was destroyed, so a grant arriving late died with it. There is
|
|
418
|
+
* one long-lived node now: a late grant ADDS a circuit address after the walk has counted
|
|
419
|
+
* what it holds, so the receiver would advertise a relay that is in neither `sr.relayPeerIds`
|
|
420
|
+
* nor the gater's reserved set — the ledger and the advertised addresses disagreeing, which
|
|
421
|
+
* is what the watchdog then churns on.
|
|
422
|
+
*
|
|
423
|
+
* So a late grant is GIVEN BACK rather than kept. Releasing costs a dial; keeping it costs a
|
|
424
|
+
* disagreement no operator can see, and a slot on a relay we already decided against.
|
|
425
|
+
*/
|
|
426
|
+
const askP = node.listenOnCircuit(circuitAddr).then(() => "asked");
|
|
427
|
+
void askP.catch(() => { });
|
|
428
|
+
outcome = await Promise.race([
|
|
429
|
+
askP,
|
|
430
|
+
new Promise((resolve) => {
|
|
431
|
+
timer = setTimeout(() => resolve(timedOut), this.#ctx.srReservationTimeoutMs);
|
|
432
|
+
}),
|
|
433
|
+
]);
|
|
434
|
+
if (outcome === timedOut) {
|
|
435
|
+
void askP.then(() => {
|
|
436
|
+
this.#ctx.logger.warn("session.standing_receiver.reservation.late_grant_released", {
|
|
437
|
+
agentName,
|
|
438
|
+
circuitAddr,
|
|
439
|
+
correlationId,
|
|
440
|
+
impact: "this relay answered after the walk had moved on, so its circuit was not " +
|
|
441
|
+
"counted or advertised. Given back rather than held: a slot nobody knows about is " +
|
|
442
|
+
"one the relay cannot reuse and this agent cannot rely on. ⚠️ This drops EVERY " +
|
|
443
|
+
"circuit — libp2p's reservation store is shared across listeners and cannot " +
|
|
444
|
+
"release one — so the receiver is rebuilt by the watchdog, which is the correct " +
|
|
445
|
+
"outcome: a walk whose result is already wrong should be redone, not patched.",
|
|
446
|
+
});
|
|
447
|
+
return node.releaseAllCircuits().catch(() => false);
|
|
448
|
+
}, () => { });
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
error = extractErrorMessage(err);
|
|
453
|
+
// A fault of OURS keeps its own name rather than being re-derived from the relay connection,
|
|
454
|
+
// which is intact and irrelevant when the ask never left this process (review HIGH-2).
|
|
455
|
+
askFault = clientSideAskFault(err);
|
|
456
|
+
}
|
|
457
|
+
finally {
|
|
458
|
+
if (timer !== undefined)
|
|
459
|
+
clearTimeout(timer);
|
|
460
|
+
}
|
|
461
|
+
// The only proof that counts: an ANNOUNCED circuit address on this relay. `listen()` resolving
|
|
462
|
+
// is not enough — a relay at its slot cap completes the handshake and grants nothing.
|
|
463
|
+
const relayPeerId = relayPeerIdOf(circuitAddr);
|
|
464
|
+
if (outcome === "asked" && node.listenAddresses().some((a) => a.split("/").includes("p2p-circuit") && (relayPeerId === null || a.includes(`/p2p/${relayPeerId}/`)))) {
|
|
465
|
+
return "granted";
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* ⚠️ **THE CONNECTION CHECK APPLIES TO BOTH OUTCOMES** — review MEDIUM-6.
|
|
469
|
+
*
|
|
470
|
+
* "Asked" does not mean the relay answered: a relay that is down yields no circuit address,
|
|
471
|
+
* indistinguishable here from one that answered and granted nothing. An open connection is what
|
|
472
|
+
* separates them, and we have one to ask.
|
|
473
|
+
*
|
|
474
|
+
* The first version of this rewrite applied that check only to the `asked` branch, so a HUNG
|
|
475
|
+
* ask short-circuited to `reservation_did_not_complete_in_time` — latency — when the relay was
|
|
476
|
+
* simply gone. That is a name for where the failure surfaced, not for what went wrong, in a
|
|
477
|
+
* unit whose headline is that behaviour does not change.
|
|
478
|
+
*/
|
|
479
|
+
const connectedToRelay = node.getConnections().some((c) => c.peerId === relayPeerId);
|
|
480
|
+
const reason = askFault !== undefined
|
|
481
|
+
? askFault
|
|
482
|
+
: !connectedToRelay
|
|
483
|
+
? "relay_unreachable"
|
|
484
|
+
: outcome === "asked"
|
|
485
|
+
? "relay_granted_no_reservation"
|
|
486
|
+
: "reservation_did_not_complete_in_time";
|
|
487
|
+
this.#ctx.srLastRejectionReason.set(agentName, reason);
|
|
488
|
+
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
489
|
+
agentName,
|
|
490
|
+
circuitAddr,
|
|
491
|
+
reason,
|
|
492
|
+
...(error !== "" ? { error } : {}),
|
|
493
|
+
correlationId,
|
|
494
|
+
});
|
|
495
|
+
return "declined";
|
|
332
496
|
}
|
|
333
497
|
/** One standing-receiver create attempt (extracted for the M8B F14 retry loop). */
|
|
334
498
|
async #tryCreateStandingReceiver(agentName, correlationId) {
|
|
@@ -413,43 +577,28 @@ export class StandingReceivers {
|
|
|
413
577
|
});
|
|
414
578
|
autoNat.emitInitialResult();
|
|
415
579
|
/**
|
|
416
|
-
*
|
|
417
|
-
*
|
|
580
|
+
* ⚠️ **A RECEIVER IS INSTALLED HOLDING NOTHING, AND THAT IS NOW A CONSTANT — 056-SLOTDEAD.**
|
|
581
|
+
*
|
|
582
|
+
* This block used to derive what the node had come up holding: `heldRelayIdsOf(node)`, the
|
|
583
|
+
* matching circuit addresses, the count for the reachability line, and the gater's inbound
|
|
584
|
+
* carve-out. Every one of those read the result of the login walk. `#startReceiverNode` returns
|
|
585
|
+
* a TCP-only node now (055-ONDEMAND), so all four were computing `[]` — reachable, referenced,
|
|
586
|
+
* and unable to produce a different answer on any input.
|
|
418
587
|
*
|
|
419
|
-
* The
|
|
420
|
-
*
|
|
421
|
-
*
|
|
422
|
-
*
|
|
423
|
-
* absent on every tick forever, and it rebuilt on the 30-second grid — churning the very
|
|
424
|
-
* reservations this unit exists to conserve. A candidate is a relay we ASKED; only a held
|
|
425
|
-
* address is a relay that ANSWERED, and the fallback conflated the two.
|
|
588
|
+
* The distinction the deleted comment was defending — a CANDIDATE is a relay we asked, a HELD
|
|
589
|
+
* address is one that answered — is still the rule, and it still lives in `heldRelayIdsOf`. It
|
|
590
|
+
* is enforced where a reservation is actually taken (`takeReservationForSession`) and where one
|
|
591
|
+
* is checked (the watchdog), which is where it belongs. Nothing is held here to check.
|
|
426
592
|
*
|
|
427
|
-
* The
|
|
428
|
-
*
|
|
429
|
-
* `/p2p/<id>/p2p-circuit` form, that address yields no id and is not counted as held — so the
|
|
430
|
-
* receiver reads as degraded and gets rebuilt, instead of reading as healthy against a relay
|
|
431
|
-
* nobody is connected to. Degrading toward "rebuild" is the safe direction; the other one is
|
|
432
|
-
* the silent unreachability this whole file exists to kill.
|
|
593
|
+
* The gater is left as constructed: its reserved set starts empty, and `takeReservationForSession`
|
|
594
|
+
* widens it for the one relay that grants. Setting it to `[]` here only restated that.
|
|
433
595
|
*/
|
|
434
|
-
const heldRelayPeerIds = heldRelayIdsOf(node);
|
|
435
|
-
const circuitAddrs = heldRelayPeerIds.length;
|
|
436
|
-
const heldCircuitAddrs = node.listenAddresses().filter((a) => a.includes("/p2p-circuit"));
|
|
437
|
-
// DOD-M15-ASSIGN-1 review N3, widened by 032-RELAYSPREAD: the relays this receiver actually
|
|
438
|
-
// reserved with earn the inbound AutoNAT carve-out — nothing else does. Populated only from
|
|
439
|
-
// reservations that genuinely completed, so a directory that merely NAMES a relay cannot dial
|
|
440
|
-
// in behind it, however many relays it names.
|
|
441
|
-
gater.setReservedRelayPeers(heldRelayPeerIds);
|
|
442
|
-
// The re-spread clock starts HERE, at the build, not at the epoch. Otherwise the first decay
|
|
443
|
-
// re-spreads instantly — undoing the "a lost relay does not rebuild the receiver" rule seconds
|
|
444
|
-
// after it fires, and changing the peer id of an agent that just lost one relay of three. The
|
|
445
|
-
// ratchet this guards against runs over hours; nothing about it needs answering in a second.
|
|
446
|
-
this.#ctx.srLastRespreadAt.set(agentName, Date.now());
|
|
447
596
|
this.#ctx.standingReceivers.set(agentName, {
|
|
448
597
|
node,
|
|
449
598
|
gater,
|
|
450
599
|
autoNat,
|
|
451
600
|
seed,
|
|
452
|
-
relayPeerIds:
|
|
601
|
+
relayPeerIds: [],
|
|
453
602
|
});
|
|
454
603
|
this.#ctx.logger.info("session.node.created", {
|
|
455
604
|
sessionId,
|
|
@@ -457,69 +606,74 @@ export class StandingReceivers {
|
|
|
457
606
|
sessionPeerId: node.getPeerId(),
|
|
458
607
|
correlationId,
|
|
459
608
|
});
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
// reservation with every relay it knows" read as true in an audit: the outcome was one and the
|
|
493
|
-
// request was one too, and a single field could report neither.
|
|
494
|
-
// relaysOffered — how many relays were in the candidate list (deduped by relay peer id in
|
|
495
|
-
// `#reservationCircuitAddrs`, so it counts relays, not addresses).
|
|
496
|
-
// reservationsHeld — how many reservations this node actually holds, counted the only way
|
|
497
|
-
// that proves a grant: ANNOUNCED /p2p-circuit listen addresses. `start()`
|
|
498
|
-
// resolving is not enough — a relay out of reservation slots completes the
|
|
499
|
-
// handshake, grants nothing, and leaves a node that looks started and is
|
|
500
|
-
// dialable by nobody.
|
|
609
|
+
/**
|
|
610
|
+
* ⚠️ **DOD-M15-RELAYAUTH-1'S INSTALL-TIME PROOF IS GONE — 056-SLOTDEAD, and it could not have
|
|
611
|
+
* run since 055-ONDEMAND.**
|
|
612
|
+
*
|
|
613
|
+
* It read: *"authenticate to the reservation relay NOW, not when a session first needs one"* —
|
|
614
|
+
* because the relay revokes a reservation whose holder has not proven key possession to it, and
|
|
615
|
+
* a login-time slot had to survive that grace window with no session in sight. It looped over
|
|
616
|
+
* the relays this receiver held. It holds none, so the loop's body never executed.
|
|
617
|
+
*
|
|
618
|
+
* **The requirement it served is not gone; it moved, and it moved to the stronger place.** A
|
|
619
|
+
* reservation is now taken by `takeReservationForSession`, which proves FIRST and asks second on
|
|
620
|
+
* the same connection (`DOD-M15-RELAYPROVE-ORDER-1`). "Proven before the slot exists" is
|
|
621
|
+
* structural there, rather than a second best-effort call racing a grace window.
|
|
622
|
+
*/
|
|
623
|
+
/**
|
|
624
|
+
* DOD-NAT-REACHABILITY-1 observability: what did this receiver come up able to use?
|
|
625
|
+
*
|
|
626
|
+
* ⚠️ **`reservationsHeld` WAS DROPPED, NOT RENAMED — 056-SLOTDEAD.** It carried
|
|
627
|
+
* `heldRelayIdsOf(node).length` and was the one number in this line an operator would act on.
|
|
628
|
+
* Since 055-ONDEMAND a receiver is installed holding nothing by design, so it reported `0` on
|
|
629
|
+
* every healthy login for every agent — a measurement that had become a constant while still
|
|
630
|
+
* reading as a measurement. That is worse than not reporting it: the number an operator trusts
|
|
631
|
+
* to mean "this agent is deaf" now means nothing at all.
|
|
632
|
+
*
|
|
633
|
+
* `relaysOffered` stays and is still a real count: how many relays are in the candidate list
|
|
634
|
+
* (deduped by relay peer id in `reservationCircuitAddrs`, so it counts relays, not addresses).
|
|
635
|
+
* **Zero of them is the condition worth seeing here** — an agent with no candidate cannot take a
|
|
636
|
+
* slot when an offer arrives, and will refuse the call.
|
|
637
|
+
*
|
|
638
|
+
* What an agent actually holds is reported where it is now decided: `session.offer.reservation`
|
|
639
|
+
* at the moment a slot is asked for, and `getStandingReceiverReachability` for `cello_status`.
|
|
640
|
+
*/
|
|
501
641
|
this.#ctx.logger.info("session.standing_receiver.reachability", {
|
|
502
642
|
agentName,
|
|
503
643
|
relaysOffered: reservations.addrs.length,
|
|
504
|
-
reservationsHeld: circuitAddrs,
|
|
505
644
|
correlationId,
|
|
506
645
|
});
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
646
|
+
/**
|
|
647
|
+
* ⚠️ **`reservation.none` IS NOT EMITTED AT BUILD ANY MORE — 055-ONDEMAND.**
|
|
648
|
+
*
|
|
649
|
+
* It means *"this agent is offered relays and holds none, so nobody behind NAT can dial it"* —
|
|
650
|
+
* a warn that fired 481 times over 17 days and drove a whole retry story. Under on-demand,
|
|
651
|
+
* holding none at build is the DESIGN: a slot is taken when an offer arrives and given back at
|
|
652
|
+
* the seal. Leaving the warn here would fire it on every healthy login for every agent, which
|
|
653
|
+
* is not a smaller version of the old problem but a worse one — an alarm that is wrong every
|
|
654
|
+
* time trains its reader to ignore the one time it is right.
|
|
655
|
+
*
|
|
656
|
+
* The condition it named still has a home. `session.offer.reservation` reports `granted: false`
|
|
657
|
+
* when an offer could not get a circuit — the moment it actually costs someone something — and
|
|
658
|
+
* the watchdog's re-take path reports a live session that lost one.
|
|
659
|
+
*
|
|
660
|
+
* ⚠️ **AND ITS DEBUG-LEVEL REPLACEMENT WENT TOO — 056-SLOTDEAD, review F10.** A
|
|
661
|
+
* `session.standing_receiver.idle_no_reservation` line survived here, guarded on
|
|
662
|
+
* `reservations.addrs.length > 0 && circuitAddrs === 0`. The second term was always true once
|
|
663
|
+
* the login walk was deleted, so the guard was really just "this agent has candidates" and the
|
|
664
|
+
* line fired on every healthy install for every agent — a second event, under a name that reads
|
|
665
|
+
* as a fault, asserting the same by-design state the `idle` line above already reports with its
|
|
666
|
+
* `relaysAvailable` count. Two lines per install saying "normal" is how a log stops being read.
|
|
667
|
+
*/
|
|
518
668
|
// DOD-PARK-DRAIN-1: this agent has a receiver again — drain whatever parked while it did not.
|
|
519
|
-
//
|
|
520
|
-
//
|
|
521
|
-
//
|
|
522
|
-
//
|
|
669
|
+
// The defect this closes was a trigger hooked to the wrong connection: content parks when the
|
|
670
|
+
// RELAY link dies, and the drain was waiting on DIRECTORY SIGNALING to reconnect — which it
|
|
671
|
+
// never had to, having never dropped.
|
|
672
|
+
//
|
|
673
|
+
// 056-SLOTDEAD: this used to be the ONE place every path converged on, because a lost
|
|
674
|
+
// reservation rebuilt the receiver and arrived back here. The rebuilds are gone, so this now
|
|
675
|
+
// covers the INSTALL only, and the loss has its own trigger (`reservation_lost`) at the point
|
|
676
|
+
// the loss is noticed.
|
|
523
677
|
this.#ctx.park.fireParkedDrain(agentName, "standing_receiver_ready");
|
|
524
678
|
return { outcome: "installed" };
|
|
525
679
|
}
|
|
@@ -576,72 +730,6 @@ export class StandingReceivers {
|
|
|
576
730
|
this.#ctx.standingReceiverCreating.delete(agentName);
|
|
577
731
|
}
|
|
578
732
|
}
|
|
579
|
-
/**
|
|
580
|
-
* Replace an agent's reservation-less standing receiver with one that reserves.
|
|
581
|
-
*
|
|
582
|
-
* Deliberately NOT removeStandingReceiverForAgent()+ensureStandingReceiverForAgent():
|
|
583
|
-
* the public remove CLEARS #agentsWantingReceiver, so a cello_set_agent_offline landing in
|
|
584
|
-
* the window while node.stop() is awaited would find no map entry and no creating
|
|
585
|
-
* marker, leave no tombstone, and the re-ensure would then RESURRECT a receiver for
|
|
586
|
-
* an agent that asked to go dark — accepting inbound sessions for an offline agent.
|
|
587
|
-
* Here the want-flag is left intact and re-checked after the stop: a concurrent stop
|
|
588
|
-
* clears it, and the rebuild correctly no-ops.
|
|
589
|
-
*/
|
|
590
|
-
async rebuildStandingReceiver(agentName) {
|
|
591
|
-
try {
|
|
592
|
-
const sr = this.#ctx.standingReceivers.get(agentName);
|
|
593
|
-
if (sr) {
|
|
594
|
-
this.#ctx.standingReceivers.delete(agentName);
|
|
595
|
-
/**
|
|
596
|
-
* DOD-M12B-SESSION-SEED-1 (review F8): drop it zeroed, like every other seed.
|
|
597
|
-
*
|
|
598
|
-
* (review F7, STILL DECIDED AGAINST — deliberately NOT reusing this seed for the
|
|
599
|
-
* replacement — but its stated blocker is GONE and the reason has changed. Restated rather
|
|
600
|
-
* than reworded, because a decision whose premise has been reversed is a decision nobody
|
|
601
|
-
* has actually made.)
|
|
602
|
-
*
|
|
603
|
-
* Reuse is attractive: this receiver's peer id may already be inside a `session_offer_accept`
|
|
604
|
-
* the counterparty is acting on, and a rebuild in that window is the documented "we record
|
|
605
|
-
* an identity that no longer exists… every send in this direction parks forever" defect.
|
|
606
|
-
*
|
|
607
|
-
* The old blocker was that a preserved identity would reach the candidate loop, whose
|
|
608
|
-
* rejected candidates were stopped WITHOUT awaiting `start()`, putting two live nodes on one
|
|
609
|
-
* advertised peer id. **032-RELAYSPREAD already crossed that line**: the walk now runs one
|
|
610
|
-
* shared seed through every candidate, with a settlement-chained teardown, and it is safe
|
|
611
|
-
* there because the receiver's gater admits nobody inbound.
|
|
612
|
-
*
|
|
613
|
-
* What still stops reuse HERE is different and is about the OLD node, not the new one. This
|
|
614
|
-
* rebuild path awaits `sr.node.stop()`, but a stop can hang on a stuck libp2p teardown, and
|
|
615
|
-
* handing the replacement the same identity before the previous receiver is provably dead
|
|
616
|
-
* would put two nodes on a peer id a COUNTERPARTY has been told to dial — which is not the
|
|
617
|
-
* candidate case at all: that node has a content handler and can be promoted. Doing it
|
|
618
|
-
* safely needs a bounded, verified teardown first. Still follow-on work.
|
|
619
|
-
*/
|
|
620
|
-
sr.seed.fill(0);
|
|
621
|
-
try {
|
|
622
|
-
sr.autoNat.stop();
|
|
623
|
-
await sr.node.stop();
|
|
624
|
-
}
|
|
625
|
-
catch (err) {
|
|
626
|
-
this.#ctx.logger.warn("session.standing_receiver.teardown.failed", {
|
|
627
|
-
agentName,
|
|
628
|
-
error: extractErrorMessage(err),
|
|
629
|
-
});
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
// The agent may have gone offline while we were stopping the old node. Its
|
|
633
|
-
// want-flag is the authority — never resurrect a receiver it disowned.
|
|
634
|
-
if (!this.#ctx.agentsWantingReceiver.has(agentName) || this.#ctx.shuttingDown())
|
|
635
|
-
return;
|
|
636
|
-
await this.ensureStandingReceiver(agentName);
|
|
637
|
-
}
|
|
638
|
-
catch (err) {
|
|
639
|
-
this.#ctx.logger.warn("session.standing_receiver.reservation.rebuild.failed", {
|
|
640
|
-
agentName,
|
|
641
|
-
error: extractErrorMessage(err),
|
|
642
|
-
});
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
733
|
/**
|
|
646
734
|
* DOD-M12B-SESSION-SEED-1 — build a revived session node that is REACHABLE, without ever hanging.
|
|
647
735
|
*
|
|
@@ -685,21 +773,27 @@ export class StandingReceivers {
|
|
|
685
773
|
* candidate and came up on the plain floor: alive, `active`, and dialable by nobody, with
|
|
686
774
|
* every message in both directions forced through the relay park route.
|
|
687
775
|
*
|
|
688
|
-
*
|
|
689
|
-
*
|
|
690
|
-
*
|
|
691
|
-
*
|
|
776
|
+
* ⚠️ DOD-M15-RELAYPROVE-ORDER-1 — **ONE ATTEMPT NOW, exactly as `#startReceiverNode` does
|
|
777
|
+
* it.** This used to be two: ask, be refused, prove, ask again, justified by *"a reservation
|
|
778
|
+
* taken by hand on the same connection as the proof yields no dialable address."* That
|
|
779
|
+
* described taking the slot over a raw HOP stream; asking libp2p's own transport manager
|
|
780
|
+
* after the proof makes the reservation libp2p's own, and it announces the address. Measured
|
|
781
|
+
* live 2026-09-08. So the candidate comes up with no circuit address, proves, and asks once.
|
|
782
|
+
*
|
|
783
|
+
* The seed is fixed here — that is what a revival IS — so this node carries the identity the
|
|
784
|
+
* relay records, and it must STAY UP between the proof and the ask: the relay marks the
|
|
785
|
+
* CONNECTION proven, and stopping the node closes it.
|
|
692
786
|
*/
|
|
693
787
|
let revivedNode;
|
|
694
788
|
let terminalRefusal = false;
|
|
695
|
-
|
|
789
|
+
{
|
|
696
790
|
const candidate = await this.createAgentNode(agentName, {
|
|
697
791
|
sessionId,
|
|
698
792
|
connectionGater: gater,
|
|
699
793
|
nodeType: "session",
|
|
700
794
|
inboundReachable: true,
|
|
701
795
|
transportPrivateKey: seed,
|
|
702
|
-
circuitRelayListenAddrs
|
|
796
|
+
// NO `circuitRelayListenAddrs` — libp2p must not ask before the proof below has landed.
|
|
703
797
|
});
|
|
704
798
|
// KEEP THE START PROMISE. Review HIGH-3: `libp2p.stop()` opens with
|
|
705
799
|
// `if (this.status !== 'started') return`, and during the whole timeout window the status is
|
|
@@ -714,82 +808,148 @@ export class StandingReceivers {
|
|
|
714
808
|
startP.then(() => true),
|
|
715
809
|
new Promise((res) => setTimeout(() => res(false), REVIVE_RESERVATION_TIMEOUT_MS).unref?.()),
|
|
716
810
|
]).catch((err) => { startError = err; return false; });
|
|
717
|
-
if (started && candidate.listenAddresses().some((a) => a.includes("/p2p-circuit"))) {
|
|
718
|
-
this.#ctx.logger.info("session.revive.reservation.granted", { agentName, sessionId, attempts: attempt + 1 });
|
|
719
|
-
revivedNode = candidate;
|
|
720
|
-
break;
|
|
721
|
-
}
|
|
722
811
|
/**
|
|
723
|
-
*
|
|
724
|
-
*
|
|
812
|
+
* PROVE, THEN ASK — the same order as `#startReceiverNode`, for the same reason.
|
|
813
|
+
*
|
|
814
|
+
* `started` gates it because `libp2p.stop()` opens with `if (this.status !== 'started')
|
|
815
|
+
* return`, so a timed-out candidate cannot be torn down here; that case falls through to the
|
|
816
|
+
* settlement-chained teardown below, which is the only thing that reliably kills a
|
|
817
|
+
* still-starting node.
|
|
725
818
|
*
|
|
726
|
-
*
|
|
727
|
-
*
|
|
728
|
-
* would
|
|
729
|
-
* teardown below, which is the only thing that reliably kills a still-starting node.
|
|
819
|
+
* ⚠️ THE CANDIDATE IS NOT STOPPED BETWEEN THE PROOF AND THE ASK. It used to be, because the
|
|
820
|
+
* ask came from a rebuilt node. The relay marks the CONNECTION proven, so stopping here
|
|
821
|
+
* would throw away the very thing that makes the next line succeed.
|
|
730
822
|
*/
|
|
731
|
-
|
|
823
|
+
let proofDeclined = false;
|
|
824
|
+
/**
|
|
825
|
+
* The ask's own promise, when one was made. The teardown below has to wait on THIS as well as
|
|
826
|
+
* on `start()`: the node whose reservation is still in flight is the one that can come up
|
|
827
|
+
* late holding this session's peer id, and `stop()` on a node mid-ask is the same no-op the
|
|
828
|
+
* start-promise note describes.
|
|
829
|
+
*/
|
|
830
|
+
let listenP;
|
|
831
|
+
/** Set when the ask failed for a fault of OURS, so it is not re-described as the relay's. */
|
|
832
|
+
let askFault;
|
|
833
|
+
if (started) {
|
|
732
834
|
const verdict = await this.#ctx.proveToRelay(agentName, circuitAddr, candidate, sessionId, false);
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
// The refusal is about this AGENT, so the remaining candidates
|
|
739
|
-
|
|
835
|
+
// A VERDICT DECLINES; NO VERDICT DOES NOT. `unavailable` means the relay never answered —
|
|
836
|
+
// no client wired, or unreachable — and not every relay gates reservations, so the ask
|
|
837
|
+
// still goes ahead. Same rule and same reasoning as `#startReceiverNode`.
|
|
838
|
+
if (verdict === "refused_this_agent" || verdict === "refused_try_another_relay") {
|
|
839
|
+
proofDeclined = true;
|
|
840
|
+
// The agent-level refusal is about this AGENT, so the remaining candidates answer
|
|
841
|
+
// identically.
|
|
842
|
+
if (verdict === "refused_this_agent")
|
|
843
|
+
terminalRefusal = true;
|
|
740
844
|
this.#ctx.logger.warn("session.revive.reservation.declined", {
|
|
741
845
|
agentName,
|
|
742
846
|
sessionId,
|
|
743
847
|
circuitAddr,
|
|
744
|
-
reason: "relay_refused_this_agent",
|
|
745
|
-
impact:
|
|
746
|
-
"
|
|
747
|
-
|
|
848
|
+
reason: verdict === "refused_this_agent" ? "relay_refused_this_agent" : "relay_proof_refused",
|
|
849
|
+
impact: verdict === "refused_this_agent"
|
|
850
|
+
? "the relay refused this agent rather than being unwilling or unwell, so every " +
|
|
851
|
+
"other relay refuses it the same way. The session comes up reachable only via " +
|
|
852
|
+
"the relay park route; cello_status carries the cause."
|
|
853
|
+
: "this relay would not take the agent's proof. Trying the next relay.",
|
|
748
854
|
});
|
|
749
|
-
|
|
855
|
+
try {
|
|
856
|
+
await candidate.stop();
|
|
857
|
+
}
|
|
858
|
+
catch { /* best-effort */ }
|
|
750
859
|
}
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
860
|
+
else {
|
|
861
|
+
/**
|
|
862
|
+
* ASK — once, on the connection the proof was made on.
|
|
863
|
+
*
|
|
864
|
+
* ⚠️ **RACED AGAINST THE SAME DEADLINE `start()` USED TO CARRY, AND IT HAS TO BE.** The
|
|
865
|
+
* measured production failure this whole loop exists for — 10,002ms and still waiting —
|
|
866
|
+
* was a relay that never answered a reservation. That used to park `start()`, because a
|
|
867
|
+
* circuit address in the constructor made start the moment libp2p asked. The ask is here
|
|
868
|
+
* now, so a bare await here is the same hang with a new address: the revival never
|
|
869
|
+
* returns and every send on that session is refused forever.
|
|
870
|
+
*
|
|
871
|
+
* A throw is not fatal — the grant check below is the only thing that decides, and it
|
|
872
|
+
* reads the announced addresses.
|
|
873
|
+
*/
|
|
874
|
+
// Wrapped, not bare: a node that cannot take the ask at all throws SYNCHRONOUSLY, and
|
|
875
|
+
// `.catch()` on the race never sees that — it would escape the revival entirely.
|
|
876
|
+
const asked = await (async () => {
|
|
877
|
+
listenP = candidate.listenOnCircuit(circuitAddr);
|
|
878
|
+
return Promise.race([
|
|
879
|
+
listenP.then(() => true),
|
|
880
|
+
new Promise((res) => setTimeout(() => res(false), REVIVE_RESERVATION_TIMEOUT_MS).unref?.()),
|
|
881
|
+
]);
|
|
882
|
+
})().catch((err) => {
|
|
883
|
+
startError = err;
|
|
884
|
+
// Review HIGH-2, same rule as the receiver walk: a fault of OURS keeps its own name.
|
|
885
|
+
askFault = clientSideAskFault(err);
|
|
886
|
+
return false;
|
|
760
887
|
});
|
|
761
|
-
|
|
888
|
+
if (!asked && askFault === undefined) {
|
|
889
|
+
this.#ctx.logger.warn("session.revive.reservation.ask_timeout", {
|
|
890
|
+
agentName,
|
|
891
|
+
sessionId,
|
|
892
|
+
circuitAddr,
|
|
893
|
+
budgetMs: REVIVE_RESERVATION_TIMEOUT_MS,
|
|
894
|
+
impact: "this relay took the proof and then never answered the reservation. Abandoned " +
|
|
895
|
+
"on the deadline and trying the next relay — a relay that does not answer must not " +
|
|
896
|
+
"be able to hold a session down.",
|
|
897
|
+
});
|
|
898
|
+
}
|
|
762
899
|
}
|
|
763
|
-
continue;
|
|
764
900
|
}
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
:
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
901
|
+
if (!proofDeclined && started && holdsCircuit(candidate)) {
|
|
902
|
+
this.#ctx.logger.info("session.revive.reservation.granted", { agentName, sessionId });
|
|
903
|
+
revivedNode = candidate;
|
|
904
|
+
break;
|
|
905
|
+
}
|
|
906
|
+
if (!proofDeclined) {
|
|
907
|
+
// Started but granted nothing, or never started. Either way this node is not the one.
|
|
908
|
+
//
|
|
909
|
+
// Review MEDIUM-5: name WHICH of the three causes this was, the way `#startReceiverNode` does.
|
|
910
|
+
// "declined" alone stood for a relay that is full, a relay that is unreachable, and a relay
|
|
911
|
+
// that is merely slow — three different problems with three different responses, and the
|
|
912
|
+
// thrown error was discarded entirely.
|
|
913
|
+
const declineReason = askFault !== undefined
|
|
914
|
+
// A client-side fault, checked FIRST: everything below infers a cause from the relay, and
|
|
915
|
+
// the relay had nothing to do with an ask that never left this process (review HIGH-2).
|
|
916
|
+
? askFault
|
|
917
|
+
: !started
|
|
918
|
+
? startError !== undefined
|
|
919
|
+
? "relay_unreachable"
|
|
920
|
+
: "reservation_did_not_complete_in_time"
|
|
921
|
+
: // Started, proved, asked — and the ask is where a slow relay now shows up. An ask still in
|
|
922
|
+
// flight is "did not complete in time"; one that returned with nothing is a relay that
|
|
923
|
+
// answered and granted nothing.
|
|
924
|
+
listenP !== undefined && !holdsCircuit(candidate)
|
|
925
|
+
? "relay_granted_no_reservation"
|
|
926
|
+
: "reservation_did_not_complete_in_time";
|
|
927
|
+
const isLast = circuitAddr === candidateAddrs.slice(0, REVIVE_RESERVATION_CANDIDATES).at(-1);
|
|
928
|
+
this.#ctx.logger.warn("session.revive.reservation.declined", {
|
|
929
|
+
agentName,
|
|
930
|
+
sessionId,
|
|
931
|
+
circuitAddr,
|
|
932
|
+
reason: declineReason,
|
|
933
|
+
...(startError !== undefined ? { error: extractErrorMessage(startError) } : {}),
|
|
934
|
+
impact: isLast
|
|
935
|
+
? "no relay granted; the session comes up reachable only via the relay park route"
|
|
936
|
+
: "trying the next relay",
|
|
937
|
+
});
|
|
938
|
+
/**
|
|
939
|
+
* Teardown at SETTLEMENT, not now: a `stop()` issued while the node is still starting is a
|
|
940
|
+
* no-op (see above), so the only way to guarantee this node dies is to wait for its own work
|
|
941
|
+
* to finish first. Not awaited, so a hung relay cannot hold the revival up — the point is that
|
|
942
|
+
* the teardown eventually happens, not that it happens before the next candidate.
|
|
943
|
+
*
|
|
944
|
+
* ⚠️ **BOTH PROMISES, and the second one is new.** The abandoned candidate's outstanding work
|
|
945
|
+
* used to be `start()`, because that is where the reservation was taken. It is the ASK now, so
|
|
946
|
+
* waiting only on `start()` tears the node down while its reservation is still in flight —
|
|
947
|
+
* and a late grant then brings a node up on THIS SESSION'S peer id, sharing the gater, with no
|
|
948
|
+
* content handler and nothing holding a reference to kill it. That is the open endpoint
|
|
949
|
+
* review HIGH-3 exists to prevent, reintroduced through a different promise.
|
|
950
|
+
*/
|
|
951
|
+
stopWhenSettled(candidate, [startP, listenP], REVIVE_RESERVATION_TIMEOUT_MS * 2);
|
|
952
|
+
}
|
|
793
953
|
}
|
|
794
954
|
if (revivedNode)
|
|
795
955
|
return revivedNode;
|
|
@@ -975,8 +1135,43 @@ export class StandingReceivers {
|
|
|
975
1135
|
// is not "retrying" — reporting it as such sends an operator hunting a fault that is not there.
|
|
976
1136
|
if (sr.relayPeerIds.length > 0)
|
|
977
1137
|
return "reserved";
|
|
1138
|
+
/**
|
|
1139
|
+
* ⚠️ **A LIVE SESSION'S CIRCUIT COUNTS — 055-ONDEMAND, and without this the field measures the
|
|
1140
|
+
* wrong node.**
|
|
1141
|
+
*
|
|
1142
|
+
* The standing receiver holds nothing while idle, by design. A session's circuit lives on the
|
|
1143
|
+
* SESSION's node: the receiver was promoted into it and replaced. Reading only the receiver
|
|
1144
|
+
* therefore reported `retrying` for an agent in a perfectly healthy conversation — a fault where
|
|
1145
|
+
* there is none, on the surface an operator checks first.
|
|
1146
|
+
*/
|
|
1147
|
+
if (this.#ctx.anyLiveSessionHoldsCircuit(agentName))
|
|
1148
|
+
return "reserved";
|
|
978
1149
|
const retry = this.#ctx.srReservationRetry.get(agentName);
|
|
979
|
-
|
|
1150
|
+
if (retry === undefined) {
|
|
1151
|
+
/**
|
|
1152
|
+
* ⚠️ **IDLE AND READY — DOD-M15-IDLE-READY-1, AND `retrying` HERE WAS NOT MERELY THE WRONG
|
|
1153
|
+
* WORD.**
|
|
1154
|
+
*
|
|
1155
|
+
* Reaching this line means: a receiver exists, it holds no circuit, no live session holds one
|
|
1156
|
+
* either, and **nothing has ever asked for one** — no retry state was recorded. Before
|
|
1157
|
+
* 055-ONDEMAND that combination was rare and meant a real problem: an agent that wanted a slot
|
|
1158
|
+
* at login and could not get one. An agent takes a slot when somebody calls now, so this is
|
|
1159
|
+
* the ordinary resting state of every healthy idle agent on the fleet.
|
|
1160
|
+
*
|
|
1161
|
+
* Leaving it as `retrying` cost the field its only job. `retrying` said "we want a slot, we
|
|
1162
|
+
* cannot get one, we are still trying" — and once every healthy agent says it, an agent that
|
|
1163
|
+
* genuinely cannot get a slot is indistinguishable from one that is perfectly fine. A status
|
|
1164
|
+
* that reads the same whether or not anything is wrong is not a status. Observed on two live
|
|
1165
|
+
* agents the same afternoon: `reserved` on the published build, `retrying` on this one, with
|
|
1166
|
+
* nothing different about whether anyone could reach them.
|
|
1167
|
+
*
|
|
1168
|
+
* ⚠️ **AND IT IS NOT `reserved`, which was the tempting one-liner.** `reserved` is a claim an
|
|
1169
|
+
* operator acts on — a slot is held, so a counterparty behind a home router can dial in right
|
|
1170
|
+
* now. That is false for an idle agent. Two different facts, so two different words.
|
|
1171
|
+
*/
|
|
1172
|
+
return "ready";
|
|
1173
|
+
}
|
|
1174
|
+
return retry.attempts > SR_RESERVATION_MAX_RETRIES ? "unreachable" : "retrying";
|
|
980
1175
|
}
|
|
981
1176
|
/**
|
|
982
1177
|
* CELLO-M7-TRANSPORT-001: the AutoNAT service wrapping the current standing
|