@cello-protocol/daemon 0.0.208 → 0.0.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-handlers.d.ts.map +1 -1
- package/dist/agent-handlers.js +2 -1
- package/dist/agent-handlers.js.map +1 -1
- package/dist/attendance-wiring.js +2 -2
- package/dist/attendance-wiring.js.map +1 -1
- package/dist/bin/cello-daemon.js +11 -1
- package/dist/bin/cello-daemon.js.map +1 -1
- package/dist/boot-agents.d.ts.map +1 -1
- package/dist/boot-agents.js +5 -5
- package/dist/boot-agents.js.map +1 -1
- package/dist/close-session-handler.d.ts.map +1 -1
- package/dist/close-session-handler.js +26 -3
- package/dist/close-session-handler.js.map +1 -1
- package/dist/connect-or-start.js +8 -2
- package/dist/connect-or-start.js.map +1 -1
- package/dist/content-park.d.ts.map +1 -1
- package/dist/content-park.js +5 -3
- package/dist/content-park.js.map +1 -1
- package/dist/log-collapse.d.ts +66 -0
- package/dist/log-collapse.d.ts.map +1 -0
- package/dist/log-collapse.js +244 -0
- package/dist/log-collapse.js.map +1 -0
- package/dist/log-rotate.d.ts +67 -0
- package/dist/log-rotate.d.ts.map +1 -0
- package/dist/log-rotate.js +134 -0
- package/dist/log-rotate.js.map +1 -0
- package/dist/outbound-sessions.d.ts.map +1 -1
- package/dist/outbound-sessions.js +28 -0
- package/dist/outbound-sessions.js.map +1 -1
- package/dist/park-recovery.d.ts +3 -2
- package/dist/park-recovery.d.ts.map +1 -1
- package/dist/park-recovery.js +3 -2
- package/dist/park-recovery.js.map +1 -1
- package/dist/refusal-reasons.d.ts +46 -0
- package/dist/refusal-reasons.d.ts.map +1 -1
- package/dist/refusal-reasons.js +44 -0
- package/dist/refusal-reasons.js.map +1 -1
- package/dist/relay-only.d.ts +9 -2
- package/dist/relay-only.d.ts.map +1 -1
- package/dist/relay-only.js +9 -2
- package/dist/relay-only.js.map +1 -1
- package/dist/restart-seal-resolver.d.ts.map +1 -1
- package/dist/restart-seal-resolver.js +13 -0
- package/dist/restart-seal-resolver.js.map +1 -1
- package/dist/session-ceremony.d.ts +17 -0
- package/dist/session-ceremony.d.ts.map +1 -1
- package/dist/session-ceremony.js +77 -1
- package/dist/session-ceremony.js.map +1 -1
- package/dist/session-content-send.js +12 -11
- package/dist/session-content-send.js.map +1 -1
- package/dist/session-lifecycle.d.ts.map +1 -1
- package/dist/session-lifecycle.js +36 -0
- package/dist/session-lifecycle.js.map +1 -1
- package/dist/session-node-factory.js +4 -4
- package/dist/session-node-factory.js.map +1 -1
- package/dist/session-node-manager.d.ts +7 -2
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +22 -6
- package/dist/session-node-manager.js.map +1 -1
- package/dist/session-node-types.d.ts +40 -13
- package/dist/session-node-types.d.ts.map +1 -1
- package/dist/session-node-types.js +18 -0
- package/dist/session-node-types.js.map +1 -1
- package/dist/session-relay-client.d.ts +21 -0
- package/dist/session-relay-client.d.ts.map +1 -1
- package/dist/session-relay-client.js +219 -3
- package/dist/session-relay-client.js.map +1 -1
- package/dist/session-relay.d.ts +15 -1
- package/dist/session-relay.d.ts.map +1 -1
- package/dist/session-relay.js +290 -161
- package/dist/session-relay.js.map +1 -1
- package/dist/signaling-connect.d.ts.map +1 -1
- package/dist/signaling-connect.js +23 -4
- package/dist/signaling-connect.js.map +1 -1
- package/dist/signaling-wiring.js +7 -7
- package/dist/signaling-wiring.js.map +1 -1
- package/dist/standing-receivers.d.ts +53 -14
- package/dist/standing-receivers.d.ts.map +1 -1
- package/dist/standing-receivers.js +551 -544
- package/dist/standing-receivers.js.map +1 -1
- package/dist/trust-signal-sweep-tick.d.ts +15 -0
- package/dist/trust-signal-sweep-tick.d.ts.map +1 -1
- package/dist/trust-signal-sweep-tick.js +19 -2
- package/dist/trust-signal-sweep-tick.js.map +1 -1
- package/dist/trust-signal-sweep.d.ts +16 -1
- package/dist/trust-signal-sweep.d.ts.map +1 -1
- package/dist/trust-signal-sweep.js +12 -10
- package/dist/trust-signal-sweep.js.map +1 -1
- package/dist/types.d.ts +18 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +5 -5
|
@@ -3,7 +3,7 @@ import { SessionConnectionGater } from "./session-connection-gater.js";
|
|
|
3
3
|
import { relayOnlyState, publishableEndpoint } from "./relay-only.js";
|
|
4
4
|
import { extractErrorMessage } from "./error-message.js";
|
|
5
5
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
6
|
-
import { relayPeerIdOf, heldRelayIdsOf,
|
|
6
|
+
import { relayPeerIdOf, heldRelayIdsOf, SR_RESERVATION_MAX_RETRIES, REVIVE_RESERVATION_CANDIDATES, REVIVE_RESERVATION_TIMEOUT_MS, RELEASE_TELL_BUDGET_MS, OFFER_RESERVATION_GRACE_MS, clientSideAskFault, holdsCircuit, stopWhenSettled, } from "./session-node-types.js";
|
|
7
7
|
import { STANDING_RECEIVER_AGENT_NAME } from "./types.js";
|
|
8
8
|
export class StandingReceivers {
|
|
9
9
|
#ctx;
|
|
@@ -25,426 +25,474 @@ export class StandingReceivers {
|
|
|
25
25
|
}
|
|
26
26
|
async #startReceiverNode(agentName, sessionId, gater, candidateCircuitAddrs, correlationId) {
|
|
27
27
|
/**
|
|
28
|
-
*
|
|
29
|
-
* DOD-M12B-SESSION-SEED-1's seed-per-candidate.
|
|
28
|
+
* 054-SRSPLIT — **ONE NODE. IT STARTS ON TCP, THEN TAKES ITS RESERVATIONS IN PLACE.**
|
|
30
29
|
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* the
|
|
30
|
+
* This used to be a walk of throwaway PROBE nodes — one built per relay to find out whether that
|
|
31
|
+
* relay would grant, torn down, and then a FINAL node rebuilt carrying every granted address in
|
|
32
|
+
* its constructor. The probes existed for exactly one reason: a circuit listener was fixed at
|
|
33
|
+
* node creation, so the only way to ask a relay anything was to build a node to ask with.
|
|
35
34
|
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
* `#buildRevivedNode` already runs a fixed identity through this same walk for the same reason.
|
|
35
|
+
* `listenOnCircuit` (unit 1) removed that constraint, and this is the simplification it unlocks:
|
|
36
|
+
* build the receiver ONCE, and for each relay prove and ask on the node that will keep it.
|
|
37
|
+
*
|
|
38
|
+
* ⚠️ **AND IT CLOSES THE DEFECT UNIT 1 LEFT STANDING, which was recorded here rather than
|
|
39
|
+
* fixed.** The final node was built with `circuitRelayListenAddrs`, so libp2p asked at start on
|
|
40
|
+
* a fresh connection that had proved nothing on itself. It worked only because the relay
|
|
41
|
+
* remembers a proof for `PROVEN_PEER_MEMORY_MS` — two minutes, in the OTHER repo — and when that
|
|
42
|
+
* memory had expired the original refused-ask collision came back for the one node that IS the
|
|
43
|
+
* agent's front door. There is no constructor-time ask left, so that dependency is gone, and
|
|
44
|
+
* `spread.grant_not_bound` — the event that existed to catch the proof expiring between the walk
|
|
45
|
+
* and the rebuild — has nothing left to report.
|
|
46
|
+
*
|
|
47
|
+
* ⚠️ **ONE SEED, and it still matters** (032-RELAYSPREAD). The agent is ONE identity and must be
|
|
48
|
+
* dialable at ONE peer id through any of its circuits. That is now structural rather than
|
|
49
|
+
* maintained: there is one node, so there is one key, and a second identity has nowhere to come
|
|
50
|
+
* from. The old hazard this note used to carry — an abandoned probe still starting on the
|
|
51
|
+
* receiver's seed — cannot occur, because no probe is built.
|
|
54
52
|
*/
|
|
55
53
|
const receiverSeed = randomBytes(32);
|
|
56
|
-
/**
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Relay peers are allowed OUTBOUND before the node starts. Unchanged and load-bearing: our own
|
|
56
|
+
* gater would otherwise refuse our own dial, and refreshes must keep working after
|
|
57
|
+
* `setAllowedPeer()` narrows the inbound gate to a session counterparty (DOD-M15-ASSIGN-1).
|
|
58
|
+
* The gater still admits NOBODY inbound here.
|
|
59
|
+
*/
|
|
60
|
+
for (const addr of candidateCircuitAddrs) {
|
|
61
|
+
const relayPeerId = relayPeerIdOf(addr);
|
|
62
|
+
if (relayPeerId)
|
|
63
|
+
gater.setAllowedOutboundPeer(relayPeerId);
|
|
64
|
+
}
|
|
65
|
+
const node = await this.createAgentNode(agentName, {
|
|
66
|
+
sessionId,
|
|
67
|
+
connectionGater: gater,
|
|
68
|
+
nodeType: "standing_receiver",
|
|
69
|
+
// NO `circuitRelayListenAddrs` — nothing is asked for at start. See the note above.
|
|
70
|
+
transportPrivateKey: receiverSeed,
|
|
71
|
+
});
|
|
72
|
+
await node.start();
|
|
73
|
+
/**
|
|
74
|
+
* ⚠️ **THE WALK IS GONE — 055-ONDEMAND, and this is the capacity change itself.**
|
|
75
|
+
*
|
|
76
|
+
* The receiver used to visit every relay it had ever heard of and hold a slot on each, for the
|
|
77
|
+
* life of the login, against the chance that somebody called. Demand was `agents × relays`, so
|
|
78
|
+
* the tenth relay added a tenth of the fleet's demand and one relay's worth of capacity, and
|
|
79
|
+
* the ratio `agents / slots-per-relay` never improved however many relays were run.
|
|
80
|
+
*
|
|
81
|
+
* **An idle agent now holds ZERO.** A reservation is taken when an offer arrives, on the relay
|
|
82
|
+
* the directory names (`takeReservationForSession`), and given back at the seal. Demand becomes
|
|
83
|
+
* `live sessions × 1`.
|
|
84
|
+
*
|
|
85
|
+
* **What makes that safe, and it is the reason this is not a reachability regression:**
|
|
86
|
+
* - The relay's MAILBOX consults no reservation. Deposit authenticates the depositor by their
|
|
87
|
+
* Noise peer id; pull is a signature challenge on the recipient and is an OUTBOUND dial. So
|
|
88
|
+
* store-and-forward keeps working with nothing held.
|
|
89
|
+
* - The WITNESS is a separate dial on the relay's own protocol. Nothing about the seal, the
|
|
90
|
+
* hash chain or the transcript depended on a reservation.
|
|
91
|
+
* - A cold call to a LOGGED-OUT agent was already refused with nothing queued, so no
|
|
92
|
+
* capability is lost — there was never an answering machine to lose.
|
|
93
|
+
*
|
|
94
|
+
* The candidate list is still passed in and still used: it is what the offer path reserves
|
|
95
|
+
* against when it needs to, and what `cello_status` reports as relays this agent could use.
|
|
96
|
+
*/
|
|
97
|
+
this.#ctx.logger.info("session.standing_receiver.idle", {
|
|
98
|
+
agentName,
|
|
99
|
+
relaysAvailable: candidateCircuitAddrs.length,
|
|
100
|
+
correlationId,
|
|
101
|
+
impact: "this agent holds no relay reservation while idle, by design. One is taken on the " +
|
|
102
|
+
"relay the directory names when a session is offered, and given back at the seal.",
|
|
103
|
+
});
|
|
104
|
+
return { node, seed: receiverSeed };
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* 054-SRSPLIT — **TAKE ONE RESERVATION, ON A NODE THAT IS ALREADY RUNNING.**
|
|
108
|
+
*
|
|
109
|
+
* The whole relay handshake for one relay: prove over `/cello/relay/1.0.0` on a connection we
|
|
110
|
+
* open, then ask libp2p's own transport manager to listen on the circuit. The relay grants on the
|
|
111
|
+
* first ask because it marks `slot.provenForReservation` per CONNECTION at auth time and libp2p's
|
|
112
|
+
* reservation store reuses the open connection rather than dialling a new one (measured live
|
|
113
|
+
* 2026-09-08; `DOD-M15-RELAYPROVE-ORDER-1`).
|
|
114
|
+
*
|
|
115
|
+
* ⚠️ **THE NODE IS NOT STOPPED BETWEEN THE PROOF AND THE ASK.** The relay marks the CONNECTION,
|
|
116
|
+
* not the peer id, so closing it throws away the very thing that makes the ask succeed.
|
|
117
|
+
*/
|
|
118
|
+
/**
|
|
119
|
+
* 055-ONDEMAND — **take a reservation for a SESSION, on the relay the directory named.**
|
|
120
|
+
*
|
|
121
|
+
* The public face of `#takeReservation` for the offer path. An idle agent holds nothing, so this
|
|
122
|
+
* is what makes it dialable, and it lasts only as long as the session that asked for it.
|
|
123
|
+
*
|
|
124
|
+
* Returns whether one was granted. A `false` is a degradation, not a failure: the counterparty
|
|
125
|
+
* can still reach this agent directly, or through the relay's store-and-forward. The one caller
|
|
126
|
+
* that must treat it as fatal is relay-only mode, which does so at its own guard.
|
|
127
|
+
*/
|
|
128
|
+
/**
|
|
129
|
+
* 055-ONDEMAND — the abandoned-offer timers, keyed `agent::sessionIdHex`. See `#armOfferRelease`.
|
|
130
|
+
*/
|
|
131
|
+
#offerReleaseTimers = new Map();
|
|
132
|
+
/**
|
|
133
|
+
* ⚠️ **AN OFFER THAT GOES QUIET MUST NOT KEEP A SLOT — `DOD-M15-OFFER-EXPIRY-1`, RELOCATED.**
|
|
134
|
+
*
|
|
135
|
+
* The story predicted this exactly: the permanently-open door units 2 and 3 removed does not take
|
|
136
|
+
* the defect with it, it moves it somewhere more expensive. The responder reserves the moment an
|
|
137
|
+
* offer arrives; an initiator that never dials — it aborted, the directory faulted — would leave a
|
|
138
|
+
* slot held on a SHARED relay until its TTL, two hours by default. That is a cost on the exact
|
|
139
|
+
* resource this whole story exists to conserve, and "released at seal" cannot cover it because
|
|
140
|
+
* there is no seal.
|
|
141
|
+
*
|
|
142
|
+
* ⚠️ **THE BUDGET IS NOT THE DIRECTORY'S 2-SECOND ACCEPT CLOCK, and picking that would be worse
|
|
143
|
+
* than not doing this at all.** The accept is only the start: the assignment still has to be
|
|
144
|
+
* FROST-signed by a threshold of directory nodes and delivered to both parties before either
|
|
145
|
+
* builds a session. Releasing on 2 s would take the slot out from under a session that was about
|
|
146
|
+
* to begin — turning a rare abandoned offer into a common broken one.
|
|
147
|
+
*
|
|
148
|
+
* No cancellation plumbing: the timer ASKS whether the session started. A session that began and
|
|
149
|
+
* has already sealed is also not live, and that case has already released through the seal path,
|
|
150
|
+
* where the teardown releases from the session's own node.
|
|
151
|
+
*/
|
|
152
|
+
#armOfferRelease(agentName, sessionIdHex, circuitAddr, correlationId) {
|
|
153
|
+
const key = `${agentName}::${sessionIdHex}`;
|
|
154
|
+
const existing = this.#offerReleaseTimers.get(key);
|
|
155
|
+
if (existing !== undefined)
|
|
156
|
+
clearTimeout(existing);
|
|
157
|
+
const timer = setTimeout(() => {
|
|
158
|
+
this.#offerReleaseTimers.delete(key);
|
|
159
|
+
if (this.#ctx.shuttingDown())
|
|
160
|
+
return;
|
|
161
|
+
if (this.#ctx.sessionIsLive(agentName, sessionIdHex))
|
|
162
|
+
return; // it started; the seal owns it now
|
|
163
|
+
this.#ctx.logger.warn("session.reservation.offer_abandoned", {
|
|
164
|
+
agentName,
|
|
165
|
+
sessionIdHex,
|
|
166
|
+
circuitAddr,
|
|
167
|
+
heldForMs: OFFER_RESERVATION_GRACE_MS,
|
|
168
|
+
correlationId,
|
|
169
|
+
impact: "a relay slot was taken to answer an offer that never became a session — the " +
|
|
170
|
+
"initiator did not dial. Given back rather than held until the relay's TTL, which is " +
|
|
171
|
+
"two hours and is a cost on every other agent that relay serves.",
|
|
116
172
|
});
|
|
117
|
-
let timer;
|
|
118
|
-
const timedOut = Symbol("reservation_timeout");
|
|
119
|
-
let outcome = "failed";
|
|
120
|
-
let error = "";
|
|
121
|
-
// KEEP THE START PROMISE. Every candidate now carries the receiver's identity, so an
|
|
122
|
-
// abandoned one must be reliably torn down rather than best-effort — and only its own start
|
|
123
|
-
// promise says when it is stoppable (see the seed note above).
|
|
124
|
-
const startP = candidate.start();
|
|
125
|
-
try {
|
|
126
|
-
outcome = await Promise.race([
|
|
127
|
-
startP.then(() => "started"),
|
|
128
|
-
new Promise((resolve) => {
|
|
129
|
-
timer = setTimeout(() => resolve(timedOut), this.#ctx.srReservationTimeoutMs);
|
|
130
|
-
}),
|
|
131
|
-
]);
|
|
132
|
-
}
|
|
133
|
-
catch (err) {
|
|
134
|
-
error = extractErrorMessage(err);
|
|
135
|
-
}
|
|
136
|
-
finally {
|
|
137
|
-
if (timer !== undefined)
|
|
138
|
-
clearTimeout(timer);
|
|
139
|
-
}
|
|
140
173
|
/**
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
174
|
+
* ⚠️ THE RECEIVER'S OWN NODE, and at this point that is the right one: the offer never became a
|
|
175
|
+
* session, so nothing was promoted and the circuit is still on the standing receiver. (Had it
|
|
176
|
+
* been promoted, `sessionIsLive` above would have returned and we would not be here.)
|
|
144
177
|
*/
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
*
|
|
150
|
-
* ⚠️ THE NODE STAYS UP. The old shape stopped it here, because the reservation was about to
|
|
151
|
-
* be asked for by a rebuilt node. Stopping it now would close the very connection the relay
|
|
152
|
-
* marked proven, and the ask below would be refused — the connection is the thing carrying
|
|
153
|
-
* the property, not the peer id.
|
|
154
|
-
*/
|
|
155
|
-
const verdict = await this.#ctx.proveToRelay(agentName, circuitAddr, candidate, correlationId, true);
|
|
156
|
-
/**
|
|
157
|
-
* ⚠️ **A PROOF THAT NEVER REACHED A VERDICT IS NOT A REFUSAL** — and the ask goes ahead.
|
|
158
|
-
*
|
|
159
|
-
* `unavailable` means no relay answer was obtained at all: no relay client is wired, the
|
|
160
|
-
* dial failed, the stream threw. `proveReservation` already draws exactly this line for its
|
|
161
|
-
* own retry (*"`true`/`false` are VERDICTS; `transport_failed` means no verdict was reached
|
|
162
|
-
* and the question is still open"*), and it is the line that matters here for a second
|
|
163
|
-
* reason: **not every relay gates reservations.** A relay that never asks for a proof
|
|
164
|
-
* grants on the first ask, and refusing to ask because our own proof machinery was
|
|
165
|
-
* unavailable would make this client unable to reserve with it at all — a capability lost
|
|
166
|
-
* to a precaution.
|
|
167
|
-
*
|
|
168
|
-
* The cost of asking anyway is one refused ask against a gated relay we could not prove to
|
|
169
|
-
* — which is what the whole walk used to do on every relay, every time.
|
|
170
|
-
*/
|
|
171
|
-
if (verdict === "refused_this_agent" || verdict === "refused_try_another_relay") {
|
|
172
|
-
/**
|
|
173
|
-
* DOD-M15-RELAYSLOTS-1 clause 9 — **A CLIENT-SIDE REFUSAL ENDS THE WALK.**
|
|
174
|
-
*
|
|
175
|
-
* `slot_cap_exceeded` and an expired or missing token are classified `tryAnotherRelay:
|
|
176
|
-
* false` because they reproduce on every relay in the pool: the cap is per AGENT, and
|
|
177
|
-
* the token comes from the directory, not from here. Walking on costs a node build and
|
|
178
|
-
* a dial per remaining relay to arrive at the same answer, and it makes one client-side
|
|
179
|
-
* fault look like a fleet-wide outage in the logs. The refusal is already recorded where
|
|
180
|
-
* `cello_status` reads it, so stopping is not silence.
|
|
181
|
-
*
|
|
182
|
-
* ⚠️ **THE BOUNDARY THIS BRANCH RESTS ON IS ENFORCED IN `proveToRelay`, NOT HERE.** It
|
|
183
|
-
* returns `unavailable` whenever no relay verdict was reached — including a proof that
|
|
184
|
-
* failed for a transport reason, which it used to label `refused_try_another_relay`
|
|
185
|
-
* (review HIGH-1). Stated as where the property lives rather than asserted as a fact
|
|
186
|
-
* about this branch: a comment claiming "only a verdict reaches here" is true only for
|
|
187
|
-
* as long as that producer keeps its side, and the producer is in another file.
|
|
188
|
-
*/
|
|
189
|
-
const proofReason = verdict === "refused_this_agent" ? "relay_refused_this_agent" : "relay_proof_refused";
|
|
190
|
-
this.#ctx.srLastRejectionReason.set(agentName, proofReason);
|
|
191
|
-
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
192
|
-
agentName,
|
|
193
|
-
circuitAddr,
|
|
194
|
-
reason: proofReason,
|
|
195
|
-
correlationId,
|
|
196
|
-
impact: proofReason === "relay_refused_this_agent"
|
|
197
|
-
? "the relay refused this AGENT rather than this relay being unwilling or " +
|
|
198
|
-
"unwell, so every other relay would refuse it identically. Stopped here; " +
|
|
199
|
-
"cello_status carries the cause and what to do about it."
|
|
200
|
-
: "this relay would not take the agent's proof. Moving to the next relay.",
|
|
201
|
-
});
|
|
202
|
-
if (verdict === "refused_this_agent")
|
|
203
|
-
candidateRefusedAgent = true;
|
|
204
|
-
rejectionNamed = true;
|
|
205
|
-
try {
|
|
206
|
-
await candidate.stop();
|
|
207
|
-
}
|
|
208
|
-
catch { /* it may never have finished starting */ }
|
|
209
|
-
}
|
|
210
|
-
else {
|
|
211
|
-
/**
|
|
212
|
-
* ⚠️ **A PROOF THAT NEVER REACHED A VERDICT IS NOT A REFUSAL** — and the ask goes ahead.
|
|
213
|
-
*
|
|
214
|
-
* `unavailable` means no relay answer was obtained at all: no relay client is wired, the
|
|
215
|
-
* dial failed, the stream threw. `proveReservation` already draws exactly this line for its
|
|
216
|
-
* own retry (*"`true`/`false` are VERDICTS; `transport_failed` means no verdict was reached
|
|
217
|
-
* and the question is still open"*), and it matters here for a second reason: **not every
|
|
218
|
-
* relay gates reservations.** One that never asks for a proof grants on the first ask, and
|
|
219
|
-
* refusing to ask because our own proof machinery was unavailable would make this client
|
|
220
|
-
* unable to reserve with it at all — a capability lost to a precaution.
|
|
221
|
-
*
|
|
222
|
-
* The cost of asking anyway is one refused ask against a gated relay we could not prove to,
|
|
223
|
-
* which is what the walk used to spend on every relay, every time.
|
|
224
|
-
*/
|
|
225
|
-
if (verdict === "unavailable") {
|
|
226
|
-
this.#ctx.logger.warn("session.standing_receiver.prove.no_verdict", {
|
|
227
|
-
agentName,
|
|
228
|
-
circuitAddr,
|
|
229
|
-
correlationId,
|
|
230
|
-
impact: "no proof verdict was obtained from this relay — no relay client is wired, or " +
|
|
231
|
-
"it could not be reached. Asking for the reservation anyway: a relay that does not " +
|
|
232
|
-
"gate them grants it, and one that does refuses an ask that cost a single dial.",
|
|
233
|
-
});
|
|
234
|
-
}
|
|
235
|
-
/**
|
|
236
|
-
* ASK — ONCE, on the connection we just proved on. Raced against the same budget the whole
|
|
237
|
-
* candidate used to get, because this is now the only part that talks to the relay.
|
|
238
|
-
*/
|
|
239
|
-
let listenTimer;
|
|
240
|
-
const listenTimedOut = Symbol("listen_timeout");
|
|
241
|
-
let listenOutcome = listenTimedOut;
|
|
242
|
-
// INSIDE the try. A node that cannot take the ask at all throws SYNCHRONOUSLY, and outside
|
|
243
|
-
// the try that escapes the whole walk instead of failing this one candidate.
|
|
244
|
-
try {
|
|
245
|
-
listenP = candidate.listenOnCircuit(circuitAddr);
|
|
246
|
-
listenOutcome = await Promise.race([
|
|
247
|
-
listenP.then(() => "asked"),
|
|
248
|
-
new Promise((resolve) => {
|
|
249
|
-
listenTimer = setTimeout(() => resolve(listenTimedOut), this.#ctx.srReservationTimeoutMs);
|
|
250
|
-
}),
|
|
251
|
-
]);
|
|
252
|
-
}
|
|
253
|
-
catch (err) {
|
|
254
|
-
error = extractErrorMessage(err);
|
|
255
|
-
// Review HIGH-2: a fault of OURS keeps its own name all the way to the operator, instead
|
|
256
|
-
// of being re-described from the connection state as something the relay did.
|
|
257
|
-
askFault = clientSideAskFault(err);
|
|
258
|
-
listenOutcome = listenTimedOut;
|
|
259
|
-
}
|
|
260
|
-
finally {
|
|
261
|
-
if (listenTimer !== undefined)
|
|
262
|
-
clearTimeout(listenTimer);
|
|
263
|
-
}
|
|
264
|
-
// The only proof that counts: the relay actually GRANTED the reservation. `listen()`
|
|
265
|
-
// resolving is not enough — a relay that is out of reservation slots completes the
|
|
266
|
-
// handshake and simply grants nothing, leaving a node that looks started and is reachable
|
|
267
|
-
// by nobody.
|
|
268
|
-
if (listenOutcome === "asked" && holdsCircuit(candidate)) {
|
|
269
|
-
candidateGranted = true;
|
|
270
|
-
// The probe has done its job: this relay grants THIS identity. Tear it down and ask the
|
|
271
|
-
// next relay — the reservation is re-taken by the final node below, which is the only one
|
|
272
|
-
// that can listen on every granted address at once. AWAITED, because the next probe comes
|
|
273
|
-
// up on this same peer id.
|
|
274
|
-
try {
|
|
275
|
-
await candidate.stop();
|
|
276
|
-
}
|
|
277
|
-
catch { /* it may never have finished starting */ }
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
if (!candidateGranted && !rejectionNamed) {
|
|
282
|
-
const rejectionReason =
|
|
283
|
-
// A CLIENT-SIDE FAULT KEEPS ITS OWN NAME, and it is checked FIRST because everything below
|
|
284
|
-
// infers a cause from the relay connection — which is intact and irrelevant when the ask
|
|
285
|
-
// never left this process (review HIGH-2).
|
|
286
|
-
askFault !== undefined
|
|
287
|
-
? askFault
|
|
288
|
-
: outcome === "started"
|
|
289
|
-
? /**
|
|
290
|
-
* ⚠️ Review MEDIUM-7 — **"ASKED" DOES NOT MEAN THE RELAY ANSWERED.** A relay that is
|
|
291
|
-
* simply DOWN yields no circuit address — indistinguishable, here, from a relay that
|
|
292
|
-
* answered and granted nothing. Reporting that as `relay_granted_no_reservation` sends
|
|
293
|
-
* the operator to look at relay capacity for what is a network fault. An open
|
|
294
|
-
* connection to the relay peer is the thing that separates them, and we have one to
|
|
295
|
-
* ask. (Reachable now only in the narrow case where the proof landed and the
|
|
296
|
-
* connection died before the ask — the proof branch above owns every other route to a
|
|
297
|
-
* dead relay, and names it.)
|
|
298
|
-
*/
|
|
299
|
-
(candidate.getConnections().some((c) => c.peerId === relayPeerIdOf(circuitAddr))
|
|
300
|
-
? "relay_granted_no_reservation"
|
|
301
|
-
: "relay_unreachable")
|
|
302
|
-
: outcome === "failed"
|
|
303
|
-
? "relay_unreachable"
|
|
304
|
-
: "reservation_did_not_complete_in_time";
|
|
305
|
-
this.#ctx.srLastRejectionReason.set(agentName, rejectionReason);
|
|
306
|
-
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
307
|
-
agentName,
|
|
308
|
-
circuitAddr,
|
|
309
|
-
reason: rejectionReason,
|
|
310
|
-
...(error !== "" ? { error } : {}),
|
|
311
|
-
correlationId,
|
|
312
|
-
});
|
|
313
|
-
// Abandon it — but on its OWN settlement, never best-effort. `start()` may still be parked on
|
|
314
|
-
// a dial, and this candidate carries the receiver's identity: an unawaited `stop()` on a node
|
|
315
|
-
// whose status is still `starting` returns without stopping anything, and the node then goes
|
|
316
|
-
// live on our peer id with nothing left holding a reference to kill it.
|
|
317
|
-
//
|
|
318
|
-
// BOTH promises: the reservation is taken by the ASK now, so a candidate abandoned on the
|
|
319
|
-
// ask's deadline still has work in flight that `start()` settling says nothing about.
|
|
320
|
-
stopWhenSettled(candidate, [startP, listenP], this.#ctx.srReservationTimeoutMs * 2);
|
|
178
|
+
const sr = this.#ctx.standingReceivers.get(agentName);
|
|
179
|
+
if (sr) {
|
|
180
|
+
void this.releaseSessionReservation(agentName, sr.node, sessionIdHex, correlationId)
|
|
181
|
+
.catch(() => { });
|
|
321
182
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
183
|
+
}, OFFER_RESERVATION_GRACE_MS);
|
|
184
|
+
timer.unref?.();
|
|
185
|
+
this.#offerReleaseTimers.set(key, timer);
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* 055-ONDEMAND — re-take a circuit on a node that already exists, for a live session that lost it.
|
|
189
|
+
*
|
|
190
|
+
* ⚠️ **IT PROVES OVER THE DELIVERY PATH TOO.** `DOD-M15-RELAYAUTH-1` applies identically here: a
|
|
191
|
+
* holder that has not proven key possession has its reservation revoked inside the relay's grace
|
|
192
|
+
* window, about fifteen seconds. The first version of the re-take called the low-level ask
|
|
193
|
+
* directly and skipped it, so a re-taken circuit would have been revoked almost immediately while
|
|
194
|
+
* the log said "granted" — the same defect this unit already fixed one function up.
|
|
195
|
+
*/
|
|
196
|
+
async retakeReservationOn(agentName, node, circuitAddr, correlationId) {
|
|
197
|
+
const relayPeerId = relayPeerIdOf(circuitAddr);
|
|
198
|
+
const outcome = await this.#takeReservation(agentName, node, circuitAddr, correlationId);
|
|
199
|
+
if (outcome !== "granted")
|
|
200
|
+
return false;
|
|
330
201
|
/**
|
|
331
|
-
* THE
|
|
202
|
+
* ⚠️ **THE OTHER HALF OF `reservation_lost` — 056-SLOTDEAD, review F4.**
|
|
332
203
|
*
|
|
333
|
-
*
|
|
334
|
-
*
|
|
335
|
-
*
|
|
336
|
-
*
|
|
204
|
+
* The loss trigger fires the moment a reservation goes, which is exactly when the relay link is
|
|
205
|
+
* down, so the pull it starts is aimed at a relay that cannot answer. Without something on the
|
|
206
|
+
* recovery, content the counterparty parked during the outage waits for the slow periodic
|
|
207
|
+
* backstop while the relay is healthy and this agent is connected to it again.
|
|
337
208
|
*
|
|
338
|
-
*
|
|
339
|
-
*
|
|
340
|
-
*
|
|
341
|
-
*
|
|
342
|
-
*
|
|
209
|
+
* **This path, and not the two that look like it.** The watchdog's `gained` branch was the
|
|
210
|
+
* first attempt and never fired — a take records the new circuit on the receiver itself, so by
|
|
211
|
+
* the next tick there is nothing left for the watchdog to see as gained; a test caught it, which
|
|
212
|
+
* is the only reason it is not still in the tree looking correct. `takeReservationForSession`
|
|
213
|
+
* was the second, and it double-drains: on a first login it fires moments after the install
|
|
214
|
+
* drain, for the same empty mailbox.
|
|
343
215
|
*
|
|
344
|
-
*
|
|
345
|
-
*
|
|
216
|
+
* A re-take is unambiguous. It happens only when a session that HAD a circuit lost it, which is
|
|
217
|
+
* exactly the outage whose recovery this is.
|
|
346
218
|
*/
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
219
|
+
this.#ctx.park.fireParkedDrain(agentName, "reservation_regained");
|
|
220
|
+
if (relayPeerId) {
|
|
221
|
+
void this.#ctx.authenticateStandingReceiver(agentName, node, relayPeerId, circuitAddr, correlationId)
|
|
222
|
+
.catch((err) => {
|
|
223
|
+
this.#ctx.logger.warn("session.standing_receiver.relay_auth.failed", {
|
|
224
|
+
agentName, relayPeerId, correlationId,
|
|
225
|
+
error: extractErrorMessage(err),
|
|
226
|
+
impact: "a live session re-took a circuit but did not prove key possession over the " +
|
|
227
|
+
"delivery path, so the relay may revoke it inside its grace window.",
|
|
228
|
+
});
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
return true;
|
|
232
|
+
}
|
|
233
|
+
async takeReservationForSession(agentName, circuitAddr, correlationId,
|
|
234
|
+
/**
|
|
235
|
+
* ⚠️ **PRESENT ONLY ON THE OFFER PATH, AND THAT IS WHAT ARMS THE ABANDONED-OFFER RELEASE.**
|
|
236
|
+
*
|
|
237
|
+
* The first version reused `correlationId` for this, because the offer handler happens to pass
|
|
238
|
+
* the session id as its correlation id. Two meanings in one parameter is how a value ends up
|
|
239
|
+
* being trusted for something it was never chosen for: the watchdog's re-take passes a genuine
|
|
240
|
+
* correlation id, and it would have armed a timer keyed on a string that is not a session.
|
|
241
|
+
*
|
|
242
|
+
* Absent for the re-take path, which is by definition for a session that is already live.
|
|
243
|
+
*/
|
|
244
|
+
offerSessionIdHex) {
|
|
245
|
+
const sr = this.#ctx.standingReceivers.get(agentName);
|
|
246
|
+
if (!sr) {
|
|
247
|
+
this.#ctx.logger.warn("session.reservation.on_demand.no_receiver", {
|
|
248
|
+
agentName,
|
|
249
|
+
circuitAddr,
|
|
250
|
+
correlationId,
|
|
251
|
+
impact: "there is no standing receiver to hold a reservation, so this agent cannot be " +
|
|
252
|
+
"dialled for this session; it is reachable through the relay's store-and-forward only.",
|
|
253
|
+
});
|
|
254
|
+
return false;
|
|
357
255
|
}
|
|
256
|
+
// The relay must be dialable BEFORE we dial it — our own gater refuses otherwise, which is the
|
|
257
|
+
// same ordering the login walk uses and the one that cost a whole debugging session when it
|
|
258
|
+
// was missing.
|
|
259
|
+
const relayPeerId = relayPeerIdOf(circuitAddr);
|
|
260
|
+
if (relayPeerId)
|
|
261
|
+
sr.gater.setAllowedOutboundPeer(relayPeerId);
|
|
262
|
+
const outcome = await this.#takeReservation(agentName, sr.node, circuitAddr, correlationId);
|
|
358
263
|
/**
|
|
359
|
-
* ⚠️
|
|
360
|
-
*
|
|
361
|
-
*
|
|
362
|
-
*
|
|
363
|
-
*
|
|
264
|
+
* ⚠️ **THE RECEIVER'S RECORD OF WHAT IT HOLDS MUST FOLLOW, or two things go quietly wrong.**
|
|
265
|
+
* `relayPeerIds` is what the reservation watchdog compares against to decide a reservation was
|
|
266
|
+
* LOST, and what `cello_status` reports as reachability. Left at its build-time value — empty,
|
|
267
|
+
* now that nothing is taken at login — the watchdog would see a held circuit it never recorded
|
|
268
|
+
* and `cello_status` would call a reachable agent unreachable.
|
|
364
269
|
*
|
|
365
|
-
*
|
|
366
|
-
*
|
|
367
|
-
* operator was told the receiver "did not finish binding every circuit inside the deadline" and
|
|
368
|
-
* "is reachable through those" — sending them to the relay fleet for a port held by an orphan
|
|
369
|
-
* daemon on their own machine. The rejection is rethrown so it reaches
|
|
370
|
-
* `session.node.create.failed` with its own cause, exactly as it does on the no-relay path.
|
|
270
|
+
* Read from the NODE rather than appended to, and deduped by relay: libp2p announces one
|
|
271
|
+
* address per relay listen address, so a five-address relay would otherwise count five times.
|
|
371
272
|
*/
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
const started = node.start().then(() => "ok", (err) => { startError = err; return "failed"; });
|
|
375
|
-
const outcome = await Promise.race([
|
|
376
|
-
started,
|
|
377
|
-
new Promise((resolve) => {
|
|
378
|
-
// Per granted relay: each circuit listener is its own dial and its own reservation, so a
|
|
379
|
-
// pool of three must not be judged on a budget sized for one.
|
|
380
|
-
deadline = setTimeout(() => resolve("slow"), this.#ctx.srReservationTimeoutMs * grantedAddrs.length);
|
|
381
|
-
}),
|
|
382
|
-
]);
|
|
383
|
-
if (deadline !== undefined)
|
|
384
|
-
clearTimeout(deadline);
|
|
385
|
-
if (outcome === "failed")
|
|
386
|
-
throw startError;
|
|
273
|
+
const heldNow = heldRelayIdsOf(sr.node);
|
|
274
|
+
this.#ctx.standingReceivers.set(agentName, { ...sr, relayPeerIds: heldNow });
|
|
387
275
|
/**
|
|
388
|
-
*
|
|
389
|
-
* one. The receiver would simply report `reservationsHeld: 2` where 3 relays granted, with
|
|
390
|
-
* nothing naming which relay went missing or why.
|
|
276
|
+
* ⚠️ **A RELAY THAT GRANTED MUST BE ADMITTED INBOUND, OR THE RESERVATION BUYS NOTHING.**
|
|
391
277
|
*
|
|
392
|
-
*
|
|
393
|
-
*
|
|
394
|
-
*
|
|
278
|
+
* The gater's inbound carve-out is the security-sensitive half of a reservation: only relays
|
|
279
|
+
* whose own grant is confirmed earn it, so a directory that merely NAMES a relay cannot dial us
|
|
280
|
+
* through it. That set was built once from what the login walk held — empty now — so an
|
|
281
|
+
* on-demand reservation would have been taken, announced, and then refused by OUR OWN gater
|
|
282
|
+
* when the counterparty dialled through it.
|
|
283
|
+
*
|
|
284
|
+
* Recomputed from what the node HOLDS, never from what was asked: being named by the directory
|
|
285
|
+
* must not buy a foothold, and under on-demand the directory is what names the relay.
|
|
286
|
+
*/
|
|
287
|
+
sr.gater.setReservedRelayPeers(heldNow);
|
|
288
|
+
/**
|
|
289
|
+
* ⚠️ **DOD-M15-RELAYAUTH-1 STILL APPLIES, AND IT NO LONGER FIRES ON ITS OWN.**
|
|
395
290
|
*
|
|
396
|
-
*
|
|
397
|
-
*
|
|
398
|
-
*
|
|
399
|
-
*
|
|
400
|
-
*
|
|
401
|
-
*
|
|
402
|
-
* probe/final split is what forces it, and that split is what unit 2 of
|
|
403
|
-
* `M15-STORY-RESERVATIONS-ON-DEMAND` (splitting the standing receiver) deletes; the fix belongs
|
|
404
|
-
* there rather than in a second pass here.
|
|
291
|
+
* That auth is a SECOND proof, over the DELIVERY path, and it is what keeps the relay from
|
|
292
|
+
* revoking a reservation whose holder has not proven key possession to it — the grace window is
|
|
293
|
+
* about fifteen seconds. It used to run in the receiver build, over the circuits the login walk
|
|
294
|
+
* had just collected. Nothing is collected there any more, so without this line an on-demand
|
|
295
|
+
* reservation would be taken and then quietly revoked mid-session, and the agent would go
|
|
296
|
+
* unreachable while every log said the reservation was granted.
|
|
405
297
|
*
|
|
406
|
-
*
|
|
407
|
-
*
|
|
408
|
-
* candidate costs `start` + `proveToRelay` + `ask`, and **`proveToRelay` carries no deadline of
|
|
409
|
-
* its own at either call site**. So a pool of three can still exceed the two-minute memory, and
|
|
410
|
-
* the ceiling is now harder to state than it was, not easier. That missing deadline is recorded
|
|
411
|
-
* in `053-FIRSTASK`'s *Newly discovered*.
|
|
298
|
+
* Best-effort and unawaited, exactly as it is in the build path: a failure here costs the
|
|
299
|
+
* relay's own grace-window revoke, which the watchdog already treats as an ordinary loss.
|
|
412
300
|
*/
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
301
|
+
if (outcome === "granted" && offerSessionIdHex !== undefined) {
|
|
302
|
+
this.#armOfferRelease(agentName, offerSessionIdHex, circuitAddr, correlationId);
|
|
303
|
+
}
|
|
304
|
+
if (outcome === "granted" && relayPeerId) {
|
|
305
|
+
void this.#ctx.authenticateStandingReceiver(agentName, sr.node, relayPeerId, circuitAddr, correlationId)
|
|
306
|
+
.catch((err) => {
|
|
307
|
+
this.#ctx.logger.warn("session.standing_receiver.relay_auth.failed", {
|
|
308
|
+
agentName, relayPeerId, correlationId,
|
|
309
|
+
error: extractErrorMessage(err),
|
|
310
|
+
impact: "this agent has a reservation the relay may revoke within its grace window, " +
|
|
311
|
+
"because key possession was not proven over the delivery path.",
|
|
312
|
+
});
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
return outcome === "granted";
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* 055-ONDEMAND — **GIVE THIS SESSION'S SLOT BACK, FROM THE NODE THAT ACTUALLY HOLDS IT.**
|
|
319
|
+
*
|
|
320
|
+
* ⚠️ **THE FIRST VERSION LOOKED THE NODE UP BY AGENT NAME AND ALWAYS FOUND THE WRONG ONE.** When a
|
|
321
|
+
* session opens, the standing receiver is PROMOTED: `session-lifecycle` deletes it from
|
|
322
|
+
* `standingReceivers`, moves that exact node into `activeNodes`, and builds a fresh empty receiver
|
|
323
|
+
* behind it. So by seal time `standingReceivers.get(agentName)` is the new idle node, holding
|
|
324
|
+
* nothing — the release read `[]`, returned immediately, and told the relay nothing. The slot the
|
|
325
|
+
* offer took lived on the promoted node and died with it, invisible to the relay, held for the
|
|
326
|
+
* full TTL. **A release that always no-ops is worse than none: it logs success.**
|
|
327
|
+
*
|
|
328
|
+
* ⚠️ **AND THE CROSS-SESSION RECOMPUTE THE ORDER ASKED FOR IS UNNECESSARY, which is the good news
|
|
329
|
+
* in the correction.** libp2p's shared `reservationStore` is shared *within one node*. Each live
|
|
330
|
+
* session owns its OWN node, so sealing one cannot clear another session's refresh timers. There
|
|
331
|
+
* is nothing to recompute: this node is being torn down anyway, so tell its relays and drop its
|
|
332
|
+
* circuits. Re-deriving that removed a whole class of drift rather than managing it.
|
|
333
|
+
*/
|
|
334
|
+
async releaseSessionReservation(agentName, node, sessionId, correlationId) {
|
|
335
|
+
const held = heldRelayIdsOf(node);
|
|
336
|
+
if (held.length === 0)
|
|
337
|
+
return;
|
|
338
|
+
for (const relayPeerId of held) {
|
|
339
|
+
/**
|
|
340
|
+
* ⚠️ BOUNDED, BECAUSE THIS RUNS INSIDE A TEARDOWN. Telling a relay means dialling it, and an
|
|
341
|
+
* unreachable one would otherwise hold the seal open for as long as its dial takes. Measured
|
|
342
|
+
* the hard way: unbounded, this hung fourteen unrelated suites at 237s each. A seal waiting on
|
|
343
|
+
* a courtesy is worse than a slot held until its TTL.
|
|
344
|
+
*/
|
|
345
|
+
await Promise.race([
|
|
346
|
+
this.#ctx.tellRelayReleased(agentName, relayPeerId, node, correlationId),
|
|
347
|
+
new Promise((r) => setTimeout(r, RELEASE_TELL_BUDGET_MS).unref?.()),
|
|
348
|
+
]);
|
|
349
|
+
}
|
|
350
|
+
// Local half. The node is going away regardless; this stops it announcing a route it no longer
|
|
351
|
+
// holds for whatever is left of its life.
|
|
352
|
+
try {
|
|
353
|
+
await node.releaseAllCircuits();
|
|
354
|
+
}
|
|
355
|
+
catch (err) {
|
|
356
|
+
this.#ctx.logger.debug("session.reservation.release.local_failed", {
|
|
357
|
+
agentName, sessionId, correlationId, error: extractErrorMessage(err),
|
|
358
|
+
// The node is being torn down either way, so this costs nothing beyond a few more seconds
|
|
359
|
+
// of announcing a route that is already gone at the relay. Deliberately NOT claiming the
|
|
360
|
+
// watchdog repairs it (review MEDIUM-11): this unit removed the rebuild that used to.
|
|
361
|
+
impact: "the session's node keeps announcing a circuit it has released, until it stops.",
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
this.#ctx.logger.info("session.reservation.released", {
|
|
365
|
+
agentName,
|
|
366
|
+
sessionId,
|
|
367
|
+
releasedRelays: held,
|
|
368
|
+
correlationId,
|
|
369
|
+
impact: "the slot this session borrowed is back in the relay's table, rather than held until " +
|
|
370
|
+
"its two-hour TTL.",
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
async #takeReservation(agentName, node, circuitAddr, correlationId) {
|
|
374
|
+
const verdict = await this.#ctx.proveToRelay(agentName, circuitAddr, node, correlationId, true);
|
|
375
|
+
if (verdict === "refused_this_agent" || verdict === "refused_try_another_relay") {
|
|
376
|
+
const reason = verdict === "refused_this_agent" ? "relay_refused_this_agent" : "relay_proof_refused";
|
|
377
|
+
this.#ctx.srLastRejectionReason.set(agentName, reason);
|
|
378
|
+
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
419
379
|
agentName,
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
reservationsHeld: boundRelays.size,
|
|
423
|
-
walkMs: Date.now() - walkStartedAt,
|
|
380
|
+
circuitAddr,
|
|
381
|
+
reason,
|
|
424
382
|
correlationId,
|
|
425
|
-
impact:
|
|
426
|
-
"
|
|
427
|
-
|
|
428
|
-
"
|
|
429
|
-
"not the relay fleet.",
|
|
383
|
+
impact: verdict === "refused_this_agent"
|
|
384
|
+
? "the relay refused this AGENT rather than being unwilling or unwell, so every other " +
|
|
385
|
+
"relay would refuse it identically. Stopped here; cello_status carries the cause."
|
|
386
|
+
: "this relay would not take the agent's proof. Moving to the next relay.",
|
|
430
387
|
});
|
|
388
|
+
return verdict === "refused_this_agent" ? "refused_this_agent" : "declined";
|
|
431
389
|
}
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
390
|
+
/**
|
|
391
|
+
* ⚠️ **A PROOF THAT REACHED NO VERDICT IS NOT A REFUSAL** — `unavailable` means the relay never
|
|
392
|
+
* answered (no client wired, or unreachable), and NOT every relay gates reservations. Refusing
|
|
393
|
+
* to ask because our own proof path was unavailable would lose the ability to reserve with an
|
|
394
|
+
* ungated relay entirely. The boundary is enforced in `proveToRelay`, which returns
|
|
395
|
+
* `unavailable` for exactly this (review HIGH-1 on unit 1).
|
|
396
|
+
*/
|
|
397
|
+
if (verdict === "unavailable") {
|
|
398
|
+
this.#ctx.logger.warn("session.standing_receiver.prove.no_verdict", {
|
|
437
399
|
agentName,
|
|
438
|
-
|
|
439
|
-
circuitAddrs: node.listenAddresses().filter((a) => a.includes("/p2p-circuit")).length,
|
|
440
|
-
budgetMs: this.#ctx.srReservationTimeoutMs * grantedAddrs.length,
|
|
400
|
+
circuitAddr,
|
|
441
401
|
correlationId,
|
|
442
|
-
impact: "
|
|
443
|
-
"
|
|
444
|
-
"
|
|
402
|
+
impact: "no proof verdict was obtained from this relay. Asking for the reservation anyway: " +
|
|
403
|
+
"a relay that does not gate them grants it, and one that does refuses an ask that cost a " +
|
|
404
|
+
"single dial.",
|
|
445
405
|
});
|
|
446
406
|
}
|
|
447
|
-
|
|
407
|
+
let askFault;
|
|
408
|
+
let error = "";
|
|
409
|
+
let timer;
|
|
410
|
+
const timedOut = Symbol("listen_timeout");
|
|
411
|
+
let outcome = timedOut;
|
|
412
|
+
try {
|
|
413
|
+
/**
|
|
414
|
+
* ⚠️ **AN ABANDONED ASK NOW LANDS ON A NODE THAT LIVES ON** — review MEDIUM-5, and it is the
|
|
415
|
+
* mirror image of the hazard the old probe teardown existed for.
|
|
416
|
+
*
|
|
417
|
+
* When a probe timed out it was destroyed, so a grant arriving late died with it. There is
|
|
418
|
+
* one long-lived node now: a late grant ADDS a circuit address after the walk has counted
|
|
419
|
+
* what it holds, so the receiver would advertise a relay that is in neither `sr.relayPeerIds`
|
|
420
|
+
* nor the gater's reserved set — the ledger and the advertised addresses disagreeing, which
|
|
421
|
+
* is what the watchdog then churns on.
|
|
422
|
+
*
|
|
423
|
+
* So a late grant is GIVEN BACK rather than kept. Releasing costs a dial; keeping it costs a
|
|
424
|
+
* disagreement no operator can see, and a slot on a relay we already decided against.
|
|
425
|
+
*/
|
|
426
|
+
const askP = node.listenOnCircuit(circuitAddr).then(() => "asked");
|
|
427
|
+
void askP.catch(() => { });
|
|
428
|
+
outcome = await Promise.race([
|
|
429
|
+
askP,
|
|
430
|
+
new Promise((resolve) => {
|
|
431
|
+
timer = setTimeout(() => resolve(timedOut), this.#ctx.srReservationTimeoutMs);
|
|
432
|
+
}),
|
|
433
|
+
]);
|
|
434
|
+
if (outcome === timedOut) {
|
|
435
|
+
void askP.then(() => {
|
|
436
|
+
this.#ctx.logger.warn("session.standing_receiver.reservation.late_grant_released", {
|
|
437
|
+
agentName,
|
|
438
|
+
circuitAddr,
|
|
439
|
+
correlationId,
|
|
440
|
+
impact: "this relay answered after the walk had moved on, so its circuit was not " +
|
|
441
|
+
"counted or advertised. Given back rather than held: a slot nobody knows about is " +
|
|
442
|
+
"one the relay cannot reuse and this agent cannot rely on. ⚠️ This drops EVERY " +
|
|
443
|
+
"circuit — libp2p's reservation store is shared across listeners and cannot " +
|
|
444
|
+
"release one — so the receiver is rebuilt by the watchdog, which is the correct " +
|
|
445
|
+
"outcome: a walk whose result is already wrong should be redone, not patched.",
|
|
446
|
+
});
|
|
447
|
+
return node.releaseAllCircuits().catch(() => false);
|
|
448
|
+
}, () => { });
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
error = extractErrorMessage(err);
|
|
453
|
+
// A fault of OURS keeps its own name rather than being re-derived from the relay connection,
|
|
454
|
+
// which is intact and irrelevant when the ask never left this process (review HIGH-2).
|
|
455
|
+
askFault = clientSideAskFault(err);
|
|
456
|
+
}
|
|
457
|
+
finally {
|
|
458
|
+
if (timer !== undefined)
|
|
459
|
+
clearTimeout(timer);
|
|
460
|
+
}
|
|
461
|
+
// The only proof that counts: an ANNOUNCED circuit address on this relay. `listen()` resolving
|
|
462
|
+
// is not enough — a relay at its slot cap completes the handshake and grants nothing.
|
|
463
|
+
const relayPeerId = relayPeerIdOf(circuitAddr);
|
|
464
|
+
if (outcome === "asked" && node.listenAddresses().some((a) => a.split("/").includes("p2p-circuit") && (relayPeerId === null || a.includes(`/p2p/${relayPeerId}/`)))) {
|
|
465
|
+
return "granted";
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* ⚠️ **THE CONNECTION CHECK APPLIES TO BOTH OUTCOMES** — review MEDIUM-6.
|
|
469
|
+
*
|
|
470
|
+
* "Asked" does not mean the relay answered: a relay that is down yields no circuit address,
|
|
471
|
+
* indistinguishable here from one that answered and granted nothing. An open connection is what
|
|
472
|
+
* separates them, and we have one to ask.
|
|
473
|
+
*
|
|
474
|
+
* The first version of this rewrite applied that check only to the `asked` branch, so a HUNG
|
|
475
|
+
* ask short-circuited to `reservation_did_not_complete_in_time` — latency — when the relay was
|
|
476
|
+
* simply gone. That is a name for where the failure surfaced, not for what went wrong, in a
|
|
477
|
+
* unit whose headline is that behaviour does not change.
|
|
478
|
+
*/
|
|
479
|
+
const connectedToRelay = node.getConnections().some((c) => c.peerId === relayPeerId);
|
|
480
|
+
const reason = askFault !== undefined
|
|
481
|
+
? askFault
|
|
482
|
+
: !connectedToRelay
|
|
483
|
+
? "relay_unreachable"
|
|
484
|
+
: outcome === "asked"
|
|
485
|
+
? "relay_granted_no_reservation"
|
|
486
|
+
: "reservation_did_not_complete_in_time";
|
|
487
|
+
this.#ctx.srLastRejectionReason.set(agentName, reason);
|
|
488
|
+
this.#ctx.logger.warn("session.standing_receiver.relay.rejected", {
|
|
489
|
+
agentName,
|
|
490
|
+
circuitAddr,
|
|
491
|
+
reason,
|
|
492
|
+
...(error !== "" ? { error } : {}),
|
|
493
|
+
correlationId,
|
|
494
|
+
});
|
|
495
|
+
return "declined";
|
|
448
496
|
}
|
|
449
497
|
/** One standing-receiver create attempt (extracted for the M8B F14 retry loop). */
|
|
450
498
|
async #tryCreateStandingReceiver(agentName, correlationId) {
|
|
@@ -529,43 +577,28 @@ export class StandingReceivers {
|
|
|
529
577
|
});
|
|
530
578
|
autoNat.emitInitialResult();
|
|
531
579
|
/**
|
|
532
|
-
*
|
|
533
|
-
*
|
|
580
|
+
* ⚠️ **A RECEIVER IS INSTALLED HOLDING NOTHING, AND THAT IS NOW A CONSTANT — 056-SLOTDEAD.**
|
|
581
|
+
*
|
|
582
|
+
* This block used to derive what the node had come up holding: `heldRelayIdsOf(node)`, the
|
|
583
|
+
* matching circuit addresses, the count for the reachability line, and the gater's inbound
|
|
584
|
+
* carve-out. Every one of those read the result of the login walk. `#startReceiverNode` returns
|
|
585
|
+
* a TCP-only node now (055-ONDEMAND), so all four were computing `[]` — reachable, referenced,
|
|
586
|
+
* and unable to produce a different answer on any input.
|
|
534
587
|
*
|
|
535
|
-
* The
|
|
536
|
-
*
|
|
537
|
-
*
|
|
538
|
-
*
|
|
539
|
-
* absent on every tick forever, and it rebuilt on the 30-second grid — churning the very
|
|
540
|
-
* reservations this unit exists to conserve. A candidate is a relay we ASKED; only a held
|
|
541
|
-
* address is a relay that ANSWERED, and the fallback conflated the two.
|
|
588
|
+
* The distinction the deleted comment was defending — a CANDIDATE is a relay we asked, a HELD
|
|
589
|
+
* address is one that answered — is still the rule, and it still lives in `heldRelayIdsOf`. It
|
|
590
|
+
* is enforced where a reservation is actually taken (`takeReservationForSession`) and where one
|
|
591
|
+
* is checked (the watchdog), which is where it belongs. Nothing is held here to check.
|
|
542
592
|
*
|
|
543
|
-
* The
|
|
544
|
-
*
|
|
545
|
-
* `/p2p/<id>/p2p-circuit` form, that address yields no id and is not counted as held — so the
|
|
546
|
-
* receiver reads as degraded and gets rebuilt, instead of reading as healthy against a relay
|
|
547
|
-
* nobody is connected to. Degrading toward "rebuild" is the safe direction; the other one is
|
|
548
|
-
* the silent unreachability this whole file exists to kill.
|
|
593
|
+
* The gater is left as constructed: its reserved set starts empty, and `takeReservationForSession`
|
|
594
|
+
* widens it for the one relay that grants. Setting it to `[]` here only restated that.
|
|
549
595
|
*/
|
|
550
|
-
const heldRelayPeerIds = heldRelayIdsOf(node);
|
|
551
|
-
const circuitAddrs = heldRelayPeerIds.length;
|
|
552
|
-
const heldCircuitAddrs = node.listenAddresses().filter((a) => a.includes("/p2p-circuit"));
|
|
553
|
-
// DOD-M15-ASSIGN-1 review N3, widened by 032-RELAYSPREAD: the relays this receiver actually
|
|
554
|
-
// reserved with earn the inbound AutoNAT carve-out — nothing else does. Populated only from
|
|
555
|
-
// reservations that genuinely completed, so a directory that merely NAMES a relay cannot dial
|
|
556
|
-
// in behind it, however many relays it names.
|
|
557
|
-
gater.setReservedRelayPeers(heldRelayPeerIds);
|
|
558
|
-
// The re-spread clock starts HERE, at the build, not at the epoch. Otherwise the first decay
|
|
559
|
-
// re-spreads instantly — undoing the "a lost relay does not rebuild the receiver" rule seconds
|
|
560
|
-
// after it fires, and changing the peer id of an agent that just lost one relay of three. The
|
|
561
|
-
// ratchet this guards against runs over hours; nothing about it needs answering in a second.
|
|
562
|
-
this.#ctx.srLastRespreadAt.set(agentName, Date.now());
|
|
563
596
|
this.#ctx.standingReceivers.set(agentName, {
|
|
564
597
|
node,
|
|
565
598
|
gater,
|
|
566
599
|
autoNat,
|
|
567
600
|
seed,
|
|
568
|
-
relayPeerIds:
|
|
601
|
+
relayPeerIds: [],
|
|
569
602
|
});
|
|
570
603
|
this.#ctx.logger.info("session.node.created", {
|
|
571
604
|
sessionId,
|
|
@@ -573,69 +606,74 @@ export class StandingReceivers {
|
|
|
573
606
|
sessionPeerId: node.getPeerId(),
|
|
574
607
|
correlationId,
|
|
575
608
|
});
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
// reservation with every relay it knows" read as true in an audit: the outcome was one and the
|
|
609
|
-
// request was one too, and a single field could report neither.
|
|
610
|
-
// relaysOffered — how many relays were in the candidate list (deduped by relay peer id in
|
|
611
|
-
// `#reservationCircuitAddrs`, so it counts relays, not addresses).
|
|
612
|
-
// reservationsHeld — how many reservations this node actually holds, counted the only way
|
|
613
|
-
// that proves a grant: ANNOUNCED /p2p-circuit listen addresses. `start()`
|
|
614
|
-
// resolving is not enough — a relay out of reservation slots completes the
|
|
615
|
-
// handshake, grants nothing, and leaves a node that looks started and is
|
|
616
|
-
// dialable by nobody.
|
|
609
|
+
/**
|
|
610
|
+
* ⚠️ **DOD-M15-RELAYAUTH-1'S INSTALL-TIME PROOF IS GONE — 056-SLOTDEAD, and it could not have
|
|
611
|
+
* run since 055-ONDEMAND.**
|
|
612
|
+
*
|
|
613
|
+
* It read: *"authenticate to the reservation relay NOW, not when a session first needs one"* —
|
|
614
|
+
* because the relay revokes a reservation whose holder has not proven key possession to it, and
|
|
615
|
+
* a login-time slot had to survive that grace window with no session in sight. It looped over
|
|
616
|
+
* the relays this receiver held. It holds none, so the loop's body never executed.
|
|
617
|
+
*
|
|
618
|
+
* **The requirement it served is not gone; it moved, and it moved to the stronger place.** A
|
|
619
|
+
* reservation is now taken by `takeReservationForSession`, which proves FIRST and asks second on
|
|
620
|
+
* the same connection (`DOD-M15-RELAYPROVE-ORDER-1`). "Proven before the slot exists" is
|
|
621
|
+
* structural there, rather than a second best-effort call racing a grace window.
|
|
622
|
+
*/
|
|
623
|
+
/**
|
|
624
|
+
* DOD-NAT-REACHABILITY-1 observability: what did this receiver come up able to use?
|
|
625
|
+
*
|
|
626
|
+
* ⚠️ **`reservationsHeld` WAS DROPPED, NOT RENAMED — 056-SLOTDEAD.** It carried
|
|
627
|
+
* `heldRelayIdsOf(node).length` and was the one number in this line an operator would act on.
|
|
628
|
+
* Since 055-ONDEMAND a receiver is installed holding nothing by design, so it reported `0` on
|
|
629
|
+
* every healthy login for every agent — a measurement that had become a constant while still
|
|
630
|
+
* reading as a measurement. That is worse than not reporting it: the number an operator trusts
|
|
631
|
+
* to mean "this agent is deaf" now means nothing at all.
|
|
632
|
+
*
|
|
633
|
+
* `relaysOffered` stays and is still a real count: how many relays are in the candidate list
|
|
634
|
+
* (deduped by relay peer id in `reservationCircuitAddrs`, so it counts relays, not addresses).
|
|
635
|
+
* **Zero of them is the condition worth seeing here** — an agent with no candidate cannot take a
|
|
636
|
+
* slot when an offer arrives, and will refuse the call.
|
|
637
|
+
*
|
|
638
|
+
* What an agent actually holds is reported where it is now decided: `session.offer.reservation`
|
|
639
|
+
* at the moment a slot is asked for, and `getStandingReceiverReachability` for `cello_status`.
|
|
640
|
+
*/
|
|
617
641
|
this.#ctx.logger.info("session.standing_receiver.reachability", {
|
|
618
642
|
agentName,
|
|
619
643
|
relaysOffered: reservations.addrs.length,
|
|
620
|
-
reservationsHeld: circuitAddrs,
|
|
621
644
|
correlationId,
|
|
622
645
|
});
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
646
|
+
/**
|
|
647
|
+
* ⚠️ **`reservation.none` IS NOT EMITTED AT BUILD ANY MORE — 055-ONDEMAND.**
|
|
648
|
+
*
|
|
649
|
+
* It means *"this agent is offered relays and holds none, so nobody behind NAT can dial it"* —
|
|
650
|
+
* a warn that fired 481 times over 17 days and drove a whole retry story. Under on-demand,
|
|
651
|
+
* holding none at build is the DESIGN: a slot is taken when an offer arrives and given back at
|
|
652
|
+
* the seal. Leaving the warn here would fire it on every healthy login for every agent, which
|
|
653
|
+
* is not a smaller version of the old problem but a worse one — an alarm that is wrong every
|
|
654
|
+
* time trains its reader to ignore the one time it is right.
|
|
655
|
+
*
|
|
656
|
+
* The condition it named still has a home. `session.offer.reservation` reports `granted: false`
|
|
657
|
+
* when an offer could not get a circuit — the moment it actually costs someone something — and
|
|
658
|
+
* the watchdog's re-take path reports a live session that lost one.
|
|
659
|
+
*
|
|
660
|
+
* ⚠️ **AND ITS DEBUG-LEVEL REPLACEMENT WENT TOO — 056-SLOTDEAD, review F10.** A
|
|
661
|
+
* `session.standing_receiver.idle_no_reservation` line survived here, guarded on
|
|
662
|
+
* `reservations.addrs.length > 0 && circuitAddrs === 0`. The second term was always true once
|
|
663
|
+
* the login walk was deleted, so the guard was really just "this agent has candidates" and the
|
|
664
|
+
* line fired on every healthy install for every agent — a second event, under a name that reads
|
|
665
|
+
* as a fault, asserting the same by-design state the `idle` line above already reports with its
|
|
666
|
+
* `relaysAvailable` count. Two lines per install saying "normal" is how a log stops being read.
|
|
667
|
+
*/
|
|
634
668
|
// DOD-PARK-DRAIN-1: this agent has a receiver again — drain whatever parked while it did not.
|
|
635
|
-
//
|
|
636
|
-
//
|
|
637
|
-
//
|
|
638
|
-
//
|
|
669
|
+
// The defect this closes was a trigger hooked to the wrong connection: content parks when the
|
|
670
|
+
// RELAY link dies, and the drain was waiting on DIRECTORY SIGNALING to reconnect — which it
|
|
671
|
+
// never had to, having never dropped.
|
|
672
|
+
//
|
|
673
|
+
// 056-SLOTDEAD: this used to be the ONE place every path converged on, because a lost
|
|
674
|
+
// reservation rebuilt the receiver and arrived back here. The rebuilds are gone, so this now
|
|
675
|
+
// covers the INSTALL only, and the loss has its own trigger (`reservation_lost`) at the point
|
|
676
|
+
// the loss is noticed.
|
|
639
677
|
this.#ctx.park.fireParkedDrain(agentName, "standing_receiver_ready");
|
|
640
678
|
return { outcome: "installed" };
|
|
641
679
|
}
|
|
@@ -692,72 +730,6 @@ export class StandingReceivers {
|
|
|
692
730
|
this.#ctx.standingReceiverCreating.delete(agentName);
|
|
693
731
|
}
|
|
694
732
|
}
|
|
695
|
-
/**
|
|
696
|
-
* Replace an agent's reservation-less standing receiver with one that reserves.
|
|
697
|
-
*
|
|
698
|
-
* Deliberately NOT removeStandingReceiverForAgent()+ensureStandingReceiverForAgent():
|
|
699
|
-
* the public remove CLEARS #agentsWantingReceiver, so a cello_set_agent_offline landing in
|
|
700
|
-
* the window while node.stop() is awaited would find no map entry and no creating
|
|
701
|
-
* marker, leave no tombstone, and the re-ensure would then RESURRECT a receiver for
|
|
702
|
-
* an agent that asked to go dark — accepting inbound sessions for an offline agent.
|
|
703
|
-
* Here the want-flag is left intact and re-checked after the stop: a concurrent stop
|
|
704
|
-
* clears it, and the rebuild correctly no-ops.
|
|
705
|
-
*/
|
|
706
|
-
async rebuildStandingReceiver(agentName) {
|
|
707
|
-
try {
|
|
708
|
-
const sr = this.#ctx.standingReceivers.get(agentName);
|
|
709
|
-
if (sr) {
|
|
710
|
-
this.#ctx.standingReceivers.delete(agentName);
|
|
711
|
-
/**
|
|
712
|
-
* DOD-M12B-SESSION-SEED-1 (review F8): drop it zeroed, like every other seed.
|
|
713
|
-
*
|
|
714
|
-
* (review F7, STILL DECIDED AGAINST — deliberately NOT reusing this seed for the
|
|
715
|
-
* replacement — but its stated blocker is GONE and the reason has changed. Restated rather
|
|
716
|
-
* than reworded, because a decision whose premise has been reversed is a decision nobody
|
|
717
|
-
* has actually made.)
|
|
718
|
-
*
|
|
719
|
-
* Reuse is attractive: this receiver's peer id may already be inside a `session_offer_accept`
|
|
720
|
-
* the counterparty is acting on, and a rebuild in that window is the documented "we record
|
|
721
|
-
* an identity that no longer exists… every send in this direction parks forever" defect.
|
|
722
|
-
*
|
|
723
|
-
* The old blocker was that a preserved identity would reach the candidate loop, whose
|
|
724
|
-
* rejected candidates were stopped WITHOUT awaiting `start()`, putting two live nodes on one
|
|
725
|
-
* advertised peer id. **032-RELAYSPREAD already crossed that line**: the walk now runs one
|
|
726
|
-
* shared seed through every candidate, with a settlement-chained teardown, and it is safe
|
|
727
|
-
* there because the receiver's gater admits nobody inbound.
|
|
728
|
-
*
|
|
729
|
-
* What still stops reuse HERE is different and is about the OLD node, not the new one. This
|
|
730
|
-
* rebuild path awaits `sr.node.stop()`, but a stop can hang on a stuck libp2p teardown, and
|
|
731
|
-
* handing the replacement the same identity before the previous receiver is provably dead
|
|
732
|
-
* would put two nodes on a peer id a COUNTERPARTY has been told to dial — which is not the
|
|
733
|
-
* candidate case at all: that node has a content handler and can be promoted. Doing it
|
|
734
|
-
* safely needs a bounded, verified teardown first. Still follow-on work.
|
|
735
|
-
*/
|
|
736
|
-
sr.seed.fill(0);
|
|
737
|
-
try {
|
|
738
|
-
sr.autoNat.stop();
|
|
739
|
-
await sr.node.stop();
|
|
740
|
-
}
|
|
741
|
-
catch (err) {
|
|
742
|
-
this.#ctx.logger.warn("session.standing_receiver.teardown.failed", {
|
|
743
|
-
agentName,
|
|
744
|
-
error: extractErrorMessage(err),
|
|
745
|
-
});
|
|
746
|
-
}
|
|
747
|
-
}
|
|
748
|
-
// The agent may have gone offline while we were stopping the old node. Its
|
|
749
|
-
// want-flag is the authority — never resurrect a receiver it disowned.
|
|
750
|
-
if (!this.#ctx.agentsWantingReceiver.has(agentName) || this.#ctx.shuttingDown())
|
|
751
|
-
return;
|
|
752
|
-
await this.ensureStandingReceiver(agentName);
|
|
753
|
-
}
|
|
754
|
-
catch (err) {
|
|
755
|
-
this.#ctx.logger.warn("session.standing_receiver.reservation.rebuild.failed", {
|
|
756
|
-
agentName,
|
|
757
|
-
error: extractErrorMessage(err),
|
|
758
|
-
});
|
|
759
|
-
}
|
|
760
|
-
}
|
|
761
733
|
/**
|
|
762
734
|
* DOD-M12B-SESSION-SEED-1 — build a revived session node that is REACHABLE, without ever hanging.
|
|
763
735
|
*
|
|
@@ -1163,8 +1135,43 @@ export class StandingReceivers {
|
|
|
1163
1135
|
// is not "retrying" — reporting it as such sends an operator hunting a fault that is not there.
|
|
1164
1136
|
if (sr.relayPeerIds.length > 0)
|
|
1165
1137
|
return "reserved";
|
|
1138
|
+
/**
|
|
1139
|
+
* ⚠️ **A LIVE SESSION'S CIRCUIT COUNTS — 055-ONDEMAND, and without this the field measures the
|
|
1140
|
+
* wrong node.**
|
|
1141
|
+
*
|
|
1142
|
+
* The standing receiver holds nothing while idle, by design. A session's circuit lives on the
|
|
1143
|
+
* SESSION's node: the receiver was promoted into it and replaced. Reading only the receiver
|
|
1144
|
+
* therefore reported `retrying` for an agent in a perfectly healthy conversation — a fault where
|
|
1145
|
+
* there is none, on the surface an operator checks first.
|
|
1146
|
+
*/
|
|
1147
|
+
if (this.#ctx.anyLiveSessionHoldsCircuit(agentName))
|
|
1148
|
+
return "reserved";
|
|
1166
1149
|
const retry = this.#ctx.srReservationRetry.get(agentName);
|
|
1167
|
-
|
|
1150
|
+
if (retry === undefined) {
|
|
1151
|
+
/**
|
|
1152
|
+
* ⚠️ **IDLE AND READY — DOD-M15-IDLE-READY-1, AND `retrying` HERE WAS NOT MERELY THE WRONG
|
|
1153
|
+
* WORD.**
|
|
1154
|
+
*
|
|
1155
|
+
* Reaching this line means: a receiver exists, it holds no circuit, no live session holds one
|
|
1156
|
+
* either, and **nothing has ever asked for one** — no retry state was recorded. Before
|
|
1157
|
+
* 055-ONDEMAND that combination was rare and meant a real problem: an agent that wanted a slot
|
|
1158
|
+
* at login and could not get one. An agent takes a slot when somebody calls now, so this is
|
|
1159
|
+
* the ordinary resting state of every healthy idle agent on the fleet.
|
|
1160
|
+
*
|
|
1161
|
+
* Leaving it as `retrying` cost the field its only job. `retrying` said "we want a slot, we
|
|
1162
|
+
* cannot get one, we are still trying" — and once every healthy agent says it, an agent that
|
|
1163
|
+
* genuinely cannot get a slot is indistinguishable from one that is perfectly fine. A status
|
|
1164
|
+
* that reads the same whether or not anything is wrong is not a status. Observed on two live
|
|
1165
|
+
* agents the same afternoon: `reserved` on the published build, `retrying` on this one, with
|
|
1166
|
+
* nothing different about whether anyone could reach them.
|
|
1167
|
+
*
|
|
1168
|
+
* ⚠️ **AND IT IS NOT `reserved`, which was the tempting one-liner.** `reserved` is a claim an
|
|
1169
|
+
* operator acts on — a slot is held, so a counterparty behind a home router can dial in right
|
|
1170
|
+
* now. That is false for an idle agent. Two different facts, so two different words.
|
|
1171
|
+
*/
|
|
1172
|
+
return "ready";
|
|
1173
|
+
}
|
|
1174
|
+
return retry.attempts > SR_RESERVATION_MAX_RETRIES ? "unreachable" : "retrying";
|
|
1168
1175
|
}
|
|
1169
1176
|
/**
|
|
1170
1177
|
* CELLO-M7-TRANSPORT-001: the AutoNAT service wrapping the current standing
|