@cello-protocol/daemon 0.0.207 → 0.0.209
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-handlers.d.ts +12 -0
- package/dist/agent-handlers.d.ts.map +1 -1
- package/dist/agent-handlers.js +6 -2
- package/dist/agent-handlers.js.map +1 -1
- package/dist/attendance-wiring.js +2 -2
- package/dist/attendance-wiring.js.map +1 -1
- package/dist/boot-agents.d.ts.map +1 -1
- package/dist/boot-agents.js +5 -5
- package/dist/boot-agents.js.map +1 -1
- package/dist/close-session-handler.d.ts.map +1 -1
- package/dist/close-session-handler.js +26 -3
- package/dist/close-session-handler.js.map +1 -1
- package/dist/content-park.d.ts.map +1 -1
- package/dist/content-park.js +5 -3
- package/dist/content-park.js.map +1 -1
- package/dist/daemon.js +5 -4
- package/dist/daemon.js.map +1 -1
- package/dist/outbound-sessions.d.ts +27 -2
- package/dist/outbound-sessions.d.ts.map +1 -1
- package/dist/outbound-sessions.js +53 -3
- package/dist/outbound-sessions.js.map +1 -1
- package/dist/park-recovery.d.ts +3 -2
- package/dist/park-recovery.d.ts.map +1 -1
- package/dist/park-recovery.js +3 -2
- package/dist/park-recovery.js.map +1 -1
- package/dist/refusal-reasons.d.ts +46 -0
- package/dist/refusal-reasons.d.ts.map +1 -1
- package/dist/refusal-reasons.js +44 -0
- package/dist/refusal-reasons.js.map +1 -1
- package/dist/relay-only.d.ts +9 -2
- package/dist/relay-only.d.ts.map +1 -1
- package/dist/relay-only.js +9 -2
- package/dist/relay-only.js.map +1 -1
- package/dist/restart-seal-resolver.d.ts.map +1 -1
- package/dist/restart-seal-resolver.js +13 -0
- package/dist/restart-seal-resolver.js.map +1 -1
- package/dist/session-ceremony.d.ts +17 -0
- package/dist/session-ceremony.d.ts.map +1 -1
- package/dist/session-ceremony.js +77 -1
- package/dist/session-ceremony.js.map +1 -1
- package/dist/session-content-send.js +12 -11
- package/dist/session-content-send.js.map +1 -1
- package/dist/session-lifecycle.d.ts.map +1 -1
- package/dist/session-lifecycle.js +36 -0
- package/dist/session-lifecycle.js.map +1 -1
- package/dist/session-node-factory.js +4 -4
- package/dist/session-node-factory.js.map +1 -1
- package/dist/session-node-manager.d.ts +7 -2
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +22 -6
- package/dist/session-node-manager.js.map +1 -1
- package/dist/session-node-types.d.ts +79 -13
- package/dist/session-node-types.d.ts.map +1 -1
- package/dist/session-node-types.js +67 -0
- package/dist/session-node-types.js.map +1 -1
- package/dist/session-relay-client.d.ts +21 -0
- package/dist/session-relay-client.d.ts.map +1 -1
- package/dist/session-relay-client.js +219 -3
- package/dist/session-relay-client.js.map +1 -1
- package/dist/session-relay.d.ts +15 -1
- package/dist/session-relay.d.ts.map +1 -1
- package/dist/session-relay.js +264 -168
- package/dist/session-relay.js.map +1 -1
- package/dist/signaling-connect.d.ts.map +1 -1
- package/dist/signaling-connect.js +23 -4
- package/dist/signaling-connect.js.map +1 -1
- package/dist/signaling-wiring.js +7 -7
- package/dist/signaling-wiring.js.map +1 -1
- package/dist/standing-receivers.d.ts +53 -14
- package/dist/standing-receivers.d.ts.map +1 -1
- package/dist/standing-receivers.js +695 -500
- package/dist/standing-receivers.js.map +1 -1
- package/dist/trust-signal-sweep-tick.d.ts +60 -0
- package/dist/trust-signal-sweep-tick.d.ts.map +1 -0
- package/dist/trust-signal-sweep-tick.js +101 -0
- package/dist/trust-signal-sweep-tick.js.map +1 -0
- package/dist/trust-signal-sweep.d.ts +16 -1
- package/dist/trust-signal-sweep.d.ts.map +1 -1
- package/dist/trust-signal-sweep.js +40 -16
- package/dist/trust-signal-sweep.js.map +1 -1
- package/dist/types.d.ts +18 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +5 -5
package/dist/session-relay.js
CHANGED
|
@@ -130,8 +130,15 @@ export class SessionRelay {
|
|
|
130
130
|
* what looks like a fleet-wide outage.
|
|
131
131
|
*/
|
|
132
132
|
if (refusal?.tryAnotherRelay && !this.#ctx.shuttingDown) {
|
|
133
|
+
/**
|
|
134
|
+
* ⚠️ **THE QUARANTINE ACTS; THE REBUILD USED TO AND NO LONGER CAN — 056-SLOTDEAD.**
|
|
135
|
+
*
|
|
136
|
+
* This used to `rebuildStandingReceiver` so the new node would reserve against the rest of
|
|
137
|
+
* the pool. A rebuilt receiver reserves NOTHING now (055-ONDEMAND), so the rebuild bought
|
|
138
|
+
* nothing and cost the agent its transport identity. The quarantine is the half that still
|
|
139
|
+
* does the work: the next offer reserves against the pool minus this relay.
|
|
140
|
+
*/
|
|
133
141
|
this.#quarantineRelay(agentName, relayPeerId, refusal.reason);
|
|
134
|
-
void this.#ctx.receivers.rebuildStandingReceiver(agentName);
|
|
135
142
|
}
|
|
136
143
|
}
|
|
137
144
|
else {
|
|
@@ -742,18 +749,20 @@ export class SessionRelay {
|
|
|
742
749
|
this.#ctx.srReservationRetry.delete(agentName);
|
|
743
750
|
this.#ctx.srLastRejectionReason.delete(agentName);
|
|
744
751
|
}
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
752
|
+
/**
|
|
753
|
+
* ⚠️ **ENDPOINTS ARRIVING NO LONGER REBUILD THE RECEIVER — 055-ONDEMAND.**
|
|
754
|
+
*
|
|
755
|
+
* This used to rebuild so the new node could reserve with the relays that had just been
|
|
756
|
+
* announced, because reservations were fixed at node creation and acquired at login. Neither is
|
|
757
|
+
* true now: nothing is reserved at login, and a running node can take one on demand.
|
|
758
|
+
*
|
|
759
|
+
* So a rebuild here would buy nothing and cost something real. The receiver it replaces may be
|
|
760
|
+
* holding the circuit a LIVE SESSION's counterparty was told to dial — throwing that away in
|
|
761
|
+
* response to a routine directory announcement would drop the route silently, at both ends.
|
|
762
|
+
*
|
|
763
|
+
* The endpoints are still recorded above, which is all they were ever needed for: they are the
|
|
764
|
+
* candidate list an offer reserves against.
|
|
765
|
+
*/
|
|
757
766
|
}
|
|
758
767
|
/**
|
|
759
768
|
* Is this agent currently skipping this relay? The observable half of the failover decision — a
|
|
@@ -880,34 +889,45 @@ export class SessionRelay {
|
|
|
880
889
|
* That is precisely the silent-loss-of-inbound failure this whole story exists to
|
|
881
890
|
* kill, so it cannot be left to chance: we watch for it and re-pick a relay.
|
|
882
891
|
*
|
|
883
|
-
* Only receivers that HAD a reservation are watched.
|
|
884
|
-
* already degraded and already loud (reservation.none /
|
|
885
|
-
* rebuilding it on a timer would just thrash against relays we know are
|
|
892
|
+
* ⚠️ 056-SLOTDEAD: this paragraph used to end "Only receivers that HAD a reservation are watched.
|
|
893
|
+
* One that never got one is already degraded and already loud (reservation.none /
|
|
894
|
+
* reservation.timeout); rebuilding it on a timer would just thrash against relays we know are
|
|
895
|
+
* refusing." Every clause of that is now false — an idle receiver holds nothing BY DESIGN so
|
|
896
|
+
* "degraded" is the normal state, `reservation.none` is not emitted at build any more, and
|
|
897
|
+
* nothing rebuilds. What is watched is a receiver whose recorded set has shrunk, and what
|
|
898
|
+
* happens is a re-take in place for a LIVE session, never a rebuild.
|
|
886
899
|
*/
|
|
887
900
|
/**
|
|
888
|
-
* DOD-M12B-RESERVATION-RETRY-1
|
|
901
|
+
* DOD-M12B-RESERVATION-RETRY-1 / 055-ONDEMAND — is a re-attempt due, and claim it if so.
|
|
902
|
+
*
|
|
903
|
+
* ⚠️ **TWO DOC BLOCKS DESCRIBING `#retryReservationIfDue` USED TO SIT HERE — 056-SLOTDEAD, review
|
|
904
|
+
* F7.** That function is deleted. One of them said "the rebuild is the re-attempt … which is why
|
|
905
|
+
* it stays", so a reader following the comment landed on this function and concluded a rebuild
|
|
906
|
+
* ladder was still running. There is no rebuild anywhere in this file.
|
|
889
907
|
*
|
|
890
|
-
*
|
|
891
|
-
*
|
|
908
|
+
* What this does: a live session that lost its circuit re-takes it IN PLACE, on the node that
|
|
909
|
+
* already has the session, and this owns whether it is allowed to try yet. ONE ladder — a
|
|
910
|
+
* reservation is scarce, the relay holds it for its full TTL even after the client disconnects,
|
|
911
|
+
* and this file's own warning is that churning attempts across a fleet is how a relay is
|
|
912
|
+
* exhausted.
|
|
892
913
|
*/
|
|
893
|
-
#
|
|
914
|
+
#retryDue(agentName) {
|
|
894
915
|
const now = Date.now();
|
|
895
916
|
const state = this.#ctx.srReservationRetry.get(agentName)
|
|
896
917
|
?? { attempts: 0, nextAt: now + this.#ctx.srReservationRetryMs, correlationId: randomUUID() };
|
|
897
|
-
// The reason the LAST attempt was refused, captured where it is actually known.
|
|
898
918
|
const lastReason = this.#ctx.srLastRejectionReason.get(agentName);
|
|
899
919
|
if (lastReason !== undefined)
|
|
900
920
|
state.lastReason = lastReason;
|
|
901
|
-
if (
|
|
902
|
-
// First sighting — schedule, do not fire.
|
|
921
|
+
if (!this.#ctx.srReservationRetry.has(agentName)) {
|
|
922
|
+
// First sighting — schedule, do not fire. Something just tried.
|
|
903
923
|
this.#ctx.srReservationRetry.set(agentName, state);
|
|
904
|
-
return;
|
|
924
|
+
return false;
|
|
905
925
|
}
|
|
906
926
|
if (now < state.nextAt)
|
|
907
|
-
return;
|
|
927
|
+
return false;
|
|
908
928
|
if (state.attempts >= SR_RESERVATION_MAX_RETRIES) {
|
|
909
929
|
if (state.attempts === SR_RESERVATION_MAX_RETRIES) {
|
|
910
|
-
state.attempts += 1; //
|
|
930
|
+
state.attempts += 1; // report once
|
|
911
931
|
this.#ctx.srReservationRetry.set(agentName, state);
|
|
912
932
|
this.#ctx.logger.error("session.standing_receiver.reservation.gave_up", {
|
|
913
933
|
agentName,
|
|
@@ -916,51 +936,43 @@ export class SessionRelay {
|
|
|
916
936
|
// WHY, not just the consequence. Three different problems reach this one message and they
|
|
917
937
|
// need three different responses: `relay_granted_no_reservation` is relay CAPACITY (and a
|
|
918
938
|
// trustless-cello problem), `relay_unreachable` is the NETWORK, and
|
|
919
|
-
// `reservation_did_not_complete_in_time` is LATENCY
|
|
920
|
-
// can pin a slot it never uses, so its appearance is also the signal that this retry
|
|
921
|
-
// budget needs tightening.
|
|
939
|
+
// `reservation_did_not_complete_in_time` is LATENCY.
|
|
922
940
|
...(state.lastReason !== undefined ? { lastRejectionReason: state.lastReason } : {}),
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
941
|
+
/**
|
|
942
|
+
* ⚠️ **THESE TWO CAME BACK — 056-SLOTDEAD, review F9.** The deleted emitter of this SAME
|
|
943
|
+
* event name carried them; the surviving one did not, so the fields quietly vanished from
|
|
944
|
+
* an operator's view while the event kept appearing.
|
|
945
|
+
*
|
|
946
|
+
* "No relay would grant" and "there was no relay to ask" are different facts that lead to
|
|
947
|
+
* different places — the first at relay capacity, the second at this agent's directory
|
|
948
|
+
* connection. Without them they are the same sentence.
|
|
949
|
+
*
|
|
950
|
+
* `hadRelayToAsk` reads the directory pool alone; `relaysOffered` is the merged,
|
|
951
|
+
* quarantine-filtered candidate list the re-take actually walks. Two populations, so two
|
|
952
|
+
* names — the mis-naming that made `reservationsRequested` unreadable is the reason this
|
|
953
|
+
* note exists.
|
|
954
|
+
*/
|
|
933
955
|
hadRelayToAsk: (this.#ctx.directoryRelayEndpoints.get(agentName)?.length ?? 0) > 0,
|
|
934
|
-
// …and HOW MANY the walk actually asks, so this event stands on its own instead of
|
|
935
|
-
// needing the last reachability line to be read beside it. Same population and same
|
|
936
|
-
// meaning as `relaysOffered` everywhere else: the merged, quarantine-filtered candidate
|
|
937
|
-
// list.
|
|
938
956
|
relaysOffered: this.reservationCircuitAddrs(agentName).addrs.length,
|
|
939
|
-
impact: "
|
|
957
|
+
impact: "a live session lost the circuit its counterparty dials, and no relay would give " +
|
|
958
|
+
"it back inside the retry budget. Messages still reach this agent through the relay's " +
|
|
959
|
+
"store-and-forward; a direct dial to it will not connect until the session is rebuilt.",
|
|
940
960
|
});
|
|
941
961
|
}
|
|
942
|
-
return;
|
|
962
|
+
return false;
|
|
943
963
|
}
|
|
944
964
|
state.attempts += 1;
|
|
945
|
-
//
|
|
946
|
-
//
|
|
947
|
-
//
|
|
948
|
-
|
|
949
|
-
state.nextAt = now + (state.attempts >= SR_RESERVATION_MAX_RETRIES
|
|
950
|
-
? this.#ctx.srReservationRetryMs
|
|
951
|
-
: this.#ctx.srReservationRetryMs * 2 ** (state.attempts - 1));
|
|
965
|
+
// Doubling, floored at the configured interval. 056-SLOTDEAD: this said "the same shape the
|
|
966
|
+
// rebuild ladder uses" — there is no rebuild ladder any more, this IS the ladder. The reason is
|
|
967
|
+
// unchanged: a fixed short interval is what exhausts a relay.
|
|
968
|
+
state.nextAt = now + this.#ctx.srReservationRetryMs * Math.pow(2, state.attempts - 1);
|
|
952
969
|
this.#ctx.srReservationRetry.set(agentName, state);
|
|
953
|
-
this.#ctx.logger.
|
|
954
|
-
agentName,
|
|
955
|
-
attempt: state.attempts,
|
|
956
|
-
maxAttempts: SR_RESERVATION_MAX_RETRIES,
|
|
957
|
-
correlationId: state.correlationId,
|
|
958
|
-
...(state.lastReason !== undefined ? { lastRejectionReason: state.lastReason } : {}),
|
|
959
|
-
impact: "this agent currently holds no circuit reservation, so a NAT'd peer cannot dial it",
|
|
970
|
+
this.#ctx.logger.info("session.standing_receiver.reservation.retry", {
|
|
971
|
+
agentName, attempts: state.attempts, correlationId: state.correlationId,
|
|
960
972
|
});
|
|
961
|
-
|
|
973
|
+
return true;
|
|
962
974
|
}
|
|
963
|
-
#reservationWatchdogTick() {
|
|
975
|
+
async #reservationWatchdogTick() {
|
|
964
976
|
if (this.#ctx.shuttingDown)
|
|
965
977
|
return;
|
|
966
978
|
for (const [agentName, sr] of this.#ctx.standingReceivers) {
|
|
@@ -981,6 +993,66 @@ export class SessionRelay {
|
|
|
981
993
|
// full TTL even after the client disconnects, and churning attempts across a fleet is how a
|
|
982
994
|
// relay is exhausted (`#startReceiverNode` records that hazard).
|
|
983
995
|
if (sr.relayPeerIds.length === 0) {
|
|
996
|
+
/**
|
|
997
|
+
* ⚠️ **AN IDLE AGENT HOLDING NOTHING IS THE DESIGN NOW, NOT A DEGRADATION — 055-ONDEMAND.**
|
|
998
|
+
*
|
|
999
|
+
* Everything above this line was written when a receiver reserved at login and holding zero
|
|
1000
|
+
* meant an agent dialable by nobody for its whole life. A reservation is taken at OFFER time
|
|
1001
|
+
* now and given back at the seal, so zero is the correct steady state for an agent nobody is
|
|
1002
|
+
* calling — and retrying here would have every idle agent in the fleet asking relays for
|
|
1003
|
+
* slots the design says it must not hold. That is the exact churn the note below warns
|
|
1004
|
+
* about, pointed at the whole fleet instead of one receiver.
|
|
1005
|
+
*
|
|
1006
|
+
* The retry still matters for an agent that HAS a live session and lost the circuit that
|
|
1007
|
+
* session depends on, which is what the condition now says.
|
|
1008
|
+
*/
|
|
1009
|
+
const liveSessions = [...this.#ctx.activeNodes.values()].filter((e) => e.agentName === agentName);
|
|
1010
|
+
if (liveSessions.length === 0)
|
|
1011
|
+
continue;
|
|
1012
|
+
/**
|
|
1013
|
+
* ⚠️ **RE-TAKE THE SESSION'S CIRCUIT — REBUILDING THE RECEIVER NO LONGER DOES IT.**
|
|
1014
|
+
*
|
|
1015
|
+
* `#retryReservationIfDue` rebuilds the receiver, and a rebuilt receiver reserves NOTHING
|
|
1016
|
+
* (055-ONDEMAND). So on its own the retry ladder became a no-op by construction: a relay
|
|
1017
|
+
* that flapped mid-session would leave that session permanently un-dialable while the
|
|
1018
|
+
* ladder churned receivers to no effect.
|
|
1019
|
+
*
|
|
1020
|
+
* The circuit a live session needs is the one its ASSIGNMENT named, which is persisted with
|
|
1021
|
+
* the session — so ask for that one back, rather than rebuilding and hoping.
|
|
1022
|
+
*/
|
|
1023
|
+
/**
|
|
1024
|
+
* ⚠️ **ON THE SAME BUDGET AND BACKOFF AS THE LADDER IT REPLACES.** A reservation is scarce:
|
|
1025
|
+
* the relay holds it for its full TTL even after the client disconnects, and this file's own
|
|
1026
|
+
* warning is that churning attempts across a fleet is how a relay is exhausted. Asking on
|
|
1027
|
+
* every 30-second tick would be exactly that churn, wearing a new name — measured here as 52
|
|
1028
|
+
* asks where the budget allows 37.
|
|
1029
|
+
*/
|
|
1030
|
+
if (!this.#retryDue(agentName))
|
|
1031
|
+
continue;
|
|
1032
|
+
/**
|
|
1033
|
+
* ⚠️ **ON THE SESSION'S OWN NODE, NOT THE IDLE RECEIVER — and the first version got this
|
|
1034
|
+
* wrong in a way that was worse than doing nothing.**
|
|
1035
|
+
*
|
|
1036
|
+
* It called the take path, which reserves on `standingReceivers.get(agentName).node`. That
|
|
1037
|
+
* is the fresh receiver built after the promotion — a DIFFERENT peer id from the one the
|
|
1038
|
+
* counterparty was told to dial. So the circuit it obtained helped no one, and it made an
|
|
1039
|
+
* IDLE receiver hold a relay slot, which is the exact thing this unit exists to stop. It
|
|
1040
|
+
* also set `relayPeerIds` non-empty, hiding the real loss from every later tick.
|
|
1041
|
+
*/
|
|
1042
|
+
let retook = false;
|
|
1043
|
+
for (const entry of liveSessions) {
|
|
1044
|
+
const ep = this.#ctx.queries.getPersistedRelayEndpoint(agentName, entry.sessionId);
|
|
1045
|
+
if (!ep || ep.relayAddrs.length === 0)
|
|
1046
|
+
continue;
|
|
1047
|
+
const base = ep.relayAddrs[0];
|
|
1048
|
+
const circuitAddr = base.includes(`/p2p/${ep.relayPeerId}`) ? `${base}/p2p-circuit` : `${base}/p2p/${ep.relayPeerId}/p2p-circuit`;
|
|
1049
|
+
if (entry.node.listenAddresses().some((a) => a.split("/").includes("p2p-circuit")))
|
|
1050
|
+
continue; // still holds one
|
|
1051
|
+
if (await this.#ctx.retakeReservationOn(agentName, entry.node, circuitAddr, entry.correlationId))
|
|
1052
|
+
retook = true;
|
|
1053
|
+
}
|
|
1054
|
+
if (retook)
|
|
1055
|
+
continue;
|
|
984
1056
|
// …unless one has arrived since. Review F4, same class as the recompute below: the
|
|
985
1057
|
// slow-start path installs a receiver before every circuit has bound, so "held nothing at
|
|
986
1058
|
// install" is not the same fact as "holds nothing now". Adopting it here is what stops the
|
|
@@ -988,7 +1060,19 @@ export class SessionRelay {
|
|
|
988
1060
|
const arrived = heldRelayIdsOf(sr.node)
|
|
989
1061
|
.filter((id) => sr.node.getConnections().some((c) => c.peerId === id && c.status === "open"));
|
|
990
1062
|
if (arrived.length === 0) {
|
|
991
|
-
|
|
1063
|
+
/**
|
|
1064
|
+
* ⚠️ **`#retryReservationIfDue` WAS CALLED HERE AND COULD NEVER DO ANYTHING — 056-SLOTDEAD.**
|
|
1065
|
+
*
|
|
1066
|
+
* It guards on `now < state.nextAt`, and `#retryDue` above advances that same state on
|
|
1067
|
+
* this very tick, so the second call always returned immediately. Reachable, referenced,
|
|
1068
|
+
* and inert — the shape a reference scan cannot find.
|
|
1069
|
+
*
|
|
1070
|
+
* *(The first version of this order claimed it DOUBLE-ADVANCED the budget and triggered a
|
|
1071
|
+
* useless rebuild. That was wrong, and it was wrong because it was reasoned rather than
|
|
1072
|
+
* run. It was a no-op.)*
|
|
1073
|
+
*
|
|
1074
|
+
* Nothing replaces it: `#retryDue` owns the budget and the re-take above owns the work.
|
|
1075
|
+
*/
|
|
992
1076
|
continue;
|
|
993
1077
|
}
|
|
994
1078
|
sr.relayPeerIds = arrived;
|
|
@@ -1068,118 +1152,76 @@ export class SessionRelay {
|
|
|
1068
1152
|
// client for this (agent, relay) pair kept the error that ended its reader; that is the
|
|
1069
1153
|
// nearest thing to an upstream cause available here, and its absence is how 2,061 of these
|
|
1070
1154
|
// went untraced.
|
|
1071
|
-
|
|
1155
|
+
/**
|
|
1156
|
+
* ⚠️ **A DIAGNOSTIC MUST NOT BE ABLE TO KILL THE WATCHDOG TICK.** This is optional-chained on
|
|
1157
|
+
* the map lookup but the METHOD was called unguarded, so a client without it threw — an
|
|
1158
|
+
* unhandled rejection inside the tick, which takes the rest of the sweep with it. Surfaced
|
|
1159
|
+
* when 055-ONDEMAND made this branch reachable in more cases. The cause line is worth
|
|
1160
|
+
* having; it is not worth the loss detection it rides on.
|
|
1161
|
+
*/
|
|
1162
|
+
const client = this.#ctx.relayClients.get(`${agentName}::${relayPeerId}`);
|
|
1163
|
+
const upstreamReason = typeof client?.getLastReaderError === "function" ? client.getLastReaderError() : null;
|
|
1072
1164
|
this.#ctx.logger.warn("session.standing_receiver.reservation.lost", {
|
|
1073
1165
|
agentName,
|
|
1074
1166
|
relayPeerId,
|
|
1075
1167
|
reason: open.includes(relayPeerId) ? "circuit_address_vanished" : "relay_connection_gone",
|
|
1076
1168
|
...(upstreamReason ? { upstreamReason } : {}),
|
|
1077
1169
|
reservationsHeld: stillHeld.length,
|
|
1078
|
-
|
|
1170
|
+
/**
|
|
1171
|
+
* The line an operator reads, and the two cases are not the same event at all.
|
|
1172
|
+
*
|
|
1173
|
+
* ⚠️ **THE ZERO CASE PROMISED A REPAIR THAT NO LONGER HAPPENS — 056-SLOTDEAD, review F3.**
|
|
1174
|
+
* It ended "The receiver is being rebuilt against the rest of the pool", which was true
|
|
1175
|
+
* when this branch rebuilt. Nothing rebuilds now. An operator reading the old line would
|
|
1176
|
+
* wait for a recovery that was never coming, and the only thing that WOULD have told them
|
|
1177
|
+
* otherwise is a `zero_held` line at debug level they are not reading. Both branches now
|
|
1178
|
+
* say what actually happens next and who owns it.
|
|
1179
|
+
*/
|
|
1079
1180
|
impact: stillHeld.length > 0
|
|
1080
1181
|
? "this agent still holds " + stillHeld.length + " other circuit reservation(s), so it "
|
|
1081
|
-
+ "stays dialable from behind NAT
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
+ "
|
|
1182
|
+
+ "stays dialable from behind NAT. Losing one relay costs this agent nothing it can feel."
|
|
1183
|
+
: "this agent now holds NO circuit reservation. If it has no live session that is the "
|
|
1184
|
+
+ "normal idle state and costs nothing — a slot is taken when someone calls. If it "
|
|
1185
|
+
+ "DOES have a live session, that session's counterparty can no longer dial it, and "
|
|
1186
|
+
+ "the re-take above is what gets it back, on a bounded budget. Nothing rebuilds the "
|
|
1187
|
+
+ "receiver any more, so do not wait for one.",
|
|
1085
1188
|
});
|
|
1086
1189
|
}
|
|
1190
|
+
/**
|
|
1191
|
+
* ⚠️ **DRAIN ON THE LOSS ITSELF — 056-SLOTDEAD.** This used to happen one step downstream, as
|
|
1192
|
+
* a side effect of the rebuild below: lose every reservation → rebuild → the new receiver
|
|
1193
|
+
* reports ready → drain. Deleting the rebuild would have deleted the drain with it, leaving
|
|
1194
|
+
* content the counterparty parked to sit until the periodic backstop. The loss is the cause
|
|
1195
|
+
* and always was; the rebuild was only where it happened to be noticed.
|
|
1196
|
+
*/
|
|
1197
|
+
this.#ctx.park.fireParkedDrain(agentName, "reservation_lost");
|
|
1087
1198
|
if (stillHeld.length === 0) {
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1199
|
+
/**
|
|
1200
|
+
* ⚠️ **THIS REBUILT THE RECEIVER ON A JUSTIFICATION THAT STOPPED BEING TRUE IN UNIT 1, AND
|
|
1201
|
+
* UNDER ON-DEMAND IT DESTROYS AN IN-FLIGHT OFFER'S RESERVATION — 056-SLOTDEAD.**
|
|
1202
|
+
*
|
|
1203
|
+
* It read: *"only a new node can take a new reservation, because a circuit listener is fixed
|
|
1204
|
+
* at node creation."* `listenOnCircuit` removed that constraint (`DOD-M15-RELAYPROVE-ORDER-1`),
|
|
1205
|
+
* and a rebuilt receiver now reserves nothing at all.
|
|
1206
|
+
*
|
|
1207
|
+
* The state it fires in is the dangerous one: between an offer taking a reservation and the
|
|
1208
|
+
* session being created, the receiver holds exactly one circuit. Lose the connection in that
|
|
1209
|
+
* window and this rebuilt the node — discarding the slot the offer just took, after the
|
|
1210
|
+
* accept may already have advertised it. **Observed**, not reasoned: a fixture that reported
|
|
1211
|
+
* no relay connection drove this path and the test caught the rebuild.
|
|
1212
|
+
*
|
|
1213
|
+
* Nothing replaces it here. An idle agent holding zero is the design; a LIVE session that
|
|
1214
|
+
* lost its circuit is re-taken on its own node by the branch above, on a bounded budget.
|
|
1215
|
+
*/
|
|
1216
|
+
this.#ctx.logger.debug("session.standing_receiver.reservation.zero_held", {
|
|
1217
|
+
agentName,
|
|
1218
|
+
impact: "this agent holds no circuit. If it has no live session that is the idle steady " +
|
|
1219
|
+
"state; if it does, the re-take above owns getting it back.",
|
|
1220
|
+
});
|
|
1092
1221
|
continue;
|
|
1093
1222
|
}
|
|
1094
|
-
/**
|
|
1095
|
-
* STILL REACHABLE, SO THE RECEIVER STANDS, AND NOTHING ELSE HAPPENS HERE. That second half is
|
|
1096
|
-
* the part worth reading, because the obvious next line is wrong twice over.
|
|
1097
|
-
*
|
|
1098
|
-
* **A LOST CONFIGURED CIRCUIT CANNOT BE RETAKEN BY THIS NODE.** Read out of
|
|
1099
|
-
* `@libp2p/circuit-relay-v2@4.2.5`, not assumed: for an explicit relay address
|
|
1100
|
-
* `transport/listener.js#listen()` is a ONE-SHOT — it reserves once and nothing calls it
|
|
1101
|
-
* again; `reservation-store.js#removeReservation()` clears the refresh timeout and deletes
|
|
1102
|
-
* the entry; and the listener's `_onAddRelayPeer` returns early for `type === 'configured'`,
|
|
1103
|
-
* so even a later reservation would not be announced. A circuit listener is fixed at node
|
|
1104
|
-
* creation, and the only thing that takes a new one is a NEW NODE — which is exactly the
|
|
1105
|
-
* rebuild this branch exists to refuse.
|
|
1106
|
-
*
|
|
1107
|
-
* **AND RE-PROVING TO THE LOST RELAY WOULD REBUILD THE RECEIVER ANYWAY.** Review F3: an
|
|
1108
|
-
* earlier version called `authenticateStandingReceiver` here to "remove the relay-side
|
|
1109
|
-
* reason for the revocation". That function ends with `if (refusal?.tryAnotherRelay) { …
|
|
1110
|
-
* void this.#ctx.receivers.rebuildStandingReceiver(agentName); }` — and a dead or misconfigured relay is
|
|
1111
|
-
* precisely the one that answers that way. So the common case was: lose relay A while
|
|
1112
|
-
* holding B, decline to rebuild, prove to A, A refuses, rebuild the whole receiver and throw
|
|
1113
|
-
* B's healthy reservation away. The churn engine, re-entered through the back door.
|
|
1114
|
-
*
|
|
1115
|
-
* **THE BOUND, STATED PLAINLY BECAUSE IT IS A REAL SHORTFALL AGAINST THE DoD:** a lost
|
|
1116
|
-
* circuit is gone until the receiver is next rebuilt for another reason. What the agent buys
|
|
1117
|
-
* is that it never STOPS BEING REACHABLE while that is true — the surviving relays carry it,
|
|
1118
|
-
* the loss is named in the log with its cause, and the lost relay's inbound carve-out is
|
|
1119
|
-
* revoked above. That is availability, not restoration in place.
|
|
1120
|
-
*
|
|
1121
|
-
* WHICH LEAVES A RATCHET, and `#respreadIfDecayed` below is what stops it: relays are only
|
|
1122
|
-
* ever lost between rebuilds, never regained, so an agent nobody talks to walks itself back
|
|
1123
|
-
* down to one relay — the exact state this unit exists to get it out of.
|
|
1124
|
-
*/
|
|
1125
|
-
}
|
|
1126
|
-
for (const agentName of this.#ctx.standingReceivers.keys()) {
|
|
1127
|
-
if (this.#ctx.agentsWantingReceiver.has(agentName))
|
|
1128
|
-
this.#respreadIfDecayed(agentName);
|
|
1129
1223
|
}
|
|
1130
1224
|
}
|
|
1131
|
-
/**
|
|
1132
|
-
* 032-RELAYSPREAD — **AN IDLE AGENT MUST NOT RATCHET ITSELF BACK DOWN TO ONE RELAY.**
|
|
1133
|
-
*
|
|
1134
|
-
* Spreading happens when a receiver is BUILT, and between builds the count only falls: a lost
|
|
1135
|
-
* circuit cannot be retaken by a running node (a circuit listener is fixed at node creation), and
|
|
1136
|
-
* a relay the directory announces later is skipped while any circuit is held. An agent in
|
|
1137
|
-
* conversation re-spreads constantly — the receiver is handed into each session and a fresh one
|
|
1138
|
-
* is built behind it — so this is about the agent nobody has talked to for a day. It loses relays
|
|
1139
|
-
* one at a time, nothing pulls it back up, and it ends up exactly where this unit found it:
|
|
1140
|
-
* reachable through one relay, one relay away from being reachable through none.
|
|
1141
|
-
*
|
|
1142
|
-
* **THE COST OF FIXING IT IS A NEW PEER ID**, which is why it is fenced three ways rather than
|
|
1143
|
-
* simply rebuilding on sight:
|
|
1144
|
-
* - **ONLY WHEN IDLE.** A rebuild replaces the receiver's transport identity, and a counterparty
|
|
1145
|
-
* may be holding the old one from a `session_offer_accept`. With a live session for this agent
|
|
1146
|
-
* we leave it alone — a degraded spread costs redundancy, a changed peer id mid-conversation
|
|
1147
|
-
* costs the conversation.
|
|
1148
|
-
* - **ONLY WHEN THERE IS SOMETHING TO GAIN.** Holding every relay that was offered is not decay.
|
|
1149
|
-
* - **ON ITS OWN SLOW CLOCK**, never the watchdog's 30-second grid. A reservation is scarce —
|
|
1150
|
-
* the relay holds it for its full TTL even after we disconnect — so this reuses the
|
|
1151
|
-
* reservation retry interval rather than inventing a faster one.
|
|
1152
|
-
*/
|
|
1153
|
-
#respreadIfDecayed(agentName) {
|
|
1154
|
-
if (this.#ctx.shuttingDown)
|
|
1155
|
-
return;
|
|
1156
|
-
const sr = this.#ctx.standingReceivers.get(agentName);
|
|
1157
|
-
if (!sr || sr.relayPeerIds.length === 0)
|
|
1158
|
-
return; // zero held is the loud path
|
|
1159
|
-
for (const entry of this.#ctx.activeNodes.values()) {
|
|
1160
|
-
if (entry.agentName === agentName)
|
|
1161
|
-
return; // in conversation — hands off
|
|
1162
|
-
}
|
|
1163
|
-
const offered = this.reservationCircuitAddrs(agentName).addrs.length;
|
|
1164
|
-
if (sr.relayPeerIds.length >= offered)
|
|
1165
|
-
return; // nothing to gain
|
|
1166
|
-
const now = Date.now();
|
|
1167
|
-
const last = this.#ctx.srLastRespreadAt.get(agentName) ?? 0;
|
|
1168
|
-
if (now - last < this.#ctx.srReservationRetryMs)
|
|
1169
|
-
return;
|
|
1170
|
-
this.#ctx.srLastRespreadAt.set(agentName, now);
|
|
1171
|
-
this.#ctx.logger.info("session.standing_receiver.respread", {
|
|
1172
|
-
agentName,
|
|
1173
|
-
reservationsHeld: sr.relayPeerIds.length,
|
|
1174
|
-
relaysOffered: offered,
|
|
1175
|
-
impact: "this agent is idle and holds fewer relay reservations than it was offered, so its " +
|
|
1176
|
-
"receiver is being rebuilt to take the rest. Without this it can only lose relays between " +
|
|
1177
|
-
"rebuilds, and an agent nobody talks to drifts back down to a single relay — one relay " +
|
|
1178
|
-
"away from being unreachable behind NAT, which is the state this whole mechanism exists " +
|
|
1179
|
-
"to keep it out of.",
|
|
1180
|
-
});
|
|
1181
|
-
void this.#ctx.receivers.rebuildStandingReceiver(agentName);
|
|
1182
|
-
}
|
|
1183
1225
|
/** Start the reservation watchdog (idempotent). Stopped by gracefulShutdown. */
|
|
1184
1226
|
startReservationWatchdog() {
|
|
1185
1227
|
if (this.#ctx.reservationWatchdog !== null)
|
|
@@ -1189,7 +1231,7 @@ export class SessionRelay {
|
|
|
1189
1231
|
this.#ctx.park.armBackstopClock(Date.now());
|
|
1190
1232
|
this.#ctx.reservationWatchdog = setInterval(() => {
|
|
1191
1233
|
try {
|
|
1192
|
-
this.#reservationWatchdogTick();
|
|
1234
|
+
void this.#reservationWatchdogTick();
|
|
1193
1235
|
this.#ctx.park.parkedDrainBackstopTick(Date.now());
|
|
1194
1236
|
}
|
|
1195
1237
|
catch (err) {
|
|
@@ -1248,6 +1290,61 @@ export class SessionRelay {
|
|
|
1248
1290
|
* `tryAnotherRelay: false` precisely so the client STOPS walking the fleet; without the verdict,
|
|
1249
1291
|
* the loop walked it anyway, turning one client-side fault into what reads as a fleet outage.
|
|
1250
1292
|
*/
|
|
1293
|
+
/**
|
|
1294
|
+
* 055-ONDEMAND — **tell the relay this agent has finished with its slot.**
|
|
1295
|
+
*
|
|
1296
|
+
* The only thing that actually frees one: closing a circuit listener sends the relay nothing, and
|
|
1297
|
+
* the relay reclaims on its own only at the reservation TTL (two hours) or under reaper pressure.
|
|
1298
|
+
* Best-effort by design — an undelivered release costs a slot until that TTL, and must never fail
|
|
1299
|
+
* the seal that triggered it.
|
|
1300
|
+
*/
|
|
1301
|
+
async tellRelayReleased(agentName, relayPeerId, node, correlationId) {
|
|
1302
|
+
/**
|
|
1303
|
+
* ⚠️ **THE NODE IS THE SOURCE OF TRUTH FOR WHERE THIS RELAY IS — review MEDIUM-6.**
|
|
1304
|
+
*
|
|
1305
|
+
* The first version looked the relay up in `directoryRelayEndpoints`. But the reservation was
|
|
1306
|
+
* taken on the relay the OFFER named, whose addresses came off that frame; if the directory's
|
|
1307
|
+
* last announcement does not contain it — a pool relay dropped from the roster, a stale
|
|
1308
|
+
* announcement — the release warns `no_endpoint` and the slot is held to its TTL. The node
|
|
1309
|
+
* announces the circuit it actually holds, so derive the address from that and fall back to the
|
|
1310
|
+
* directory list only when there is nothing to derive from.
|
|
1311
|
+
*/
|
|
1312
|
+
const fromNode = node.listenAddresses()
|
|
1313
|
+
.filter((a) => a.split("/").includes("p2p-circuit") && a.includes(`/p2p/${relayPeerId}/`))
|
|
1314
|
+
.map((a) => a.split("/p2p-circuit")[0])
|
|
1315
|
+
.filter((a) => a.length > 0);
|
|
1316
|
+
const ep = this.#ctx.directoryRelayEndpoints.get(agentName)?.find((e) => e.relayPeerId === relayPeerId);
|
|
1317
|
+
const relayAddrs = fromNode.length > 0 ? [...new Set(fromNode)] : (ep ? [...ep.relayAddrs] : []);
|
|
1318
|
+
if (relayAddrs.length === 0) {
|
|
1319
|
+
this.#ctx.logger.warn("session.relay.reservation_release.no_endpoint", {
|
|
1320
|
+
agentName, relayPeerId, correlationId,
|
|
1321
|
+
impact: "no address is known for this relay, so it cannot be told the slot is free and " +
|
|
1322
|
+
"holds it until the reservation TTL expires.",
|
|
1323
|
+
});
|
|
1324
|
+
return;
|
|
1325
|
+
}
|
|
1326
|
+
let client;
|
|
1327
|
+
try {
|
|
1328
|
+
client = this.#ctx.detachedRelayClientBuilder?.(agentName, relayPeerId, relayAddrs, {
|
|
1329
|
+
receiptStore: this.#ctx.relayReceiptStore ?? undefined,
|
|
1330
|
+
sealLeafStore: this.#ctx.sealLeafStore ?? undefined,
|
|
1331
|
+
onlineToken: () => this.#ctx.getDirectoryOnlineToken(agentName),
|
|
1332
|
+
});
|
|
1333
|
+
if (!client)
|
|
1334
|
+
return;
|
|
1335
|
+
await client.releaseReservation(node);
|
|
1336
|
+
}
|
|
1337
|
+
catch (err) {
|
|
1338
|
+
this.#ctx.logger.warn("session.relay.reservation_release.failed", {
|
|
1339
|
+
agentName, relayPeerId, correlationId,
|
|
1340
|
+
error: extractErrorMessage(err),
|
|
1341
|
+
impact: "the relay holds this slot until its TTL expires.",
|
|
1342
|
+
});
|
|
1343
|
+
}
|
|
1344
|
+
finally {
|
|
1345
|
+
client?.close();
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1251
1348
|
async proveToRelay(agentName, circuitAddr, node, correlationId,
|
|
1252
1349
|
/**
|
|
1253
1350
|
* Whether this proof is the STANDING RECEIVER's, and may therefore write the surface
|
|
@@ -1304,12 +1401,9 @@ export class SessionRelay {
|
|
|
1304
1401
|
});
|
|
1305
1402
|
return "proven";
|
|
1306
1403
|
}
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
* leave `getLastAuthRefusal()` null, and leaving a PREVIOUS refusal in the map would have
|
|
1311
|
-
* `cello_status` explaining a cause that is no longer what is wrong.
|
|
1312
|
-
*/
|
|
1404
|
+
// ⚠️ A NULL REFUSAL IS A TRANSPORT FAILURE, NOT A VERDICT (review HIGH-1): set only on a
|
|
1405
|
+
// refusal ON THE MERITS, so the `else` clears a stale one `cello_status` would misreport, and
|
|
1406
|
+
// the RETURN below says `unavailable` — the caller decides whether to ask on it.
|
|
1313
1407
|
const refusal = client.getLastAuthRefusal();
|
|
1314
1408
|
if (surfaceAsReceiverRefusal) {
|
|
1315
1409
|
if (refusal) {
|
|
@@ -1328,14 +1422,16 @@ export class SessionRelay {
|
|
|
1328
1422
|
tryAnotherRelay: refusal?.tryAnotherRelay ?? true,
|
|
1329
1423
|
correlationId,
|
|
1330
1424
|
impact: refusal?.advice ??
|
|
1331
|
-
"the relay
|
|
1332
|
-
"
|
|
1425
|
+
"the relay said nothing about why, which is what a transport failure mid-handshake looks " +
|
|
1426
|
+
"like. No verdict was reached, so the reservation is still asked for.",
|
|
1333
1427
|
});
|
|
1334
1428
|
if (refusal && !refusal.tryAnotherRelay)
|
|
1335
1429
|
return "refused_this_agent";
|
|
1336
1430
|
if (refusal?.tryAnotherRelay && !this.#ctx.shuttingDown) {
|
|
1337
1431
|
this.#quarantineRelay(agentName, relayPeerId, refusal.reason);
|
|
1338
1432
|
}
|
|
1433
|
+
if (!refusal)
|
|
1434
|
+
return "unavailable"; // no verdict — see the refusal note above.
|
|
1339
1435
|
return "refused_try_another_relay";
|
|
1340
1436
|
}
|
|
1341
1437
|
catch (err) {
|