@cello-protocol/daemon 0.0.208 → 0.0.209

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist/agent-handlers.d.ts.map +1 -1
  2. package/dist/agent-handlers.js +2 -1
  3. package/dist/agent-handlers.js.map +1 -1
  4. package/dist/attendance-wiring.js +2 -2
  5. package/dist/attendance-wiring.js.map +1 -1
  6. package/dist/boot-agents.d.ts.map +1 -1
  7. package/dist/boot-agents.js +5 -5
  8. package/dist/boot-agents.js.map +1 -1
  9. package/dist/close-session-handler.d.ts.map +1 -1
  10. package/dist/close-session-handler.js +26 -3
  11. package/dist/close-session-handler.js.map +1 -1
  12. package/dist/content-park.d.ts.map +1 -1
  13. package/dist/content-park.js +5 -3
  14. package/dist/content-park.js.map +1 -1
  15. package/dist/outbound-sessions.d.ts.map +1 -1
  16. package/dist/outbound-sessions.js +28 -0
  17. package/dist/outbound-sessions.js.map +1 -1
  18. package/dist/park-recovery.d.ts +3 -2
  19. package/dist/park-recovery.d.ts.map +1 -1
  20. package/dist/park-recovery.js +3 -2
  21. package/dist/park-recovery.js.map +1 -1
  22. package/dist/refusal-reasons.d.ts +46 -0
  23. package/dist/refusal-reasons.d.ts.map +1 -1
  24. package/dist/refusal-reasons.js +44 -0
  25. package/dist/refusal-reasons.js.map +1 -1
  26. package/dist/relay-only.d.ts +9 -2
  27. package/dist/relay-only.d.ts.map +1 -1
  28. package/dist/relay-only.js +9 -2
  29. package/dist/relay-only.js.map +1 -1
  30. package/dist/restart-seal-resolver.d.ts.map +1 -1
  31. package/dist/restart-seal-resolver.js +13 -0
  32. package/dist/restart-seal-resolver.js.map +1 -1
  33. package/dist/session-ceremony.d.ts +17 -0
  34. package/dist/session-ceremony.d.ts.map +1 -1
  35. package/dist/session-ceremony.js +77 -1
  36. package/dist/session-ceremony.js.map +1 -1
  37. package/dist/session-content-send.js +12 -11
  38. package/dist/session-content-send.js.map +1 -1
  39. package/dist/session-lifecycle.d.ts.map +1 -1
  40. package/dist/session-lifecycle.js +36 -0
  41. package/dist/session-lifecycle.js.map +1 -1
  42. package/dist/session-node-factory.js +4 -4
  43. package/dist/session-node-factory.js.map +1 -1
  44. package/dist/session-node-manager.d.ts +7 -2
  45. package/dist/session-node-manager.d.ts.map +1 -1
  46. package/dist/session-node-manager.js +22 -6
  47. package/dist/session-node-manager.js.map +1 -1
  48. package/dist/session-node-types.d.ts +40 -13
  49. package/dist/session-node-types.d.ts.map +1 -1
  50. package/dist/session-node-types.js +18 -0
  51. package/dist/session-node-types.js.map +1 -1
  52. package/dist/session-relay-client.d.ts +21 -0
  53. package/dist/session-relay-client.d.ts.map +1 -1
  54. package/dist/session-relay-client.js +219 -3
  55. package/dist/session-relay-client.js.map +1 -1
  56. package/dist/session-relay.d.ts +15 -1
  57. package/dist/session-relay.d.ts.map +1 -1
  58. package/dist/session-relay.js +257 -161
  59. package/dist/session-relay.js.map +1 -1
  60. package/dist/signaling-connect.d.ts.map +1 -1
  61. package/dist/signaling-connect.js +23 -4
  62. package/dist/signaling-connect.js.map +1 -1
  63. package/dist/signaling-wiring.js +7 -7
  64. package/dist/signaling-wiring.js.map +1 -1
  65. package/dist/standing-receivers.d.ts +53 -14
  66. package/dist/standing-receivers.d.ts.map +1 -1
  67. package/dist/standing-receivers.js +551 -544
  68. package/dist/standing-receivers.js.map +1 -1
  69. package/dist/trust-signal-sweep-tick.d.ts +15 -0
  70. package/dist/trust-signal-sweep-tick.d.ts.map +1 -1
  71. package/dist/trust-signal-sweep-tick.js +19 -2
  72. package/dist/trust-signal-sweep-tick.js.map +1 -1
  73. package/dist/trust-signal-sweep.d.ts +16 -1
  74. package/dist/trust-signal-sweep.d.ts.map +1 -1
  75. package/dist/trust-signal-sweep.js +12 -10
  76. package/dist/trust-signal-sweep.js.map +1 -1
  77. package/dist/types.d.ts +18 -1
  78. package/dist/types.d.ts.map +1 -1
  79. package/dist/types.js.map +1 -1
  80. package/package.json +5 -5
@@ -130,8 +130,15 @@ export class SessionRelay {
130
130
  * what looks like a fleet-wide outage.
131
131
  */
132
132
  if (refusal?.tryAnotherRelay && !this.#ctx.shuttingDown) {
133
+ /**
134
+ * ⚠️ **THE QUARANTINE ACTS; THE REBUILD USED TO AND NO LONGER CAN — 056-SLOTDEAD.**
135
+ *
136
+ * This used to `rebuildStandingReceiver` so the new node would reserve against the rest of
137
+ * the pool. A rebuilt receiver reserves NOTHING now (055-ONDEMAND), so the rebuild bought
138
+ * nothing and cost the agent its transport identity. The quarantine is the half that still
139
+ * does the work: the next offer reserves against the pool minus this relay.
140
+ */
133
141
  this.#quarantineRelay(agentName, relayPeerId, refusal.reason);
134
- void this.#ctx.receivers.rebuildStandingReceiver(agentName);
135
142
  }
136
143
  }
137
144
  else {
@@ -742,18 +749,20 @@ export class SessionRelay {
742
749
  this.#ctx.srReservationRetry.delete(agentName);
743
750
  this.#ctx.srLastRejectionReason.delete(agentName);
744
751
  }
745
- if (endpoints.length === 0 || this.#ctx.shuttingDown)
746
- return;
747
- const sr = this.#ctx.standingReceivers.get(agentName);
748
- if (!sr)
749
- return; // not ensured yet the coming ensure reads the map
750
- if (sr.node.listenAddresses().some((a) => a.includes("/p2p-circuit")))
751
- return; // already reserved
752
- this.#ctx.logger.info("session.standing_receiver.reservation.rebuild", {
753
- agentName,
754
- relayPeerIds: endpoints.map((e) => e.relayPeerId),
755
- });
756
- void this.#ctx.receivers.rebuildStandingReceiver(agentName);
752
+ /**
753
+ * ⚠️ **ENDPOINTS ARRIVING NO LONGER REBUILD THE RECEIVER — 055-ONDEMAND.**
754
+ *
755
+ * This used to rebuild so the new node could reserve with the relays that had just been
756
+ * announced, because reservations were fixed at node creation and acquired at login. Neither is
757
+ * true now: nothing is reserved at login, and a running node can take one on demand.
758
+ *
759
+ * So a rebuild here would buy nothing and cost something real. The receiver it replaces may be
760
+ * holding the circuit a LIVE SESSION's counterparty was told to dial — throwing that away in
761
+ * response to a routine directory announcement would drop the route silently, at both ends.
762
+ *
763
+ * The endpoints are still recorded above, which is all they were ever needed for: they are the
764
+ * candidate list an offer reserves against.
765
+ */
757
766
  }
758
767
  /**
759
768
  * Is this agent currently skipping this relay? The observable half of the failover decision — a
@@ -880,35 +889,45 @@ export class SessionRelay {
880
889
  * That is precisely the silent-loss-of-inbound failure this whole story exists to
881
890
  * kill, so it cannot be left to chance: we watch for it and re-pick a relay.
882
891
  *
883
- * Only receivers that HAD a reservation are watched. One that never got one is
884
- * already degraded and already loud (reservation.none / reservation.timeout);
885
- * rebuilding it on a timer would just thrash against relays we know are refusing.
892
+ * ⚠️ 056-SLOTDEAD: this paragraph used to end "Only receivers that HAD a reservation are watched.
893
+ * One that never got one is already degraded and already loud (reservation.none /
894
+ * reservation.timeout); rebuilding it on a timer would just thrash against relays we know are
895
+ * refusing." Every clause of that is now false — an idle receiver holds nothing BY DESIGN so
896
+ * "degraded" is the normal state, `reservation.none` is not emitted at build any more, and
897
+ * nothing rebuilds. What is watched is a receiver whose recorded set has shrunk, and what
898
+ * happens is a re-take in place for a LIVE session, never a rebuild.
886
899
  */
887
900
  /**
888
- * DOD-M12B-RESERVATION-RETRY-1 ask again for a reservation the relay refused.
901
+ * DOD-M12B-RESERVATION-RETRY-1 / 055-ONDEMAND is a re-attempt due, and claim it if so.
902
+ *
903
+ * ⚠️ **TWO DOC BLOCKS DESCRIBING `#retryReservationIfDue` USED TO SIT HERE — 056-SLOTDEAD, review
904
+ * F7.** That function is deleted. One of them said "the rebuild is the re-attempt … which is why
905
+ * it stays", so a reader following the comment landed on this function and concluded a rebuild
906
+ * ladder was still running. There is no rebuild anywhere in this file.
889
907
  *
890
- * The rebuild is the re-attempt. ⚠️ It is a CHOICE, not the only option: its old justification
891
- * ("a circuit listener is fixed at node creation") stopped being true when `listenOnCircuit`
892
- * landed. A fresh receiver re-runs the whole walk, proof included, which is why it stays.
908
+ * What this does: a live session that lost its circuit re-takes it IN PLACE, on the node that
909
+ * already has the session, and this owns whether it is allowed to try yet. ONE ladder a
910
+ * reservation is scarce, the relay holds it for its full TTL even after the client disconnects,
911
+ * and this file's own warning is that churning attempts across a fleet is how a relay is
912
+ * exhausted.
893
913
  */
894
- #retryReservationIfDue(agentName) {
914
+ #retryDue(agentName) {
895
915
  const now = Date.now();
896
916
  const state = this.#ctx.srReservationRetry.get(agentName)
897
917
  ?? { attempts: 0, nextAt: now + this.#ctx.srReservationRetryMs, correlationId: randomUUID() };
898
- // The reason the LAST attempt was refused, captured where it is actually known.
899
918
  const lastReason = this.#ctx.srLastRejectionReason.get(agentName);
900
919
  if (lastReason !== undefined)
901
920
  state.lastReason = lastReason;
902
- if (state.attempts === 0 && !this.#ctx.srReservationRetry.has(agentName)) {
903
- // First sighting — schedule, do not fire. The creation attempt just happened.
921
+ if (!this.#ctx.srReservationRetry.has(agentName)) {
922
+ // First sighting — schedule, do not fire. Something just tried.
904
923
  this.#ctx.srReservationRetry.set(agentName, state);
905
- return;
924
+ return false;
906
925
  }
907
926
  if (now < state.nextAt)
908
- return;
927
+ return false;
909
928
  if (state.attempts >= SR_RESERVATION_MAX_RETRIES) {
910
929
  if (state.attempts === SR_RESERVATION_MAX_RETRIES) {
911
- state.attempts += 1; // mark as reported, so this fires exactly once
930
+ state.attempts += 1; // report once
912
931
  this.#ctx.srReservationRetry.set(agentName, state);
913
932
  this.#ctx.logger.error("session.standing_receiver.reservation.gave_up", {
914
933
  agentName,
@@ -917,51 +936,43 @@ export class SessionRelay {
917
936
  // WHY, not just the consequence. Three different problems reach this one message and they
918
937
  // need three different responses: `relay_granted_no_reservation` is relay CAPACITY (and a
919
938
  // trustless-cello problem), `relay_unreachable` is the NETWORK, and
920
- // `reservation_did_not_complete_in_time` is LATENCY — and the only one of the three that
921
- // can pin a slot it never uses, so its appearance is also the signal that this retry
922
- // budget needs tightening.
939
+ // `reservation_did_not_complete_in_time` is LATENCY.
923
940
  ...(state.lastReason !== undefined ? { lastRejectionReason: state.lastReason } : {}),
924
- // "No relay would grant" and "there was no relay to ask" are different facts and lead to
925
- // different placesthe first at relay capacity, the second at this agent's directory
926
- // connection. Without this they are the same sentence.
927
- //
928
- // 032-RELAYSPREAD: this was also called `reservationsRequested` — the same mis-naming as
929
- // the reachability events, in its worst form, because here the value is a BOOLEAN under a
930
- // name that reads as a count. It is NOT `relaysOffered`: that field counts the merged
931
- // candidate list the walk actually asks (directory pool + persisted endpoints, minus
932
- // quarantine), and this reads the directory pool alone. Two populations must not share
933
- // one field name, so this one is named for what it measures.
941
+ /**
942
+ * ⚠️ **THESE TWO CAME BACK 056-SLOTDEAD, review F9.** The deleted emitter of this SAME
943
+ * event name carried them; the surviving one did not, so the fields quietly vanished from
944
+ * an operator's view while the event kept appearing.
945
+ *
946
+ * "No relay would grant" and "there was no relay to ask" are different facts that lead to
947
+ * different places the first at relay capacity, the second at this agent's directory
948
+ * connection. Without them they are the same sentence.
949
+ *
950
+ * `hadRelayToAsk` reads the directory pool alone; `relaysOffered` is the merged,
951
+ * quarantine-filtered candidate list the re-take actually walks. Two populations, so two
952
+ * names — the mis-naming that made `reservationsRequested` unreadable is the reason this
953
+ * note exists.
954
+ */
934
955
  hadRelayToAsk: (this.#ctx.directoryRelayEndpoints.get(agentName)?.length ?? 0) > 0,
935
- // …and HOW MANY the walk actually asks, so this event stands on its own instead of
936
- // needing the last reachability line to be read beside it. Same population and same
937
- // meaning as `relaysOffered` everywhere else: the merged, quarantine-filtered candidate
938
- // list.
939
956
  relaysOffered: this.reservationCircuitAddrs(agentName).addrs.length,
940
- impact: "no relay would grant this agent a circuit reservation, so anyone behind NAT cannot reach or dial it — inbound sessions will only arrive from peers that can connect directly, and everything else falls back to the relay's store-and-forward",
957
+ impact: "a live session lost the circuit its counterparty dials, and no relay would give " +
958
+ "it back inside the retry budget. Messages still reach this agent through the relay's " +
959
+ "store-and-forward; a direct dial to it will not connect until the session is rebuilt.",
941
960
  });
942
961
  }
943
- return;
962
+ return false;
944
963
  }
945
964
  state.attempts += 1;
946
- // The FINAL attempt gets a fixed settle window rather than another doubled wait: at the top of
947
- // the ladder that would be 80 minutes of silence after the last thing we did, which is a long
948
- // time to tell an operator nothing. Every earlier attempt doubles, which is what keeps a fleet
949
- // off a scarce relay.
950
- state.nextAt = now + (state.attempts >= SR_RESERVATION_MAX_RETRIES
951
- ? this.#ctx.srReservationRetryMs
952
- : this.#ctx.srReservationRetryMs * 2 ** (state.attempts - 1));
965
+ // Doubling, floored at the configured interval. 056-SLOTDEAD: this said "the same shape the
966
+ // rebuild ladder uses" there is no rebuild ladder any more, this IS the ladder. The reason is
967
+ // unchanged: a fixed short interval is what exhausts a relay.
968
+ state.nextAt = now + this.#ctx.srReservationRetryMs * Math.pow(2, state.attempts - 1);
953
969
  this.#ctx.srReservationRetry.set(agentName, state);
954
- this.#ctx.logger.warn("session.standing_receiver.reservation.retry", {
955
- agentName,
956
- attempt: state.attempts,
957
- maxAttempts: SR_RESERVATION_MAX_RETRIES,
958
- correlationId: state.correlationId,
959
- ...(state.lastReason !== undefined ? { lastRejectionReason: state.lastReason } : {}),
960
- impact: "this agent currently holds no circuit reservation, so a NAT'd peer cannot dial it",
970
+ this.#ctx.logger.info("session.standing_receiver.reservation.retry", {
971
+ agentName, attempts: state.attempts, correlationId: state.correlationId,
961
972
  });
962
- void this.#ctx.receivers.rebuildStandingReceiver(agentName);
973
+ return true;
963
974
  }
964
- #reservationWatchdogTick() {
975
+ async #reservationWatchdogTick() {
965
976
  if (this.#ctx.shuttingDown)
966
977
  return;
967
978
  for (const [agentName, sr] of this.#ctx.standingReceivers) {
@@ -982,6 +993,66 @@ export class SessionRelay {
982
993
  // full TTL even after the client disconnects, and churning attempts across a fleet is how a
983
994
  // relay is exhausted (`#startReceiverNode` records that hazard).
984
995
  if (sr.relayPeerIds.length === 0) {
996
+ /**
997
+ * ⚠️ **AN IDLE AGENT HOLDING NOTHING IS THE DESIGN NOW, NOT A DEGRADATION — 055-ONDEMAND.**
998
+ *
999
+ * Everything above this line was written when a receiver reserved at login and holding zero
1000
+ * meant an agent dialable by nobody for its whole life. A reservation is taken at OFFER time
1001
+ * now and given back at the seal, so zero is the correct steady state for an agent nobody is
1002
+ * calling — and retrying here would have every idle agent in the fleet asking relays for
1003
+ * slots the design says it must not hold. That is the exact churn the note below warns
1004
+ * about, pointed at the whole fleet instead of one receiver.
1005
+ *
1006
+ * The retry still matters for an agent that HAS a live session and lost the circuit that
1007
+ * session depends on, which is what the condition now says.
1008
+ */
1009
+ const liveSessions = [...this.#ctx.activeNodes.values()].filter((e) => e.agentName === agentName);
1010
+ if (liveSessions.length === 0)
1011
+ continue;
1012
+ /**
1013
+ * ⚠️ **RE-TAKE THE SESSION'S CIRCUIT — REBUILDING THE RECEIVER NO LONGER DOES IT.**
1014
+ *
1015
+ * `#retryReservationIfDue` rebuilds the receiver, and a rebuilt receiver reserves NOTHING
1016
+ * (055-ONDEMAND). So on its own the retry ladder became a no-op by construction: a relay
1017
+ * that flapped mid-session would leave that session permanently un-dialable while the
1018
+ * ladder churned receivers to no effect.
1019
+ *
1020
+ * The circuit a live session needs is the one its ASSIGNMENT named, which is persisted with
1021
+ * the session — so ask for that one back, rather than rebuilding and hoping.
1022
+ */
1023
+ /**
1024
+ * ⚠️ **ON THE SAME BUDGET AND BACKOFF AS THE LADDER IT REPLACES.** A reservation is scarce:
1025
+ * the relay holds it for its full TTL even after the client disconnects, and this file's own
1026
+ * warning is that churning attempts across a fleet is how a relay is exhausted. Asking on
1027
+ * every 30-second tick would be exactly that churn, wearing a new name — measured here as 52
1028
+ * asks where the budget allows 37.
1029
+ */
1030
+ if (!this.#retryDue(agentName))
1031
+ continue;
1032
+ /**
1033
+ * ⚠️ **ON THE SESSION'S OWN NODE, NOT THE IDLE RECEIVER — and the first version got this
1034
+ * wrong in a way that was worse than doing nothing.**
1035
+ *
1036
+ * It called the take path, which reserves on `standingReceivers.get(agentName).node`. That
1037
+ * is the fresh receiver built after the promotion — a DIFFERENT peer id from the one the
1038
+ * counterparty was told to dial. So the circuit it obtained helped no one, and it made an
1039
+ * IDLE receiver hold a relay slot, which is the exact thing this unit exists to stop. It
1040
+ * also set `relayPeerIds` non-empty, hiding the real loss from every later tick.
1041
+ */
1042
+ let retook = false;
1043
+ for (const entry of liveSessions) {
1044
+ const ep = this.#ctx.queries.getPersistedRelayEndpoint(agentName, entry.sessionId);
1045
+ if (!ep || ep.relayAddrs.length === 0)
1046
+ continue;
1047
+ const base = ep.relayAddrs[0];
1048
+ const circuitAddr = base.includes(`/p2p/${ep.relayPeerId}`) ? `${base}/p2p-circuit` : `${base}/p2p/${ep.relayPeerId}/p2p-circuit`;
1049
+ if (entry.node.listenAddresses().some((a) => a.split("/").includes("p2p-circuit")))
1050
+ continue; // still holds one
1051
+ if (await this.#ctx.retakeReservationOn(agentName, entry.node, circuitAddr, entry.correlationId))
1052
+ retook = true;
1053
+ }
1054
+ if (retook)
1055
+ continue;
985
1056
  // …unless one has arrived since. Review F4, same class as the recompute below: the
986
1057
  // slow-start path installs a receiver before every circuit has bound, so "held nothing at
987
1058
  // install" is not the same fact as "holds nothing now". Adopting it here is what stops the
@@ -989,7 +1060,19 @@ export class SessionRelay {
989
1060
  const arrived = heldRelayIdsOf(sr.node)
990
1061
  .filter((id) => sr.node.getConnections().some((c) => c.peerId === id && c.status === "open"));
991
1062
  if (arrived.length === 0) {
992
- this.#retryReservationIfDue(agentName);
1063
+ /**
1064
+ * ⚠️ **`#retryReservationIfDue` WAS CALLED HERE AND COULD NEVER DO ANYTHING — 056-SLOTDEAD.**
1065
+ *
1066
+ * It guards on `now < state.nextAt`, and `#retryDue` above advances that same state on
1067
+ * this very tick, so the second call always returned immediately. Reachable, referenced,
1068
+ * and inert — the shape a reference scan cannot find.
1069
+ *
1070
+ * *(The first version of this order claimed it DOUBLE-ADVANCED the budget and triggered a
1071
+ * useless rebuild. That was wrong, and it was wrong because it was reasoned rather than
1072
+ * run. It was a no-op.)*
1073
+ *
1074
+ * Nothing replaces it: `#retryDue` owns the budget and the re-take above owns the work.
1075
+ */
993
1076
  continue;
994
1077
  }
995
1078
  sr.relayPeerIds = arrived;
@@ -1069,117 +1152,75 @@ export class SessionRelay {
1069
1152
  // client for this (agent, relay) pair kept the error that ended its reader; that is the
1070
1153
  // nearest thing to an upstream cause available here, and its absence is how 2,061 of these
1071
1154
  // went untraced.
1072
- const upstreamReason = this.#ctx.relayClients.get(`${agentName}::${relayPeerId}`)?.getLastReaderError();
1155
+ /**
1156
+ * ⚠️ **A DIAGNOSTIC MUST NOT BE ABLE TO KILL THE WATCHDOG TICK.** This is optional-chained on
1157
+ * the map lookup but the METHOD was called unguarded, so a client without it threw — an
1158
+ * unhandled rejection inside the tick, which takes the rest of the sweep with it. Surfaced
1159
+ * when 055-ONDEMAND made this branch reachable in more cases. The cause line is worth
1160
+ * having; it is not worth the loss detection it rides on.
1161
+ */
1162
+ const client = this.#ctx.relayClients.get(`${agentName}::${relayPeerId}`);
1163
+ const upstreamReason = typeof client?.getLastReaderError === "function" ? client.getLastReaderError() : null;
1073
1164
  this.#ctx.logger.warn("session.standing_receiver.reservation.lost", {
1074
1165
  agentName,
1075
1166
  relayPeerId,
1076
1167
  reason: open.includes(relayPeerId) ? "circuit_address_vanished" : "relay_connection_gone",
1077
1168
  ...(upstreamReason ? { upstreamReason } : {}),
1078
1169
  reservationsHeld: stillHeld.length,
1079
- // The line an operator reads, and the two cases are not the same event at all.
1170
+ /**
1171
+ * The line an operator reads, and the two cases are not the same event at all.
1172
+ *
1173
+ * ⚠️ **THE ZERO CASE PROMISED A REPAIR THAT NO LONGER HAPPENS — 056-SLOTDEAD, review F3.**
1174
+ * It ended "The receiver is being rebuilt against the rest of the pool", which was true
1175
+ * when this branch rebuilt. Nothing rebuilds now. An operator reading the old line would
1176
+ * wait for a recovery that was never coming, and the only thing that WOULD have told them
1177
+ * otherwise is a `zero_held` line at debug level they are not reading. Both branches now
1178
+ * say what actually happens next and who owns it.
1179
+ */
1080
1180
  impact: stillHeld.length > 0
1081
1181
  ? "this agent still holds " + stillHeld.length + " other circuit reservation(s), so it "
1082
- + "stays dialable from behind NAT and the receiver is NOT rebuilt. Losing one relay "
1083
- + "costs this agent nothing it can feel."
1084
- : "this agent now holds NO circuit reservation, so nobody behind a home router can "
1085
- + "reach it. The receiver is being rebuilt against the rest of the pool.",
1182
+ + "stays dialable from behind NAT. Losing one relay costs this agent nothing it can feel."
1183
+ : "this agent now holds NO circuit reservation. If it has no live session that is the "
1184
+ + "normal idle state and costs nothing a slot is taken when someone calls. If it "
1185
+ + "DOES have a live session, that session's counterparty can no longer dial it, and "
1186
+ + "the re-take above is what gets it back, on a bounded budget. Nothing rebuilds the "
1187
+ + "receiver any more, so do not wait for one.",
1086
1188
  });
1087
1189
  }
1190
+ /**
1191
+ * ⚠️ **DRAIN ON THE LOSS ITSELF — 056-SLOTDEAD.** This used to happen one step downstream, as
1192
+ * a side effect of the rebuild below: lose every reservation → rebuild → the new receiver
1193
+ * reports ready → drain. Deleting the rebuild would have deleted the drain with it, leaving
1194
+ * content the counterparty parked to sit until the periodic backstop. The loss is the cause
1195
+ * and always was; the rebuild was only where it happened to be noticed.
1196
+ */
1197
+ this.#ctx.park.fireParkedDrain(agentName, "reservation_lost");
1088
1198
  if (stillHeld.length === 0) {
1089
- // ZERO HELD IS STILL THE LOUD, STRUCTURAL CASE — the agent is unreachable behind NAT and
1090
- // only a new node can take a new reservation, because a circuit listener is fixed at node
1091
- // creation.
1092
- void this.#ctx.receivers.rebuildStandingReceiver(agentName);
1199
+ /**
1200
+ * ⚠️ **THIS REBUILT THE RECEIVER ON A JUSTIFICATION THAT STOPPED BEING TRUE IN UNIT 1, AND
1201
+ * UNDER ON-DEMAND IT DESTROYS AN IN-FLIGHT OFFER'S RESERVATION — 056-SLOTDEAD.**
1202
+ *
1203
+ * It read: *"only a new node can take a new reservation, because a circuit listener is fixed
1204
+ * at node creation."* `listenOnCircuit` removed that constraint (`DOD-M15-RELAYPROVE-ORDER-1`),
1205
+ * and a rebuilt receiver now reserves nothing at all.
1206
+ *
1207
+ * The state it fires in is the dangerous one: between an offer taking a reservation and the
1208
+ * session being created, the receiver holds exactly one circuit. Lose the connection in that
1209
+ * window and this rebuilt the node — discarding the slot the offer just took, after the
1210
+ * accept may already have advertised it. **Observed**, not reasoned: a fixture that reported
1211
+ * no relay connection drove this path and the test caught the rebuild.
1212
+ *
1213
+ * Nothing replaces it here. An idle agent holding zero is the design; a LIVE session that
1214
+ * lost its circuit is re-taken on its own node by the branch above, on a bounded budget.
1215
+ */
1216
+ this.#ctx.logger.debug("session.standing_receiver.reservation.zero_held", {
1217
+ agentName,
1218
+ impact: "this agent holds no circuit. If it has no live session that is the idle steady " +
1219
+ "state; if it does, the re-take above owns getting it back.",
1220
+ });
1093
1221
  continue;
1094
1222
  }
1095
- /**
1096
- * STILL REACHABLE, SO THE RECEIVER STANDS, AND NOTHING ELSE HAPPENS HERE. That second half is
1097
- * the part worth reading, because the obvious next line is wrong twice over.
1098
- *
1099
- * **A LOST CONFIGURED CIRCUIT CANNOT BE RETAKEN BY THIS NODE.** Read out of
1100
- * `@libp2p/circuit-relay-v2@4.2.5`, not assumed: for an explicit relay address
1101
- * `transport/listener.js#listen()` is a ONE-SHOT — it reserves once and nothing calls it
1102
- * again; `reservation-store.js#removeReservation()` clears the refresh timeout and deletes
1103
- * the entry; and the listener's `_onAddRelayPeer` returns early for `type === 'configured'`,
1104
- * so even a later reservation would not be announced. A circuit listener is fixed at node
1105
- * creation, and the only thing that takes a new one is a NEW NODE — which is exactly the
1106
- * rebuild this branch exists to refuse.
1107
- *
1108
- * **AND RE-PROVING TO THE LOST RELAY WOULD REBUILD THE RECEIVER ANYWAY.** Review F3: an
1109
- * earlier version called `authenticateStandingReceiver` here to "remove the relay-side
1110
- * reason for the revocation". That function ends with `if (refusal?.tryAnotherRelay) { …
1111
- * void this.#ctx.receivers.rebuildStandingReceiver(agentName); }` — and a dead or misconfigured relay is
1112
- * precisely the one that answers that way. So the common case was: lose relay A while
1113
- * holding B, decline to rebuild, prove to A, A refuses, rebuild the whole receiver and throw
1114
- * B's healthy reservation away. The churn engine, re-entered through the back door.
1115
- *
1116
- * **THE BOUND, STATED PLAINLY BECAUSE IT IS A REAL SHORTFALL AGAINST THE DoD:** a lost
1117
- * circuit is gone until the receiver is next rebuilt for another reason. What the agent buys
1118
- * is that it never STOPS BEING REACHABLE while that is true — the surviving relays carry it,
1119
- * the loss is named in the log with its cause, and the lost relay's inbound carve-out is
1120
- * revoked above. That is availability, not restoration in place.
1121
- *
1122
- * WHICH LEAVES A RATCHET, and `#respreadIfDecayed` below is what stops it: relays are only
1123
- * ever lost between rebuilds, never regained, so an agent nobody talks to walks itself back
1124
- * down to one relay — the exact state this unit exists to get it out of.
1125
- */
1126
- }
1127
- for (const agentName of this.#ctx.standingReceivers.keys()) {
1128
- if (this.#ctx.agentsWantingReceiver.has(agentName))
1129
- this.#respreadIfDecayed(agentName);
1130
- }
1131
- }
1132
- /**
1133
- * 032-RELAYSPREAD — **AN IDLE AGENT MUST NOT RATCHET ITSELF BACK DOWN TO ONE RELAY.**
1134
- *
1135
- * Spreading happens when a receiver is BUILT, and between builds the count only falls: a lost
1136
- * circuit is not retaken from here (`listenOnCircuit` could; nothing does), and a relay the
1137
- * directory announces later is skipped while any circuit is held. An agent in
1138
- * conversation re-spreads constantly — the receiver is handed into each session and a fresh one
1139
- * is built behind it — so this is about the agent nobody has talked to for a day. It loses relays
1140
- * one at a time, nothing pulls it back up, and it ends up exactly where this unit found it:
1141
- * reachable through one relay, one relay away from being reachable through none.
1142
- *
1143
- * **THE COST OF FIXING IT IS A NEW PEER ID**, which is why it is fenced three ways rather than
1144
- * simply rebuilding on sight:
1145
- * - **ONLY WHEN IDLE.** A rebuild replaces the receiver's transport identity, and a counterparty
1146
- * may be holding the old one from a `session_offer_accept`. With a live session for this agent
1147
- * we leave it alone — a degraded spread costs redundancy, a changed peer id mid-conversation
1148
- * costs the conversation.
1149
- * - **ONLY WHEN THERE IS SOMETHING TO GAIN.** Holding every relay that was offered is not decay.
1150
- * - **ON ITS OWN SLOW CLOCK**, never the watchdog's 30-second grid. A reservation is scarce —
1151
- * the relay holds it for its full TTL even after we disconnect — so this reuses the
1152
- * reservation retry interval rather than inventing a faster one.
1153
- */
1154
- #respreadIfDecayed(agentName) {
1155
- if (this.#ctx.shuttingDown)
1156
- return;
1157
- const sr = this.#ctx.standingReceivers.get(agentName);
1158
- if (!sr || sr.relayPeerIds.length === 0)
1159
- return; // zero held is the loud path
1160
- for (const entry of this.#ctx.activeNodes.values()) {
1161
- if (entry.agentName === agentName)
1162
- return; // in conversation — hands off
1163
1223
  }
1164
- const offered = this.reservationCircuitAddrs(agentName).addrs.length;
1165
- if (sr.relayPeerIds.length >= offered)
1166
- return; // nothing to gain
1167
- const now = Date.now();
1168
- const last = this.#ctx.srLastRespreadAt.get(agentName) ?? 0;
1169
- if (now - last < this.#ctx.srReservationRetryMs)
1170
- return;
1171
- this.#ctx.srLastRespreadAt.set(agentName, now);
1172
- this.#ctx.logger.info("session.standing_receiver.respread", {
1173
- agentName,
1174
- reservationsHeld: sr.relayPeerIds.length,
1175
- relaysOffered: offered,
1176
- impact: "this agent is idle and holds fewer relay reservations than it was offered, so its " +
1177
- "receiver is being rebuilt to take the rest. Without this it can only lose relays between " +
1178
- "rebuilds, and an agent nobody talks to drifts back down to a single relay — one relay " +
1179
- "away from being unreachable behind NAT, which is the state this whole mechanism exists " +
1180
- "to keep it out of.",
1181
- });
1182
- void this.#ctx.receivers.rebuildStandingReceiver(agentName);
1183
1224
  }
1184
1225
  /** Start the reservation watchdog (idempotent). Stopped by gracefulShutdown. */
1185
1226
  startReservationWatchdog() {
@@ -1190,7 +1231,7 @@ export class SessionRelay {
1190
1231
  this.#ctx.park.armBackstopClock(Date.now());
1191
1232
  this.#ctx.reservationWatchdog = setInterval(() => {
1192
1233
  try {
1193
- this.#reservationWatchdogTick();
1234
+ void this.#reservationWatchdogTick();
1194
1235
  this.#ctx.park.parkedDrainBackstopTick(Date.now());
1195
1236
  }
1196
1237
  catch (err) {
@@ -1249,6 +1290,61 @@ export class SessionRelay {
1249
1290
  * `tryAnotherRelay: false` precisely so the client STOPS walking the fleet; without the verdict,
1250
1291
  * the loop walked it anyway, turning one client-side fault into what reads as a fleet outage.
1251
1292
  */
1293
+ /**
1294
+ * 055-ONDEMAND — **tell the relay this agent has finished with its slot.**
1295
+ *
1296
+ * The only thing that actually frees one: closing a circuit listener sends the relay nothing, and
1297
+ * the relay reclaims on its own only at the reservation TTL (two hours) or under reaper pressure.
1298
+ * Best-effort by design — an undelivered release costs a slot until that TTL, and must never fail
1299
+ * the seal that triggered it.
1300
+ */
1301
+ async tellRelayReleased(agentName, relayPeerId, node, correlationId) {
1302
+ /**
1303
+ * ⚠️ **THE NODE IS THE SOURCE OF TRUTH FOR WHERE THIS RELAY IS — review MEDIUM-6.**
1304
+ *
1305
+ * The first version looked the relay up in `directoryRelayEndpoints`. But the reservation was
1306
+ * taken on the relay the OFFER named, whose addresses came off that frame; if the directory's
1307
+ * last announcement does not contain it — a pool relay dropped from the roster, a stale
1308
+ * announcement — the release warns `no_endpoint` and the slot is held to its TTL. The node
1309
+ * announces the circuit it actually holds, so derive the address from that and fall back to the
1310
+ * directory list only when there is nothing to derive from.
1311
+ */
1312
+ const fromNode = node.listenAddresses()
1313
+ .filter((a) => a.split("/").includes("p2p-circuit") && a.includes(`/p2p/${relayPeerId}/`))
1314
+ .map((a) => a.split("/p2p-circuit")[0])
1315
+ .filter((a) => a.length > 0);
1316
+ const ep = this.#ctx.directoryRelayEndpoints.get(agentName)?.find((e) => e.relayPeerId === relayPeerId);
1317
+ const relayAddrs = fromNode.length > 0 ? [...new Set(fromNode)] : (ep ? [...ep.relayAddrs] : []);
1318
+ if (relayAddrs.length === 0) {
1319
+ this.#ctx.logger.warn("session.relay.reservation_release.no_endpoint", {
1320
+ agentName, relayPeerId, correlationId,
1321
+ impact: "no address is known for this relay, so it cannot be told the slot is free and " +
1322
+ "holds it until the reservation TTL expires.",
1323
+ });
1324
+ return;
1325
+ }
1326
+ let client;
1327
+ try {
1328
+ client = this.#ctx.detachedRelayClientBuilder?.(agentName, relayPeerId, relayAddrs, {
1329
+ receiptStore: this.#ctx.relayReceiptStore ?? undefined,
1330
+ sealLeafStore: this.#ctx.sealLeafStore ?? undefined,
1331
+ onlineToken: () => this.#ctx.getDirectoryOnlineToken(agentName),
1332
+ });
1333
+ if (!client)
1334
+ return;
1335
+ await client.releaseReservation(node);
1336
+ }
1337
+ catch (err) {
1338
+ this.#ctx.logger.warn("session.relay.reservation_release.failed", {
1339
+ agentName, relayPeerId, correlationId,
1340
+ error: extractErrorMessage(err),
1341
+ impact: "the relay holds this slot until its TTL expires.",
1342
+ });
1343
+ }
1344
+ finally {
1345
+ client?.close();
1346
+ }
1347
+ }
1252
1348
  async proveToRelay(agentName, circuitAddr, node, correlationId,
1253
1349
  /**
1254
1350
  * Whether this proof is the STANDING RECEIVER's, and may therefore write the surface