@serfab/cadre-core 1.6.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/README.md +8 -6
  2. package/dist/cadre-node.d.ts +548 -137
  3. package/dist/cadre-node.js +1208 -383
  4. package/dist/cadre-node.js.map +1 -1
  5. package/dist/control-authorization.d.ts +2 -2
  6. package/dist/control-authorization.js +1 -0
  7. package/dist/control-authorization.js.map +1 -1
  8. package/dist/control-cohort.d.ts +6 -0
  9. package/dist/control-cohort.js +6 -0
  10. package/dist/control-cohort.js.map +1 -1
  11. package/dist/control-database.d.ts +46 -8
  12. package/dist/control-database.js +97 -18
  13. package/dist/control-database.js.map +1 -1
  14. package/dist/control-formation-recorder.d.ts +37 -14
  15. package/dist/control-formation-recorder.js +60 -10
  16. package/dist/control-formation-recorder.js.map +1 -1
  17. package/dist/control-read-retry.d.ts +13 -12
  18. package/dist/control-read-retry.js +14 -12
  19. package/dist/control-read-retry.js.map +1 -1
  20. package/dist/control-schema.d.ts +1 -1
  21. package/dist/control-schema.js +67 -5
  22. package/dist/control-schema.js.map +1 -1
  23. package/dist/control-stream.d.ts +9 -1
  24. package/dist/control-stream.js +24 -1
  25. package/dist/control-stream.js.map +1 -1
  26. package/dist/control-write-retry.d.ts +12 -2
  27. package/dist/control-write-retry.js +13 -2
  28. package/dist/control-write-retry.js.map +1 -1
  29. package/dist/delegate-admission.d.ts +11 -8
  30. package/dist/delegate-admission.js +19 -11
  31. package/dist/delegate-admission.js.map +1 -1
  32. package/dist/diagnostics/connection-path.d.ts +0 -7
  33. package/dist/diagnostics/connection-path.js +0 -7
  34. package/dist/diagnostics/connection-path.js.map +1 -1
  35. package/dist/formation-approval.d.ts +7 -0
  36. package/dist/formation-approval.js +9 -3
  37. package/dist/formation-approval.js.map +1 -1
  38. package/dist/fs-atomic.d.ts +11 -2
  39. package/dist/fs-atomic.js +13 -3
  40. package/dist/fs-atomic.js.map +1 -1
  41. package/dist/hibernation-manager.d.ts +111 -8
  42. package/dist/hibernation-manager.js +234 -49
  43. package/dist/hibernation-manager.js.map +1 -1
  44. package/dist/index.d.ts +13 -6
  45. package/dist/index.js +27 -5
  46. package/dist/index.js.map +1 -1
  47. package/dist/joined-strand-store.d.ts +177 -0
  48. package/dist/joined-strand-store.js +431 -0
  49. package/dist/joined-strand-store.js.map +1 -0
  50. package/dist/key-store-file.js +14 -5
  51. package/dist/key-store-file.js.map +1 -1
  52. package/dist/key-store.d.ts +6 -0
  53. package/dist/key-store.js.map +1 -1
  54. package/dist/link-budget.d.ts +367 -69
  55. package/dist/link-budget.js +375 -72
  56. package/dist/link-budget.js.map +1 -1
  57. package/dist/membership-connection-gater.d.ts +30 -43
  58. package/dist/membership-connection-gater.js +31 -42
  59. package/dist/membership-connection-gater.js.map +1 -1
  60. package/dist/node-local-snapshot.d.ts +5 -4
  61. package/dist/node-local-snapshot.js +6 -3
  62. package/dist/node-local-snapshot.js.map +1 -1
  63. package/dist/peer-dial.d.ts +24 -4
  64. package/dist/peer-dial.js +74 -5
  65. package/dist/peer-dial.js.map +1 -1
  66. package/dist/peer-join-backfill.d.ts +35 -15
  67. package/dist/peer-join-backfill.js +115 -55
  68. package/dist/peer-join-backfill.js.map +1 -1
  69. package/dist/push-fanout.d.ts +6 -0
  70. package/dist/push-fanout.js +6 -0
  71. package/dist/push-fanout.js.map +1 -1
  72. package/dist/relay-reservation.d.ts +3 -1
  73. package/dist/relay-reservation.js +3 -1
  74. package/dist/relay-reservation.js.map +1 -1
  75. package/dist/relay-server.d.ts +79 -0
  76. package/dist/relay-server.js +74 -0
  77. package/dist/relay-server.js.map +1 -0
  78. package/dist/seed-bootstrap.d.ts +57 -4
  79. package/dist/seed-bootstrap.js +107 -83
  80. package/dist/seed-bootstrap.js.map +1 -1
  81. package/dist/storage-scope.d.ts +33 -17
  82. package/dist/storage-scope.js +44 -23
  83. package/dist/storage-scope.js.map +1 -1
  84. package/dist/strand-addr-protocol.d.ts +58 -21
  85. package/dist/strand-addr-protocol.js +133 -85
  86. package/dist/strand-addr-protocol.js.map +1 -1
  87. package/dist/strand-database.d.ts +20 -2
  88. package/dist/strand-database.js +42 -5
  89. package/dist/strand-database.js.map +1 -1
  90. package/dist/strand-first-sync-gate.d.ts +59 -21
  91. package/dist/strand-first-sync-gate.js +58 -19
  92. package/dist/strand-first-sync-gate.js.map +1 -1
  93. package/dist/strand-formation-deadlines.d.ts +151 -0
  94. package/dist/strand-formation-deadlines.js +171 -0
  95. package/dist/strand-formation-deadlines.js.map +1 -0
  96. package/dist/strand-formation-manager.d.ts +63 -30
  97. package/dist/strand-formation-manager.js +67 -41
  98. package/dist/strand-formation-manager.js.map +1 -1
  99. package/dist/strand-formation-protocol.d.ts +45 -35
  100. package/dist/strand-formation-protocol.js +59 -122
  101. package/dist/strand-formation-protocol.js.map +1 -1
  102. package/dist/strand-instance-manager.d.ts +152 -6
  103. package/dist/strand-instance-manager.js +331 -42
  104. package/dist/strand-instance-manager.js.map +1 -1
  105. package/dist/strand-membership-writer.d.ts +6 -0
  106. package/dist/strand-membership-writer.js +11 -0
  107. package/dist/strand-membership-writer.js.map +1 -1
  108. package/dist/strand-peer-book-file.d.ts +12 -0
  109. package/dist/strand-peer-book-file.js +37 -0
  110. package/dist/strand-peer-book-file.js.map +1 -0
  111. package/dist/strand-peer-book-protocol.d.ts +168 -0
  112. package/dist/strand-peer-book-protocol.js +356 -0
  113. package/dist/strand-peer-book-protocol.js.map +1 -0
  114. package/dist/strand-peer-book-swap.d.ts +104 -0
  115. package/dist/strand-peer-book-swap.js +284 -0
  116. package/dist/strand-peer-book-swap.js.map +1 -0
  117. package/dist/strand-peer-book.d.ts +159 -0
  118. package/dist/strand-peer-book.js +361 -0
  119. package/dist/strand-peer-book.js.map +1 -0
  120. package/dist/strand-peer-observer.d.ts +88 -0
  121. package/dist/strand-peer-observer.js +180 -0
  122. package/dist/strand-peer-observer.js.map +1 -0
  123. package/dist/strand-revocation-enforcer.js +2 -1
  124. package/dist/strand-revocation-enforcer.js.map +1 -1
  125. package/dist/strand-solicitation.d.ts +36 -20
  126. package/dist/strand-solicitation.js +4 -4
  127. package/dist/strand-solicitation.js.map +1 -1
  128. package/dist/strand-wake-protocol.d.ts +80 -23
  129. package/dist/strand-wake-protocol.js +89 -64
  130. package/dist/strand-wake-protocol.js.map +1 -1
  131. package/dist/types.d.ts +181 -33
  132. package/dist/types.js +4 -3
  133. package/dist/types.js.map +1 -1
  134. package/package.json +11 -7
  135. package/src/cadre-node.ts +1316 -398
  136. package/src/control-authorization.ts +2 -1
  137. package/src/control-cohort.ts +6 -0
  138. package/src/control-database.ts +122 -18
  139. package/src/control-formation-recorder.ts +354 -305
  140. package/src/control-read-retry.ts +14 -12
  141. package/src/control-schema.ts +67 -5
  142. package/src/control-stream.ts +25 -1
  143. package/src/control-write-retry.ts +13 -2
  144. package/src/delegate-admission.ts +23 -13
  145. package/src/diagnostics/connection-path.ts +0 -7
  146. package/src/formation-approval.ts +610 -604
  147. package/src/fs-atomic.ts +14 -4
  148. package/src/hibernation-manager.ts +638 -437
  149. package/src/index.ts +91 -6
  150. package/src/joined-strand-store.ts +498 -0
  151. package/src/key-store-file.ts +14 -5
  152. package/src/key-store.ts +6 -0
  153. package/src/link-budget.ts +393 -74
  154. package/src/membership-connection-gater.ts +31 -45
  155. package/src/node-local-snapshot.ts +293 -289
  156. package/src/peer-dial.ts +81 -7
  157. package/src/peer-join-backfill.ts +120 -57
  158. package/src/push-fanout.ts +6 -0
  159. package/src/relay-reservation.ts +3 -1
  160. package/src/relay-server.ts +98 -0
  161. package/src/seed-bootstrap.ts +133 -91
  162. package/src/storage-scope.ts +44 -23
  163. package/src/strand-addr-protocol.ts +547 -458
  164. package/src/strand-database.ts +287 -242
  165. package/src/strand-first-sync-gate.ts +58 -19
  166. package/src/strand-formation-deadlines.ts +227 -0
  167. package/src/strand-formation-manager.ts +700 -647
  168. package/src/strand-formation-protocol.ts +103 -150
  169. package/src/strand-instance-manager.ts +387 -45
  170. package/src/strand-membership-writer.ts +12 -0
  171. package/src/strand-peer-book-file.ts +39 -0
  172. package/src/strand-peer-book-protocol.ts +466 -0
  173. package/src/strand-peer-book-swap.ts +338 -0
  174. package/src/strand-peer-book.ts +475 -0
  175. package/src/strand-peer-observer.ts +235 -0
  176. package/src/strand-revocation-enforcer.ts +2 -1
  177. package/src/strand-solicitation.ts +519 -500
  178. package/src/strand-wake-protocol.ts +401 -355
  179. package/src/types.ts +185 -33
package/src/peer-dial.ts CHANGED
@@ -24,8 +24,9 @@
24
24
  */
25
25
 
26
26
  import debug from 'debug';
27
- import type { Connection } from '@libp2p/interface';
28
- import type { Multiaddr } from '@multiformats/multiaddr';
27
+ import type { Connection, PeerId } from '@libp2p/interface';
28
+ import { peerIdFromString } from '@libp2p/peer-id';
29
+ import type { Component, Multiaddr } from '@multiformats/multiaddr';
29
30
  import { withDeadline } from './control-stream.js';
30
31
  import { relayedDialBudgetMs } from './link-budget.js';
31
32
 
@@ -33,8 +34,9 @@ const log = debug('sereus:cadre:peer-dial');
33
34
 
34
35
  /**
35
36
  * Default limit on ONE address's dial attempt, in ms, at the DEFAULT declared link round trip:
36
- * `RELAYED_DIAL_ROUND_TRIPS` (4) x `DECLARED_LINK_ROUND_TRIP_MS` (3500 ms) = 14 000 ms. It was
37
- * a fixed 8000 ms before it was derived.
37
+ * `RELAYED_DIAL_ROUND_TRIPS` (4) x `DECLARED_LINK_ROUND_TRIP_MS` (3500 ms) plus the called
38
+ * machine's `ADMISSION_DECISION_TIMEOUT_MS` (2000 ms) = 16 000 ms (`relayedDialBudgetMs`). It
39
+ * was a fixed 8000 ms before it was derived.
38
40
  *
39
41
  * It has to cover the slowest address that should succeed, because an attempt
40
42
  * that needs longer fails the same way on every retry. That is a relayed dial on
@@ -117,9 +119,28 @@ export const DEFAULT_PEER_DIAL_BUDGET: Readonly<PeerDialBudget> = {
117
119
 
118
120
  /** The slice of a libp2p node {@link dialPeerAddrs} uses. */
119
121
  export interface AddrDialer {
122
+ /** This node's id, to recognise an address that relays through this node ({@link relaysThrough}). */
123
+ readonly peerId: PeerId;
120
124
  dial(addr: Multiaddr, options: { signal: AbortSignal }): Promise<Connection>;
121
125
  }
122
126
 
127
+ /**
128
+ * Thrown by {@link dialPeerAddrs} when it was given addresses but every one reaches the peer by
129
+ * relaying through this node, so none was dialed.
130
+ *
131
+ * This is where a relay lands for a peer that holds a reservation on it (a phone, or any node that
132
+ * cannot listen): the address that peer publishes is `/<relay transport>/p2p/<relay>/p2p-circuit/p2p/<peer>`,
133
+ * and dialing it from the relay itself fails with libp2p's `Can not dial self`. Nothing this node
134
+ * does can repair it; once the peer's connection drops, only the peer can reconnect. Callers log it
135
+ * as that rather than as a dial failure.
136
+ */
137
+ export class SelfRelayOnlyError extends Error {
138
+ constructor(label: string) {
139
+ super(`${label}: every candidate address relays through this node; only the peer can reconnect`);
140
+ this.name = 'SelfRelayOnlyError';
141
+ }
142
+ }
143
+
123
144
  /** One candidate's outcome, kept so the thrown error can name every attempt. */
124
145
  interface AddrAttemptFailure {
125
146
  addr: Multiaddr;
@@ -131,23 +152,31 @@ interface AddrAttemptFailure {
131
152
  * first connection that forms. Direct addresses are tried before relayed ones
132
153
  * ({@link directBeforeRelayed}); otherwise the given order is kept.
133
154
  *
155
+ * Addresses that relay through this node are dropped first ({@link relaysThrough}):
156
+ * dialing one could only fail with `Can not dial self`. When that leaves nothing
157
+ * of a non-empty list, this throws {@link SelfRelayOnlyError} without dialing.
158
+ *
134
159
  * Addresses naming a transport this node lacks are not filtered out first: a
135
160
  * one-address `dial()` rejects those at once (`NoValidAddressesError`), so they
136
161
  * cost a log line, not time.
137
162
  *
138
- * Throws when no address connects — see {@link tryAddrsInTurn} for the error.
163
+ * Otherwise throws when no address connects — see {@link tryAddrsInTurn} for the error.
139
164
  *
140
165
  * @param label Names the dial in timeout messages and logs, e.g.
141
166
  * `reconcileControlCohort dial of sibling <peerId>`; each attempt reads
142
167
  * `<label> via <addr>`.
143
168
  */
144
- export function dialPeerAddrs(
169
+ export async function dialPeerAddrs(
145
170
  dialer: AddrDialer,
146
171
  addrs: readonly Multiaddr[],
147
172
  budget: PeerDialBudget,
148
173
  label: string,
149
174
  ): Promise<Connection> {
150
- return tryAddrsInTurn(directBeforeRelayed(addrs), budget, label, (addr, signal) => dialer.dial(addr, { signal }));
175
+ const dialable = withoutSelfRelayed(dialer, addrs, label);
176
+ if (addrs.length > 0 && dialable.length === 0) {
177
+ throw new SelfRelayOnlyError(label);
178
+ }
179
+ return tryAddrsInTurn(directBeforeRelayed(dialable), budget, label, (addr, signal) => dialer.dial(addr, { signal }));
151
180
  }
152
181
 
153
182
  /**
@@ -221,6 +250,51 @@ export function directBeforeRelayed(addrs: readonly Multiaddr[]): Multiaddr[] {
221
250
  return [...addrs.filter((addr) => !relayed(addr)), ...addrs.filter(relayed)];
222
251
  }
223
252
 
253
+ /** `addrs` without those that relay through `dialer` itself, each dropped one logged. */
254
+ function withoutSelfRelayed(dialer: AddrDialer, addrs: readonly Multiaddr[], label: string): Multiaddr[] {
255
+ return addrs.filter((addr) => {
256
+ if (!relaysThrough(addr, dialer.peerId)) {
257
+ return true;
258
+ }
259
+ log('%s: skipping %s — it relays through this node', label, addr.toString());
260
+ return false;
261
+ });
262
+ }
263
+
264
+ /**
265
+ * Whether `addr` reaches its target by relaying through `relay` at any hop.
266
+ *
267
+ * A circuit address names its relay in the `p2p` component immediately before
268
+ * the `p2p-circuit` marker (`/<transport>/p2p/<relay>/p2p-circuit/p2p/<target>`),
269
+ * so the components are walked rather than the text searched: the target's id
270
+ * after the marker, a bare `/p2p-circuit` naming no relay, and a multi-hop chain
271
+ * each read correctly that way. Same rule as db-p2p's `routesThroughRelay`, which
272
+ * its package does not export.
273
+ */
274
+ function relaysThrough(addr: Multiaddr, relay: PeerId): boolean {
275
+ const components = addr.getComponents();
276
+ return components.some((component, i) => component.name === 'p2p-circuit' && namesPeer(components[i - 1], relay));
277
+ }
278
+
279
+ /**
280
+ * Whether `component` is a `p2p` component naming `peer`. Compared as canonical
281
+ * strings because a multiaddr keeps a peer id in whichever form it was written
282
+ * (base58 or CIDv1). `PeerId.equals` is avoided because test doubles supply the
283
+ * dialer's `peerId` with `toString` only.
284
+ */
285
+ function namesPeer(component: Component | undefined, peer: PeerId): boolean {
286
+ if (component?.name !== 'p2p' || component.value === undefined) {
287
+ return false;
288
+ }
289
+ const expected = peer.toString();
290
+ try {
291
+ return peerIdFromString(component.value).toString() === expected;
292
+ } catch (err) {
293
+ log('cannot read %s as a peer id; treating it as not this node: %o', component.value, err);
294
+ return false;
295
+ }
296
+ }
297
+
224
298
  function allAttemptsFailed(label: string, failures: AddrAttemptFailure[]): Error {
225
299
  if (failures.length === 0) {
226
300
  return new Error(`${label}: no candidate addresses`);
@@ -1,17 +1,28 @@
1
1
  import debug from 'debug';
2
- import type { Connection, Libp2p, PeerId } from '@libp2p/interface';
2
+ import type { IdentifyResult, Libp2p, PeerId } from '@libp2p/interface';
3
3
  import type { ActionId, IPeerNetwork } from '@optimystic/db-core';
4
- import { BlockTransferClient, type BlockCommitProof, type IRawStorage } from '@optimystic/db-p2p';
4
+ import { BlockTransferClient, buildBlockTransferProtocol, type BlockCommitProof, type IRawStorage } from '@optimystic/db-p2p';
5
5
  import { peerJoinPushBudget } from './link-budget.js';
6
6
 
7
7
  // Peer-join whole-store block catch-up, shared by the STRAND networks and the CONTROL
8
- // network. On `connection:open` (debounced) it pushes every committed, materialized block
9
- // in the local raw store to the new peer, so a machine that joined after blocks were
8
+ // network. On a new peer's `peer:identify` result — libp2p's identify protocol runs once per
9
+ // connection and reports the protocols the remote supports — naming this network's own
10
+ // block-transfer protocol, it schedules a debounced push of every committed, materialized
11
+ // block in the local raw store to that peer, so a machine that joined after blocks were
10
12
  // committed still ends up physically holding them. Optimystic has no cohort-join catch-up
11
13
  // of its own; a block committed while its writer was alone has exactly one holder forever
12
14
  // without this (named collection-header blocks are written once, at collection creation,
13
15
  // and their revision never moves again — so no later commit ever carries them anywhere).
14
16
  //
17
+ // A peer whose identify does not name the protocol — a bare circuit relay, a bootstrap node,
18
+ // or (on the control network) a stranger the inbound gate admits without membership — is
19
+ // never scheduled at all: it cannot receive anything, since the protocol id is namespaced per
20
+ // network and the dial would simply fail. Before this filter existed, such a peer WAS
21
+ // scheduled, failed every dial, and after `PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES` produced a
22
+ // `console.warn` naming it — gotchoices/sereus#18 is two relay-only parties restarting and
23
+ // printing exactly that warning about their own relay, forever, on every reconnect. See
24
+ // {@link speaksBlockTransfer}.
25
+ //
15
26
  // NOTE: this copies the WHOLE local store to every newly connected peer, which is right
16
27
  // while a network is one party's handful of machines (see docs/architecture.md →
17
28
  // Replication cluster size — the control network replicates to the whole party by intent,
@@ -26,12 +37,10 @@ import { peerJoinPushBudget } from './link-budget.js';
26
37
  // Membership — the one place the two networks differ, expressed as the optional
27
38
  // `authorizePeer` dep:
28
39
  //
29
- // - STRAND networks pass none. Anything connected on a strand's own libp2p network already
30
- // receives cohort replicas of new commits, so pushing older blocks to it exposes nothing
31
- // new; adding a gate would diverge from what ordinary replication already does there. A
32
- // peer that does NOT speak the strand's block-transfer protocol (a bare circuit relay or
33
- // bootstrap node the strand node also connects to) cannot receive anything: the protocol
34
- // id is namespaced per strand, so the dial fails and the push is dropped.
40
+ // - STRAND networks pass none. Anything that speaks the strand's own block-transfer protocol
41
+ // already receives cohort replicas of new commits, so pushing older blocks to it exposes
42
+ // nothing new; adding a gate would diverge from what ordinary replication already does
43
+ // there.
35
44
  // - The CONTROL network MUST pass one, because that argument does not carry over: its
36
45
  // inbound connection gate deliberately admits non-members in several states (an
37
46
  // un-enrolled node taking its seed, an open enrollment window, an outstanding
@@ -40,8 +49,8 @@ import { peerJoinPushBudget } from './link-budget.js';
40
49
  // peer would hand a stranger the party's entire membership, addresses and strand list.
41
50
  // `CadreNode` passes `isAuthorizedMember`; the gate is consulted at PUSH time, not at
42
51
  // schedule time, and fails closed on a thrown check. A denied run is not memoized, so
43
- // the peer is retried on its next `connection:open` — or sooner, via
44
- // `scheduleConnectedPeers()` on a membership change (the production join order is
52
+ // the peer is retried on its next `peer:identify` (a reconnect re-runs identify) — or
53
+ // sooner, via `scheduleConnectedPeers()` on a membership change (the production join order is
45
54
  // connect-then-authorize, so the denial at dial time is the expected first pass).
46
55
  //
47
56
  // Both per-push deadlines are DERIVED from the declared link round trip
@@ -54,22 +63,23 @@ import { peerJoinPushBudget } from './link-budget.js';
54
63
  // `packages/integration-tests/src/scenarios/relayed-dial-cost-by-latency.integration.ts`.
55
64
  //
56
65
  // It does NOT work at every speed. Above the declared link, libp2p's own dial and
57
- // inbound-upgrade limits — derived from the same declaration — abandon the connection before
58
- // any deadline here is consulted, and the listener's one makes that failure look like an absent
59
- // peer rather than a timeout. That is also what happens when the machine being caught up
60
- // declared a FASTER link than this one, since its limit is the listener's. See
61
- // `link-budget.ts` ("libp2p's own two limits", "What still fails at the supported link").
66
+ // inbound-upgrade limits — which Optimystic derives from the same declaration — abandon the
67
+ // connection before any deadline here is consulted, and the listener's one makes that failure
68
+ // look like an absent peer rather than a timeout. That is also what happens when the machine
69
+ // being caught up declared a FASTER link than this one, since its limit is the listener's. See
70
+ // `link-budget.ts` ("Optimystic's deadlines, from the same declaration", "What still fails at
71
+ // the supported link").
62
72
  //
63
73
  // RETRY, and why it backs off. A run whose PUSH FAILED — the transport threw, which is what
64
74
  // a dial or response deadline expiring looks like here — re-arms on a doubling backoff
65
- // (`retryBackoffMs` to `maxRetryBackoffMs`), and a `connection:open` arriving while that wait
75
+ // (`retryBackoffMs` to `maxRetryBackoffMs`), and a `peer:identify` arriving while that wait
66
76
  // is outstanding is DROPPED rather than collapsing it back to the debounce. Both halves are
67
77
  // load-bearing:
68
78
  //
69
79
  // - Without the re-arm, a transient push failure over a stable connection left the peer
70
80
  // partially copied until its next reconnect (read repair still covered reads meanwhile).
71
81
  // - Without dropping churn inside the wait, a peer that cannot be reached at all is
72
- // re-dialled on every `connection:open` forever. That is not hypothetical: the 2026-09-26
82
+ // re-dialled on every `peer:identify` forever. That is not hypothetical: the 2026-09-26
73
83
  // relayed reproduction shows one peer re-opening a connection about every 14.5 s for a
74
84
  // 200-second run, each event starting a catch-up whose dial could not possibly finish.
75
85
  // The connection kept re-appearing because Optimystic's own block-transfer push path
@@ -87,13 +97,13 @@ import { peerJoinPushBudget } from './link-budget.js';
87
97
  // {@link Chunk.proofs}). Re-arming on those would re-push the WHOLE store to that peer every
88
98
  // `maxRetryBackoffMs` for as long as the node runs, and report the failure below as a link
89
99
  // budget problem when nothing about the link is wrong. All three still leave the peer
90
- // un-memoized, so its next `connection:open` retries — the behaviour that predates the
100
+ // un-memoized, so its next `peer:identify` retries — the behaviour that predates the
91
101
  // backoff, and the right one for a verdict rather than a timeout. Denial in particular is
92
102
  // re-driven on purpose by `scheduleConnectedPeers()` the moment the membership commit lands,
93
103
  // because the control network's join order is connect-then-authorize.
94
104
  //
95
105
  // A re-arm is also skipped when the peer is no longer CONNECTED. This module's trigger is
96
- // `connection:open`, so a peer that went away already has one: re-arming instead would leave
106
+ // `peer:identify`, so a peer that went away already has one: re-arming instead would leave
97
107
  // a machine dialing a peer it cannot see once a minute for the rest of its uptime.
98
108
  //
99
109
  // After `PEER_JOIN_BACKFILL_WARN_AFTER_FAILURES` consecutive failures one `console.warn`
@@ -101,16 +111,25 @@ import { peerJoinPushBudget } from './link-budget.js';
101
111
  // nothing at all: the connection simply never appears and only a DEBUG log mentions a dial
102
112
  // timeout.
103
113
  //
104
- // NOTE: a failing dial to a non-speaking peer costs one dial timeout per attempt, and is
105
- // never memoized (only clean runs are). Bounded by the backoff above plus the
106
- // unreachable-peer bail in `runCatchUp` — one dial per attempt, not one per chunk. If a node
107
- // ever holds many such connections, or the enumeration ahead of the first chunk gets
108
- // expensive, pre-check `libp2p.peerStore` for this network's block-transfer protocol before
109
- // enumerating.
114
+ // NOTE: the protocol check happens once — either when a connection's `peer:identify` fires,
115
+ // or, for a peer already connected when this instance starts, when `scheduleConnectedPeers()`
116
+ // reads its protocols back from the peer store — and is never repeated once a peer is
117
+ // scheduled. A peer that stops speaking the protocol after being scheduled (should that ever
118
+ // happen) is not de-scheduled; `runCatchUp`'s own unreachable-peer bail is what would catch
119
+ // that case instead. `peer:identify` also fires on every identify-push from a connected peer
120
+ // (address or protocol changes), which only re-enters the debounce — cheap, and dropped
121
+ // outright for caught-up peers and peers inside a backoff wait.
122
+ //
123
+ // NOTE: a same-network peer whose identify never completes on a connection (the identify
124
+ // stream times out over a slow relayed link, say) is not caught up until a later identify
125
+ // succeeds — a reconnect, an identify-push, or on the control network a membership-change
126
+ // re-drive. Accepted: every db-p2p node runs identify, and read repair still covers reads
127
+ // meanwhile; if relayed links ever routinely outlast identify's timeout, raise that timeout
128
+ // in the node builder rather than scheduling unidentified peers here.
110
129
  //
111
130
  // NOTE: a peer that disconnects INSIDE a backoff wait still costs the one attempt that wait
112
131
  // was already armed for — the connectivity check is made when the re-arm is decided, not when
113
- // the timer fires, and this module subscribes to `connection:open` only. One dial, not a
132
+ // the timer fires, and this module subscribes to `peer:identify` only. One dial, not a
114
133
  // recurring one; if a node ever holds many transient peers, subscribe to `connection:close`
115
134
  // and clear the pending timer there.
116
135
 
@@ -124,6 +143,18 @@ const log = debug('sereus:cadre:peer-join-backfill');
124
143
  */
125
144
  export const MAX_BLOCK_MESSAGE_BYTES = 8 * 1024 * 1024;
126
145
 
146
+ /**
147
+ * Whether an identified peer's protocol list names this network's own block-transfer
148
+ * protocol — the one gate that decides whether {@link PeerJoinBackfill} schedules a peer at
149
+ * all. A peer that lacks it (a bare circuit relay, a bootstrap node, a stranger the control
150
+ * network's inbound gate admitted without membership) cannot receive a push no matter how
151
+ * long it is dialed, since the protocol id is namespaced per network. Exported because a
152
+ * later strand peer-address ticket's identify-driven observation needs the identical test.
153
+ */
154
+ export function speaksBlockTransfer(protocols: readonly string[], protocolPrefix: string): boolean {
155
+ return protocols.includes(buildBlockTransferProtocol(protocolPrefix));
156
+ }
157
+
127
158
  /**
128
159
  * Consecutive failed catch-up runs against ONE peer before the module says so on
129
160
  * `console.warn` rather than only in its DEBUG log. At the default backoff that is reached
@@ -146,7 +177,8 @@ export interface PeerJoinBackfillConfig {
146
177
  maxChunkBlocks?: number;
147
178
  /**
148
179
  * Per-push dial deadline, ms. Default {@link peerJoinPushBudget}'s `dialTimeoutMs` — four link
149
- * round trips at the declared link, 14 000 ms as shipped. NOT a fixed number: a relayed dial
180
+ * round trips at the declared link plus the called machine's admission decision, 16 000 ms as
181
+ * shipped. NOT a fixed number: a relayed dial
150
182
  * costs a fixed number of exchanges, so a host on a slower link moves this (and every other
151
183
  * cadre dial budget) by declaring `NetworkConfig.linkRoundTripMs`. Naming it here still wins
152
184
  * over the derived value — `link-budget.ts` has the counts and the measurement.
@@ -277,7 +309,7 @@ function base64WireBytes(rawBytes: number): number {
277
309
  * Best-effort throughout: nothing here throws into a libp2p event handler or into a
278
310
  * runtime bring-up; a failed chunk is logged and the run continues. A peer is marked
279
311
  * fully caught up ONLY after a run with no thrown chunk and an empty `missing` list, so
280
- * the next `connection:open` from a peer whose catch-up failed retries it.
312
+ * the next `peer:identify` from a peer whose catch-up failed retries it.
281
313
  */
282
314
  export class PeerJoinBackfill {
283
315
  private readonly config: Required<PeerJoinBackfillConfig>;
@@ -303,31 +335,43 @@ export class PeerJoinBackfill {
303
335
  private readonly failures = new Map<string, number>();
304
336
  /**
305
337
  * Epoch ms before which a peer's catch-up must not be re-run, set alongside a backoff timer
306
- * that will run it. Its presence is what makes `connection:open` churn free: a schedule
338
+ * that will run it. Its presence is what makes `peer:identify` churn free: a schedule
307
339
  * arriving inside the wait is dropped, rather than collapsing the backoff to the debounce.
308
340
  */
309
341
  private readonly retryAfter = new Map<string, number>();
310
342
  /** Peers already reported on `console.warn`; reset when a run finally lands cleanly. */
311
343
  private readonly warned = new Set<string>();
312
- private readonly onConnectionOpen: (evt: CustomEvent<Connection>) => void;
344
+ private readonly onPeerIdentify: (evt: CustomEvent<IdentifyResult>) => void;
313
345
  private loggedNoListBlockIds = false;
314
346
 
315
347
  constructor(private readonly deps: PeerJoinBackfillDeps, config?: PeerJoinBackfillConfig) {
316
348
  this.config = { ...DEFAULT_PEER_JOIN_BACKFILL, ...config };
317
349
  this.createPushClient = deps.createPushClient
318
350
  ?? ((peerId) => new BlockTransferClient(peerId, deps.peerNetwork, deps.protocolPrefix));
319
- this.onConnectionOpen = (evt) => this.schedulePeer(evt.detail.remotePeer);
351
+ this.onPeerIdentify = (evt) => {
352
+ const { peerId, protocols } = evt.detail;
353
+ if (speaksBlockTransfer(protocols, this.deps.protocolPrefix)) {
354
+ this.schedulePeer(peerId);
355
+ }
356
+ };
320
357
  }
321
358
 
322
- /** Subscribe to connection:open AND schedule a catch-up for peers already connected. */
359
+ /**
360
+ * Subscribe to `peer:identify` AND schedule a catch-up for already-connected peers whose
361
+ * stored protocols already name this network's block-transfer protocol — a peer that does
362
+ * not speak it (a bare circuit relay, a bootstrap node) is never scheduled at all. See
363
+ * {@link speaksBlockTransfer} and the module comment.
364
+ */
323
365
  start(): void {
324
366
  if (this.started || this.stopped || !this.config.enabled) return;
325
367
  this.started = true;
326
- this.deps.libp2p.addEventListener('connection:open', this.onConnectionOpen);
327
- // A runtime rebuilt over live connections (resumeStrand) never sees their
328
- // connection:open, so walk what is already connected once.
329
- const scheduled = this.scheduleConnectedPeers();
330
- log('[%s] started (%d peer(s) already connected)', this.deps.label, scheduled);
368
+ this.deps.libp2p.addEventListener('peer:identify', this.onPeerIdentify);
369
+ // A runtime rebuilt over live connections (resumeStrand) never sees their peer:identify —
370
+ // it already fired before this instance existed, but its result is persisted in the peer
371
+ // store — so walk what is already connected once.
372
+ void this.scheduleConnectedPeers().then((scheduled) => {
373
+ log('[%s] started (%d peer(s) already connected)', this.deps.label, scheduled);
374
+ });
331
375
  }
332
376
 
333
377
  /** Unsubscribe, clear timers; in-flight runs observe the stopped flag and bail. */
@@ -335,7 +379,7 @@ export class PeerJoinBackfill {
335
379
  if (this.stopped) return;
336
380
  this.stopped = true;
337
381
  if (this.started) {
338
- this.deps.libp2p.removeEventListener('connection:open', this.onConnectionOpen);
382
+ this.deps.libp2p.removeEventListener('peer:identify', this.onPeerIdentify);
339
383
  }
340
384
  for (const timer of this.timers.values()) {
341
385
  clearTimeout(timer);
@@ -347,27 +391,46 @@ export class PeerJoinBackfill {
347
391
  }
348
392
 
349
393
  /**
350
- * (Re-)schedule a debounced catch-up for every currently-connected peer not yet caught
351
- * up. Idempotent and cheap (caught-up peers are skipped before any timer is set, and a
352
- * peer whose run is in flight is deferred to the end of that run rather than dropped).
353
- * Driven by {@link start}, and — on a gated network — by the embedder whenever
354
- * membership changes, so a peer whose first pass was denied (connected before it was
355
- * authorized: the production join order) is retried without waiting for a reconnect.
356
- * Returns how many distinct peers were considered (for the start() log) — a peer holding
357
- * several connections is one peer, and is scheduled once.
394
+ * (Re-)schedule a debounced catch-up for every currently-connected peer whose protocols, as
395
+ * already stored in the libp2p peer store, name this network's block-transfer protocol. A
396
+ * peer not yet in the peer store — its identify has not finished — is left to the
397
+ * `peer:identify` handler rather than dialed blind; one whose stored protocols are known
398
+ * and lack ours is skipped outright. Idempotent and cheap otherwise (caught-up peers are
399
+ * skipped before any timer is set, and a peer whose run is in flight is deferred to the end
400
+ * of that run rather than dropped). Driven by {@link start}, and — on a gated network — by
401
+ * the embedder whenever membership changes, so a peer whose first pass was denied (connected
402
+ * before it was authorized: the production join order) is retried without waiting for a
403
+ * reconnect. Returns how many distinct peers were considered (for the start() log) — a peer
404
+ * holding several connections is one peer, and is scheduled once.
358
405
  */
359
- scheduleConnectedPeers(): number {
406
+ async scheduleConnectedPeers(): Promise<number> {
360
407
  if (this.stopped) return 0;
361
- const peers = new Set<string>();
408
+ const peers = new Map<string, PeerId>();
362
409
  for (const connection of this.deps.libp2p.getConnections()) {
363
- const key = connection.remotePeer.toString();
364
- if (peers.has(key)) continue;
365
- peers.add(key);
366
- this.schedulePeer(connection.remotePeer);
410
+ peers.set(connection.remotePeer.toString(), connection.remotePeer);
367
411
  }
412
+ await Promise.all([...peers.values()].map((peerId) => this.scheduleIfKnownToSpeak(peerId)));
368
413
  return peers.size;
369
414
  }
370
415
 
416
+ /** Schedules `peerId` only if the peer store already knows it speaks this network's protocol. */
417
+ private async scheduleIfKnownToSpeak(peerId: PeerId): Promise<void> {
418
+ let protocols: string[];
419
+ try {
420
+ ({ protocols } = await this.deps.libp2p.peerStore.get(peerId));
421
+ } catch (error) {
422
+ // NotFoundError: identify has not finished. Leave it to the `peer:identify` handler
423
+ // rather than dialing blind. Anything else is unexpected — log it and skip the peer.
424
+ if ((error as Error).name !== 'NotFoundError') {
425
+ log('[%s] peer store read for %s failed — not scheduling: %o', this.deps.label, peerId.toString(), error);
426
+ }
427
+ return;
428
+ }
429
+ if (speaksBlockTransfer(protocols, this.deps.protocolPrefix)) {
430
+ this.schedulePeer(peerId);
431
+ }
432
+ }
433
+
371
434
  /** Debounced entry point for connection churn: one run per peer per settle window. */
372
435
  private schedulePeer(peerId: PeerId): void {
373
436
  const key = peerId.toString();
@@ -380,7 +443,7 @@ export class PeerJoinBackfill {
380
443
  }
381
444
  // A peer whose last run failed is already re-armed on a backoff timer that will run it, so
382
445
  // DROP this schedule instead of shortening the wait — otherwise a peer that cannot be
383
- // reached is re-dialled on every connection:open forever (see the module comment's retry
446
+ // reached is re-dialled on every peer:identify forever (see the module comment's retry
384
447
  // paragraph). A denied run never gets here: it does not set a backoff.
385
448
  const retryAt = this.retryAfter.get(key);
386
449
  if (retryAt !== undefined && Date.now() < retryAt) return;
@@ -420,7 +483,7 @@ export class PeerJoinBackfill {
420
483
  // rejection are all verdicts a retry cannot change, and re-arming on them would
421
484
  // re-push the whole store to that peer forever. See the module comment.
422
485
  //
423
- // Gated on `started` too: the re-arm exists to REPLACE a connection:open-driven retry,
486
+ // Gated on `started` too: the re-arm exists to REPLACE a peer:identify-driven retry,
424
487
  // so a caller driving `catchUpPeer` by hand against a backfill that was never started
425
488
  // owns its own retry policy and must not be left holding a background timer.
426
489
  this.scheduleRetryWithBackoff(peerId);
@@ -452,7 +515,7 @@ export class PeerJoinBackfill {
452
515
  /**
453
516
  * Re-arm one peer's catch-up after a run whose push failed, on a wait that doubles
454
517
  * per consecutive failure up to `maxRetryBackoffMs`. Sets {@link retryAfter} alongside the
455
- * timer, which is what makes `connection:open` churn inside the wait free.
518
+ * timer, which is what makes `peer:identify` churn inside the wait free.
456
519
  */
457
520
  private scheduleRetryWithBackoff(peerId: PeerId): void {
458
521
  const key = peerId.toString();
@@ -198,6 +198,12 @@ export class PushFanoutService {
198
198
  * double-wake). Only a dial/transport REJECTION means the phone is suspended
199
199
  * and unreachable over the control network, which is the one case that falls
200
200
  * through to FCM/APNs.
201
+ *
202
+ * NOTE: the platform push waits for the whole direct dial to fail, up to
203
+ * `DEFAULT_WAKE_DIAL_BUDGET_MS` (46 s at the default declared link). A suspended
204
+ * phone typically fails its relay address fast and spends the rest on its direct
205
+ * one. If that delay shows up, start the platform push in parallel with the dial,
206
+ * or after its first attempt.
201
207
  */
202
208
  private async wakePeer(peerId: string, strandId: string, reason: string): Promise<void> {
203
209
  // A wake of either kind is now being attempted → arm the cooldown up front.
@@ -143,7 +143,9 @@ export interface RelayReservationState {
143
143
  /**
144
144
  * How long {@link driveRelayReservation} waits for a reservation to appear, at the DEFAULT
145
145
  * declared link round trip: `RELAY_RESERVATION_ROUND_TRIPS` (4) x `DECLARED_LINK_ROUND_TRIP_MS`
146
- * (3500 ms) = 14 000 ms, where it was a fixed 10_000 before it was derived.
146
+ * (3500 ms), plus 2 x `ADMISSION_DECISION_TIMEOUT_MS` (2000 ms) for a party-run relay's
147
+ * connection and reservation decisions, = 18 000 ms (`relayReservationBudgetMs`), where it was a
148
+ * fixed 10_000 before it was derived.
147
149
  *
148
150
  * Counted rather than chosen because this one deadline bounds the whole drive — dial the relay,
149
151
  * request the reservation, wait — and each of those costs a fixed number of exchanges, not a
@@ -0,0 +1,98 @@
1
+ /**
2
+ * The circuit-relay SERVER a party-run node runs — whether it runs one, and with which
3
+ * `@libp2p/circuit-relay-v2` init — resolved once from `NetworkConfig` for both node
4
+ * kinds: the control node (`cadre-node.ts` → `buildControlNodeOptions`) and every strand
5
+ * node (`strand-instance-manager.ts` → `buildStrandRuntime`). One function decides both
6
+ * answers so the two build sites cannot drift apart.
7
+ *
8
+ * WHY cadre sets an init at all. libp2p's server defaults `applyDefaultLimit` to true,
9
+ * which stamps every reservation with `Limit { data: 128 KiB, duration: 2 min }`; libp2p
10
+ * marks each connection relayed under it "limited" and resets it once either cap is
11
+ * reached, and db-p2p's database protocols refuse a limited connection outright (they do
12
+ * not set `runOnLimitedConnection`). A party-run relay exists to carry that party's own
13
+ * database traffic, so the cap cut off every sync and chat history forwarded through it
14
+ * (gotchoices/sereus#19). The dedicated relays (`ops/docker/libp2p-infra`, and the
15
+ * `startDedicatedRelay` test fixture that stands in for it) already turn the limit off;
16
+ * this is the same policy for the party-run kind.
17
+ */
18
+
19
+ import type { CircuitRelayServerInit, ServerReservationStoreInit } from '@libp2p/circuit-relay-v2';
20
+ import type { NetworkConfig, NodeProfile } from './types.js';
21
+
22
+ /**
23
+ * Default size of a party-run relay's reservation store, raised from libp2p's 15.
24
+ *
25
+ * A NAT'd machine reserves one slot for its control node and one for each strand it
26
+ * serves, all on the same relay (`strand-network-config.ts` hands strand nodes the
27
+ * control node's `network.relayAddrs`), so the store needs
28
+ * `NAT'd machines × (1 + strands each)` slots. At libp2p's 15, three phones serving four
29
+ * strands each already fill it. 128 covers, for example, ten NAT'd machines serving eleven
30
+ * strands each with the unauthorized-reservation budget's 8 slots
31
+ * (`MAX_UNAUTHORIZED_RELAY_RESERVATIONS`) still free on top — one party's worth, where the
32
+ * dedicated relay's 500 is sized for many parties. A reservation costs the server a map
33
+ * entry and a timer, not a connection's worth of state, so the headroom is cheap.
34
+ */
35
+ export const PARTY_RELAY_MAX_RESERVATIONS = 128;
36
+
37
+ /**
38
+ * How long a party-run relay holds a reservation that is not refreshed: libp2p's own
39
+ * default (2 h), set EXPLICITLY so that cadre, not a libp2p constant it cannot import,
40
+ * owns the value the unauthorized-reservation budget has to agree with (see
41
+ * {@link ResolvedRelayServer.init}).
42
+ */
43
+ export const PARTY_RELAY_RESERVATION_TTL_MS = 2 * 60 * 60 * 1000;
44
+
45
+ /**
46
+ * The reservation-store settings every party-run relay resolves unless the caller
47
+ * overrides them one key at a time through `network.relayServerInit.reservations`.
48
+ * `applyDefaultLimit: false` is the point of this module — see the module doc.
49
+ */
50
+ const PARTY_RELAY_RESERVATION_DEFAULTS = {
51
+ applyDefaultLimit: false,
52
+ maxReservations: PARTY_RELAY_MAX_RESERVATIONS,
53
+ reservationTtl: PARTY_RELAY_RESERVATION_TTL_MS
54
+ } as const satisfies ServerReservationStoreInit;
55
+
56
+ /** Reservation settings with the three keys cadre always resolves made definite. */
57
+ export type ResolvedRelayReservations = ServerReservationStoreInit
58
+ & Required<Pick<ServerReservationStoreInit, keyof typeof PARTY_RELAY_RESERVATION_DEFAULTS>>;
59
+
60
+ /** What {@link resolveRelayServer} decides for one node. */
61
+ export interface ResolvedRelayServer {
62
+ /** Does this node run the relay server? Passed as db-p2p's `relay`. */
63
+ enabled: boolean;
64
+ /**
65
+ * The server's init, passed as db-p2p's `relayServerInit` when {@link enabled}: the
66
+ * party-run defaults with the caller's `network.relayServerInit` merged over them.
67
+ * `reservations.reservationTtl` is always set, which is what lets the control node's
68
+ * unauthorized-reservation budget expire its entries on the server's own clock.
69
+ */
70
+ init: CircuitRelayServerInit & { reservations: ResolvedRelayReservations };
71
+ }
72
+
73
+ /**
74
+ * Resolve the relay server for a node built from `network` with `profile`.
75
+ *
76
+ * `enabled`: an explicit `network.enableRelay` wins; otherwise storage-profile nodes run
77
+ * the server (they are the machines with the connectivity and uptime to be a relay).
78
+ *
79
+ * `init`: MERGED, not replaced. `reservations` is merged key by key over the party-run
80
+ * defaults, so a caller who sets only `maxReservations` does not silently get the data
81
+ * limit back; every other top-level key is taken from the caller as given. A key set to
82
+ * `undefined` counts as unset, so the default beneath it holds.
83
+ */
84
+ export function resolveRelayServer(network: NetworkConfig | undefined, profile: NodeProfile): ResolvedRelayServer {
85
+ const override = network?.relayServerInit;
86
+ return {
87
+ enabled: network?.enableRelay ?? (profile === 'storage'),
88
+ init: {
89
+ ...override,
90
+ reservations: { ...PARTY_RELAY_RESERVATION_DEFAULTS, ...definedEntries(override?.reservations) }
91
+ }
92
+ };
93
+ }
94
+
95
+ /** `obj` without its `undefined`-valued keys, so spreading it never erases a default. */
96
+ function definedEntries<T extends object>(obj: T | undefined): Partial<T> {
97
+ return Object.fromEntries(Object.entries(obj ?? {}).filter(([, value]) => value !== undefined)) as Partial<T>;
98
+ }