@serfab/cadre-core 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -255
- package/dist/announce-addrs.d.ts +49 -0
- package/dist/announce-addrs.js +78 -0
- package/dist/announce-addrs.js.map +1 -0
- package/dist/cadre-node.d.ts +484 -58
- package/dist/cadre-node.js +953 -182
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +26 -0
- package/dist/control-cohort.js +26 -0
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +222 -125
- package/dist/control-database.js +356 -266
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +8 -7
- package/dist/control-formation-recorder.js +8 -7
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +103 -0
- package/dist/control-read-retry.js +145 -0
- package/dist/control-read-retry.js.map +1 -0
- package/dist/control-retry.d.ts +101 -0
- package/dist/control-retry.js +108 -0
- package/dist/control-retry.js.map +1 -0
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +805 -774
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +65 -31
- package/dist/control-write-retry.js +33 -103
- package/dist/control-write-retry.js.map +1 -1
- package/dist/delegate-admission.d.ts +17 -15
- package/dist/delegate-admission.js +17 -15
- package/dist/delegate-admission.js.map +1 -1
- package/dist/index.d.ts +7 -5
- package/dist/index.js +18 -9
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +211 -25
- package/dist/membership-connection-gater.js +309 -29
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/peer-addr-book.d.ts +92 -0
- package/dist/peer-addr-book.js +220 -0
- package/dist/peer-addr-book.js.map +1 -0
- package/dist/peer-record.d.ts +26 -0
- package/dist/peer-record.js +32 -0
- package/dist/peer-record.js.map +1 -1
- package/dist/relay-addrs.d.ts +76 -26
- package/dist/relay-addrs.js +115 -27
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +20 -15
- package/dist/relay-reservation.js +18 -14
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +7 -0
- package/dist/seed-bootstrap.js +15 -13
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-addr-protocol.js +16 -0
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-backfill.js +13 -4
- package/dist/strand-backfill.js.map +1 -1
- package/dist/strand-cohort.d.ts +4 -37
- package/dist/strand-cohort.js +4 -15
- package/dist/strand-cohort.js.map +1 -1
- package/dist/strand-database.d.ts +11 -16
- package/dist/strand-database.js +19 -7
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +18 -17
- package/dist/strand-formation-manager.js +25 -27
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +7 -7
- package/dist/strand-formation-protocol.js +8 -8
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +53 -21
- package/dist/strand-instance-manager.js +128 -37
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +113 -19
- package/dist/strand-membership-writer.js +219 -51
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +3 -3
- package/dist/strand-solicitation.js +4 -5
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +47 -2
- package/dist/strand-wake-protocol.js +76 -6
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/types.d.ts +107 -54
- package/dist/types.js.map +1 -1
- package/package.json +8 -7
- package/src/announce-addrs.ts +93 -0
- package/src/arachnode-stub.ts +159 -159
- package/src/bootstrap-peer-store-file.ts +44 -44
- package/src/bootstrap-peer-store.ts +172 -172
- package/src/cadre-node.ts +1026 -194
- package/src/canonical-json.ts +30 -30
- package/src/control-authorization.ts +114 -114
- package/src/control-cohort.ts +130 -103
- package/src/control-database.ts +376 -298
- package/src/control-formation-recorder.ts +305 -304
- package/src/control-read-retry.ts +167 -0
- package/src/control-retry.ts +190 -0
- package/src/control-schema.ts +817 -786
- package/src/control-stream.ts +205 -205
- package/src/control-write-retry.ts +383 -473
- package/src/delegate-admission.ts +304 -302
- package/src/device-token.ts +94 -94
- package/src/diagnostics/connection-path.ts +277 -277
- package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
- package/src/ed25519-key.ts +134 -134
- package/src/enrollment.ts +199 -199
- package/src/file-durable-slot.ts +61 -61
- package/src/formation-approval.ts +604 -604
- package/src/fs-atomic.ts +107 -107
- package/src/hibernation-manager.ts +437 -437
- package/src/identity-key.ts +109 -109
- package/src/index.ts +435 -406
- package/src/key-store-file.ts +132 -132
- package/src/key-store.ts +99 -99
- package/src/membership-connection-gater.ts +368 -39
- package/src/node-local-snapshot.ts +260 -260
- package/src/peer-addr-book.ts +250 -0
- package/src/peer-authorization.ts +259 -259
- package/src/peer-record.ts +197 -164
- package/src/push-credentials.ts +84 -84
- package/src/push-node.ts +64 -64
- package/src/push-notifier-apns.ts +247 -247
- package/src/push-notifier-fcm.ts +220 -220
- package/src/push-notifier-shared.ts +28 -28
- package/src/push-notifier.ts +52 -52
- package/src/relay-addrs.ts +134 -27
- package/src/relay-reservation.ts +22 -17
- package/src/seed-bootstrap.ts +15 -15
- package/src/seed-trust-policy.ts +140 -140
- package/src/strand-addr-protocol.ts +458 -441
- package/src/strand-backfill.ts +28 -5
- package/src/strand-cohort.ts +38 -77
- package/src/strand-database.ts +202 -204
- package/src/strand-formation-manager.ts +26 -28
- package/src/strand-formation-protocol.ts +692 -692
- package/src/strand-instance-manager.ts +636 -553
- package/src/strand-member-key.ts +39 -39
- package/src/strand-member-registry.ts +182 -182
- package/src/strand-membership-writer.ts +1627 -1430
- package/src/strand-solicitation.ts +471 -472
- package/src/strand-transport-key.ts +58 -58
- package/src/strand-wake-payload.ts +32 -32
- package/src/strand-wake-protocol.ts +400 -314
- package/src/strand-watcher.ts +314 -314
- package/src/trusted-owner-store-file.ts +41 -41
- package/src/trusted-owner-store.ts +171 -171
- package/src/types.ts +1354 -1303
package/dist/cadre-node.js
CHANGED
|
@@ -2,6 +2,7 @@ import debug from 'debug';
|
|
|
2
2
|
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
3
3
|
import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
4
4
|
import { createLibp2pNode } from '@optimystic/db-p2p';
|
|
5
|
+
import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
|
|
5
6
|
import { multiaddr } from '@multiformats/multiaddr';
|
|
6
7
|
import { CONTROL_CLUSTER_POLICY, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
7
8
|
import { sign } from '@optimystic/quereus-plugin-crypto';
|
|
@@ -11,24 +12,27 @@ import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
|
11
12
|
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
12
13
|
import { MemoryTrustedOwnerStore } from './trusted-owner-store.js';
|
|
13
14
|
import { MemoryBootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
15
|
+
import { mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
14
16
|
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
15
17
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
16
|
-
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr,
|
|
18
|
+
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, withTrailingPeerId, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
17
19
|
import { signDeviceTokenRecord, verifyDeviceTokenSignature, isPushPlatform } from './device-token.js';
|
|
18
20
|
import { StrandWatcher } from './strand-watcher.js';
|
|
19
21
|
import { StrandInstanceManager } from './strand-instance-manager.js';
|
|
20
|
-
import { deriveCohortMembers
|
|
21
|
-
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
22
|
+
import { deriveCohortMembers } from './strand-cohort.js';
|
|
23
|
+
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
24
|
+
import { withDeadline } from './control-stream.js';
|
|
22
25
|
import { EnrollmentService } from './enrollment.js';
|
|
23
26
|
import { HibernationManager } from './hibernation-manager.js';
|
|
24
27
|
import { ControlDatabase } from './control-database.js';
|
|
25
28
|
import { SeedBootstrapService } from './seed-bootstrap.js';
|
|
26
29
|
import { StrandSolicitationService } from './strand-solicitation.js';
|
|
27
|
-
import { createMembershipConnectionGater, DEFAULT_ENROLLMENT_WINDOW_MS } from './membership-connection-gater.js';
|
|
30
|
+
import { createMembershipConnectionGater, DEFAULT_ENROLLMENT_WINDOW_MS, UnauthorizedReservationBudget } from './membership-connection-gater.js';
|
|
28
31
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
29
32
|
import { StrandAddrService, collectStrandAddrs } from './strand-addr-protocol.js';
|
|
30
33
|
import { DelegateAdmissionStore, extractCircuitRelayTargets, dueRelayAnnounces, pruneStoppedStrandAnnounces, peerStrandKey } from './delegate-admission.js';
|
|
31
|
-
import { resolveListenAddrs } from './relay-addrs.js';
|
|
34
|
+
import { relayCircuitAddrs, resolveListenAddrs, RelayReservationFailedError } from './relay-addrs.js';
|
|
35
|
+
import { replacesAdvertisedAddrs, resolveAnnounceAddrs } from './announce-addrs.js';
|
|
32
36
|
import { superviseRelayReservation, resolveRelayReservationState } from './relay-reservation.js';
|
|
33
37
|
import { PushFanoutService } from './push-fanout.js';
|
|
34
38
|
import { summarizeConnectionPaths, classifyTransport } from './diagnostics/connection-path.js';
|
|
@@ -41,6 +45,18 @@ const timing = debug('sereus:cadre:timing');
|
|
|
41
45
|
* `RTCPeerConnection` reaching `connected` and libp2p surfacing the connection.
|
|
42
46
|
*/
|
|
43
47
|
const TURN_CONSUME_WINDOW_MS = 1000;
|
|
48
|
+
/**
|
|
49
|
+
* How often a running strand re-resolves its siblings' strand-network addresses
|
|
50
|
+
* and re-merges them into its own libp2p address book
|
|
51
|
+
* ({@link CadreNode.refreshStrandPeerAddrs}). Overridable per node via
|
|
52
|
+
* `network.controlCohort.strandAddrRefreshMs`.
|
|
53
|
+
*
|
|
54
|
+
* Ten minutes sits comfortably inside the peerStore's one-hour address expiry
|
|
55
|
+
* (`MAX_ADDRESS_AGE`, see `peer-addr-book.ts`) with headroom for several missed
|
|
56
|
+
* passes, and far above the 15 s reconcile cadence the refresh rides on, so the
|
|
57
|
+
* strand-addr RPC fan-out stays cheap.
|
|
58
|
+
*/
|
|
59
|
+
export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
44
60
|
/**
|
|
45
61
|
* A circuit relay as a strand-addr RPC target: dialed by peerId, with its
|
|
46
62
|
* direct addr riding along as the fallback for a relay we hold no connection to.
|
|
@@ -110,6 +126,14 @@ export class CadreNode {
|
|
|
110
126
|
this.pushFanoutService = null;
|
|
111
127
|
/** Backing field for the {@link running} / {@link isRunning} getters. */
|
|
112
128
|
this._running = false;
|
|
129
|
+
/**
|
|
130
|
+
* The control database's own raw storage (cache-wrapped), resolved once per
|
|
131
|
+
* `start()` and released in {@link cleanup}. Owning it is what keeps
|
|
132
|
+
* `config.storage.provider('control')` a once-per-runtime call: a `stop()` then
|
|
133
|
+
* `start()` cycle on this object re-resolves against a live cache, and never
|
|
134
|
+
* orphans the previous wrapper's registration in the shared cache pool.
|
|
135
|
+
*/
|
|
136
|
+
this.controlStorage = null;
|
|
113
137
|
/**
|
|
114
138
|
* In-flight {@link serviceWake} operations keyed by strandId. Coalesces
|
|
115
139
|
* concurrent on-demand wakes for the same strand into one runtime build + one
|
|
@@ -125,10 +149,11 @@ export class CadreNode {
|
|
|
125
149
|
this.windowWaiters = new Set();
|
|
126
150
|
this.eventHandlers = new Map();
|
|
127
151
|
/**
|
|
128
|
-
* Relay multiaddrs the caller asked {@link reserveRelays} to reserve through
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
152
|
+
* Relay multiaddrs the caller asked {@link reserveRelays} to reserve through —
|
|
153
|
+
* or, on a node that names `network.relayAddrs`, the ones
|
|
154
|
+
* {@link driveControlRelayReservation} asked for at the end of {@link start}.
|
|
155
|
+
* Empty (the default) means neither happened, so {@link getRelayReservationState}
|
|
156
|
+
* reports `none`.
|
|
132
157
|
*/
|
|
133
158
|
this.relayReserveAddrs = [];
|
|
134
159
|
/**
|
|
@@ -143,6 +168,23 @@ export class CadreNode {
|
|
|
143
168
|
* supervisor to hold one — i.e. the pre-start `control node unavailable` case.
|
|
144
169
|
*/
|
|
145
170
|
this.relayReserveError = null;
|
|
171
|
+
/**
|
|
172
|
+
* True from just before the control libp2p node is created until
|
|
173
|
+
* {@link ControlDatabase.initialize} has settled — the connection gate's
|
|
174
|
+
* BRING-UP QUIET PERIOD (`membership-connection-gater.ts`), during which this
|
|
175
|
+
* node refuses every control connection in both directions.
|
|
176
|
+
*
|
|
177
|
+
* The invariant: the control database is built while this node holds zero
|
|
178
|
+
* control connections. Every same-party sibling connected in that window joins
|
|
179
|
+
* the Optimystic cohort the bring-up's block probes consult, and a sibling that
|
|
180
|
+
* has not yet replicated this node's `CadrePeer` row refuses them all — which
|
|
181
|
+
* fails `start()` outright and cannot be retried into convergence (writing the
|
|
182
|
+
* row that would clear the refusal needs the database being built).
|
|
183
|
+
*
|
|
184
|
+
* Cleared on FAILURE as well as success ({@link cleanup} clears it), so
|
|
185
|
+
* teardown is never gated.
|
|
186
|
+
*/
|
|
187
|
+
this.controlBringUpInFlight = false;
|
|
146
188
|
/** Map of strandId -> sAppConfig for sAppId filtering and management */
|
|
147
189
|
this.sAppConfigs = new Map();
|
|
148
190
|
/**
|
|
@@ -203,10 +245,12 @@ export class CadreNode {
|
|
|
203
245
|
/**
|
|
204
246
|
* In-memory delegate admission grants (see `delegate-admission.ts`): the
|
|
205
247
|
* strand-node transport peerIds this party's members have announced over the
|
|
206
|
-
* strand-addr RPC, admitted at the CONNECTION
|
|
207
|
-
* strand node can hold a circuit-relay reservation on this
|
|
208
|
-
*
|
|
209
|
-
*
|
|
248
|
+
* strand-addr RPC, admitted at the CONNECTION and RESERVATION levels so a
|
|
249
|
+
* member's NAT'd strand node can hold a circuit-relay reservation on this
|
|
250
|
+
* node (without spending the unauthorized-reservation budget). Consulted
|
|
251
|
+
* ONLY by {@link admitInboundControlConnection} and
|
|
252
|
+
* {@link admitControlRelayReservation}; the fail-closed per-stream gate
|
|
253
|
+
* ({@link authorizeInboundControlStream}) never honors it.
|
|
210
254
|
*/
|
|
211
255
|
this.delegateAdmission = new DelegateAdmissionStore();
|
|
212
256
|
/**
|
|
@@ -218,6 +262,16 @@ export class CadreNode {
|
|
|
218
262
|
* strand is no longer running are pruned on each reconcile pass.
|
|
219
263
|
*/
|
|
220
264
|
this.delegateAnnounceAt = new Map();
|
|
265
|
+
/**
|
|
266
|
+
* When each running strand last had its siblings' strand addresses re-resolved
|
|
267
|
+
* into its own address book, keyed by strandId. Throttles
|
|
268
|
+
* {@link refreshStrandPeerAddrs} to once per {@link STRAND_PEER_ADDR_REFRESH_MS}
|
|
269
|
+
* per strand so the 15 s reconcile tick never becomes per-tick RPC chatter.
|
|
270
|
+
* Entries for strands that are no longer running are pruned on each pass, so a
|
|
271
|
+
* hibernated-then-resumed strand refreshes immediately rather than inheriting a
|
|
272
|
+
* stale stamp.
|
|
273
|
+
*/
|
|
274
|
+
this.strandPeerAddrRefreshAt = new Map();
|
|
221
275
|
/**
|
|
222
276
|
* Cold-start bootstrap dial targets: the owner-flagged peers of every seed
|
|
223
277
|
* this node has applied, keyed by peer id, valued by the seed's multiaddr
|
|
@@ -357,6 +411,7 @@ export class CadreNode {
|
|
|
357
411
|
this.config = config;
|
|
358
412
|
this.strandManager = new StrandInstanceManager();
|
|
359
413
|
this.enrollmentService = new EnrollmentService();
|
|
414
|
+
this.unauthorizedRelayReservations = new UnauthorizedReservationBudget(config.network?.unauthorizedRelayReservationCap);
|
|
360
415
|
// Create hibernation manager with callbacks
|
|
361
416
|
const hibernationCallbacks = {
|
|
362
417
|
onIdle: async (strandId) => this.handleStrandIdle(strandId),
|
|
@@ -468,16 +523,12 @@ export class CadreNode {
|
|
|
468
523
|
return;
|
|
469
524
|
}
|
|
470
525
|
log('Starting CadreNode for party: %s', this.config.controlNetwork.partyId);
|
|
471
|
-
// NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
472
|
-
// diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
473
|
-
// second such warning ever appears here, route both through a `CadreNodeEvents` entry the
|
|
474
|
-
// embedder surfaces instead of growing a console surface inside a library.
|
|
475
|
-
if (this.config.network?.announceAddrs && this.config.network.announceAddrs.length > 0) {
|
|
476
|
-
console.warn('network.announceAddrs is set but not yet supported (no upstream db-p2p option to apply it) ' +
|
|
477
|
-
'— this node will keep advertising its listen/relay addresses instead.');
|
|
478
|
-
}
|
|
479
526
|
try {
|
|
480
527
|
const tTotal = performance.now();
|
|
528
|
+
// Config pre-flight, inside the guarded region: resolving the listen addrs it reads
|
|
529
|
+
// throws on a malformed `relayAddrs` entry, and that failure belongs on the same
|
|
530
|
+
// logged-and-cleaned-up path as every other one in start().
|
|
531
|
+
this.warnIfAnnounceAddrsDiscardRelay();
|
|
481
532
|
// Install the WebRTC TURN-relay tracker BEFORE any libp2p bring-up, so it
|
|
482
533
|
// wraps globalThis.RTCPeerConnection before the control node can create one.
|
|
483
534
|
// Inert on Node.js (no RTCPeerConnection); disposed in cleanup().
|
|
@@ -496,6 +547,24 @@ export class CadreNode {
|
|
|
496
547
|
// closed before any network bring-up, and the retained dial targets are
|
|
497
548
|
// loaded before the first reconcile pass could consult them.
|
|
498
549
|
this.initializeBootstrapPeerStore();
|
|
550
|
+
// Arm the connection gate's BRING-UP QUIET PERIOD before the libp2p node
|
|
551
|
+
// exists, so no connection can form ahead of it: `@libp2p/bootstrap` emits
|
|
552
|
+
// its discovery events ~1 s after `libp2p.start()` and the connection
|
|
553
|
+
// manager auto-dials from there, which is INSIDE createControlNode below.
|
|
554
|
+
// The invariant being protected is "zero control connections during
|
|
555
|
+
// control-database bring-up" — see the field's doc and
|
|
556
|
+
// `membership-connection-gater.ts` → "The bring-up quiet period".
|
|
557
|
+
//
|
|
558
|
+
// NOTE: without this gate the bootstrap arm is a RACE, not a guarantee — its
|
|
559
|
+
// safety margin is bring-up duration versus @libp2p/bootstrap's 1 s discovery
|
|
560
|
+
// fuse, and bring-up duration is (raw-storage operations) × per-operation
|
|
561
|
+
// latency. The cold-start operation count and the 50-90 ms/op figure a loaded
|
|
562
|
+
// phone actually sees live in `control-database.ts`'s `loadSchema` note (and
|
|
563
|
+
// are pinned by `control-start-storage-op-budget.spec.ts`): at the top of that
|
|
564
|
+
// range bring-up is 9-15 s, far past the fuse. So if this gate is ever removed
|
|
565
|
+
// or bypassed, that arm reopens for exactly the nodes most likely to hit it —
|
|
566
|
+
// phones joining a party, whose membership row has not replicated yet.
|
|
567
|
+
this.controlBringUpInFlight = true;
|
|
499
568
|
// Create the control network libp2p node
|
|
500
569
|
let t0 = performance.now();
|
|
501
570
|
this.controlNode = await this.createControlNode();
|
|
@@ -516,6 +585,21 @@ export class CadreNode {
|
|
|
516
585
|
});
|
|
517
586
|
await this.controlDatabase.initialize();
|
|
518
587
|
timing('[start] controlDatabase.initialize: %dms', Math.round(performance.now() - t0));
|
|
588
|
+
// The database exists locally now, so the quiet period has done its job and
|
|
589
|
+
// this node can be in the conversation. A failure above never reaches here —
|
|
590
|
+
// `cleanup()` on the catch path opens the gate instead, so teardown (and a
|
|
591
|
+
// retry by the embedder) is not gated.
|
|
592
|
+
//
|
|
593
|
+
// NOTE: the window ends HERE, not at the end of start(), so the awaited control-DB
|
|
594
|
+
// work that follows — `strandWatcher.start()`'s first poll — can run against a
|
|
595
|
+
// cohort a connection just joined. Fine now: every block that poll reads was
|
|
596
|
+
// created locally by the initialize() above, so a refusing sibling has nothing to
|
|
597
|
+
// refuse. If a boot ever fails with `BlockUnavailableError` from BELOW this line,
|
|
598
|
+
// that assumption broke. Do not simply move this clear later without measuring:
|
|
599
|
+
// `handleStrandAdded` runs inside `strandWatcher.start()` and seeds each strand
|
|
600
|
+
// node from CONNECTED siblings (`resolveCohortSeed`), so a longer window trades a
|
|
601
|
+
// bring-up hazard for boot-time strand seeds that are always empty.
|
|
602
|
+
this.controlBringUpInFlight = false;
|
|
519
603
|
log('Control database initialized');
|
|
520
604
|
// Attach the per-stream gate to the control DB's membership hub, so every
|
|
521
605
|
// committed `CadrePeer` write re-materializes the authorized snapshot on its
|
|
@@ -581,12 +665,27 @@ export class CadreNode {
|
|
|
581
665
|
// lands the snapshot is empty, which the stream gate treats as cold
|
|
582
666
|
// start and admits.
|
|
583
667
|
void this.refreshMembershipGate('start');
|
|
668
|
+
// Reserve the configured relays LAST, so every step above ran against a
|
|
669
|
+
// cohort of one (see driveControlRelayReservation). Ahead of
|
|
670
|
+
// scheduleSelfRegistration, so this node's first published `CadrePeer` row
|
|
671
|
+
// already carries the `/p2p-circuit` address the reservation earns it.
|
|
672
|
+
// Throws on a relay that will not have us — start() is fail-fast for a node
|
|
673
|
+
// whose operator named a relay.
|
|
674
|
+
await this.driveControlRelayReservation();
|
|
584
675
|
// Schedule self-registration in background
|
|
585
676
|
this.scheduleSelfRegistration();
|
|
586
677
|
}
|
|
587
678
|
catch (error) {
|
|
588
679
|
log('Failed to start CadreNode: %o', error);
|
|
680
|
+
// `driveControlRelayReservation` throws AFTER `control:connected` was emitted,
|
|
681
|
+
// so a subscriber can have seen this node come up. Balance the edge rather than
|
|
682
|
+
// leaving it dangling — `cleanup()` has torn the node down either way, and the
|
|
683
|
+
// events are what an embedder drives its "am I connected" UI from.
|
|
684
|
+
const wasConnected = this._running;
|
|
589
685
|
await this.cleanup();
|
|
686
|
+
if (wasConnected) {
|
|
687
|
+
this.emit('control:disconnected', undefined);
|
|
688
|
+
}
|
|
590
689
|
throw error;
|
|
591
690
|
}
|
|
592
691
|
}
|
|
@@ -624,19 +723,7 @@ export class CadreNode {
|
|
|
624
723
|
return;
|
|
625
724
|
}
|
|
626
725
|
log('Stopping CadreNode');
|
|
627
|
-
// Stop the retry loop BEFORE tearing the node down, not after: `cleanup()`
|
|
628
|
-
// stops the control node partway through, and a tick that fires during it
|
|
629
|
-
// would dial and then poll a half-torn-down node for the whole reserve
|
|
630
|
-
// timeout — logging spurious dial failures and holding the process open.
|
|
631
|
-
this.relayReserveSupervisor?.stop();
|
|
632
|
-
this.relayReserveSupervisor = null;
|
|
633
726
|
await this.cleanup();
|
|
634
|
-
// Drop the rest of the relay-reservation posture: a stopped node holds no
|
|
635
|
-
// reservation, so a restarted instance must not report the previous run's
|
|
636
|
-
// `reserved`.
|
|
637
|
-
this.relayReserveAddrs = [];
|
|
638
|
-
this.relayReserveError = null;
|
|
639
|
-
this._running = false;
|
|
640
727
|
this.emit('control:disconnected', undefined);
|
|
641
728
|
log('CadreNode stopped');
|
|
642
729
|
}
|
|
@@ -805,6 +892,72 @@ export class CadreNode {
|
|
|
805
892
|
async createControlNode() {
|
|
806
893
|
return await createLibp2pNode(this.buildControlNodeOptions());
|
|
807
894
|
}
|
|
895
|
+
/**
|
|
896
|
+
* The control database's cache-wrapped raw storage, resolved from
|
|
897
|
+
* `config.storage.provider` on first use and then held in {@link controlStorage}
|
|
898
|
+
* for the rest of this runtime.
|
|
899
|
+
*
|
|
900
|
+
* Resolving once is the contract `RawStorageProvider` (types.ts) states: a provider is
|
|
901
|
+
* called once per scope per runtime lifetime. Re-entering it per call would mint a
|
|
902
|
+
* second store over one backend — a fresh, cold cache, with the previous wrapper's
|
|
903
|
+
* registration orphaned in the process-wide pool.
|
|
904
|
+
*
|
|
905
|
+
* Lazy rather than eager in `start()` so the pure-unit call path
|
|
906
|
+
* (`cadre-node-control-node-options.spec.ts` calls `buildControlNodeOptions` on a
|
|
907
|
+
* bare `new CadreNode`) still resolves — and, on that path too, only once.
|
|
908
|
+
*
|
|
909
|
+
* Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's
|
|
910
|
+
* `cached-storage.ts`) because a control start's cost is its raw-storage
|
|
911
|
+
* operation count.
|
|
912
|
+
*/
|
|
913
|
+
resolveControlStorage() {
|
|
914
|
+
if (this.controlStorage) {
|
|
915
|
+
return this.controlStorage;
|
|
916
|
+
}
|
|
917
|
+
const provider = this.config.storage?.provider;
|
|
918
|
+
if (!provider) {
|
|
919
|
+
return undefined;
|
|
920
|
+
}
|
|
921
|
+
const resolved = typeof provider === 'function' ? provider('control') : provider;
|
|
922
|
+
this.controlStorage = wrapStorageWithCache(resolved, 'control');
|
|
923
|
+
return this.controlStorage;
|
|
924
|
+
}
|
|
925
|
+
/**
|
|
926
|
+
* Warn the operator when `network.announceAddrs` will silently cost this node the
|
|
927
|
+
* relay reachability it also configured.
|
|
928
|
+
*
|
|
929
|
+
* A non-empty announce set REPLACES everything libp2p advertises, so the
|
|
930
|
+
* `/p2p-circuit` address earned by a relay reservation is dropped from the node's
|
|
931
|
+
* advertised addresses even though the reservation itself is still held — peers
|
|
932
|
+
* behind NAT stop being able to reach it through that relay. Not an error: an
|
|
933
|
+
* operator whose relay slot is decorative may genuinely want only the announced
|
|
934
|
+
* address, so this reports and proceeds.
|
|
935
|
+
*
|
|
936
|
+
* Keyed off the RESOLVED listen addrs rather than `relayAddrs` alone, so a
|
|
937
|
+
* hand-written `/p2p-circuit` entry in `listenAddrs` — the same reservation by the
|
|
938
|
+
* longer route — is caught too.
|
|
939
|
+
*
|
|
940
|
+
* NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
941
|
+
* diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
942
|
+
* second such warning ever appears here, route both through a `CadreNodeEvents` entry the
|
|
943
|
+
* embedder surfaces instead of growing a console surface inside a library.
|
|
944
|
+
*/
|
|
945
|
+
warnIfAnnounceAddrsDiscardRelay() {
|
|
946
|
+
const network = this.config.network;
|
|
947
|
+
if (!replacesAdvertisedAddrs(network)) {
|
|
948
|
+
return;
|
|
949
|
+
}
|
|
950
|
+
const listensOnCircuit = (resolveListenAddrs(network) ?? [])
|
|
951
|
+
.some((addr) => addr.includes('/p2p-circuit'));
|
|
952
|
+
if (!listensOnCircuit) {
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
console.warn('network.announceAddrs is set alongside a circuit-relay listener (from network.relayAddrs, ' +
|
|
956
|
+
'or a /p2p-circuit entry in network.listenAddrs). A non-empty announce set REPLACES every ' +
|
|
957
|
+
'address this node advertises, so the /p2p-circuit address earned from the relay reservation ' +
|
|
958
|
+
'will not be advertised and peers will stop reaching this node through that relay. ' +
|
|
959
|
+
'Use network.appendAnnounceAddrs instead to advertise an extra address without discarding the rest.');
|
|
960
|
+
}
|
|
808
961
|
/**
|
|
809
962
|
* Map this node's config onto the control network's libp2p node options.
|
|
810
963
|
*
|
|
@@ -814,19 +967,17 @@ export class CadreNode {
|
|
|
814
967
|
* only caller in production is `createControlNode`.
|
|
815
968
|
*/
|
|
816
969
|
buildControlNodeOptions() {
|
|
817
|
-
const { controlNetwork, network,
|
|
970
|
+
const { controlNetwork, network, profile } = this.config;
|
|
818
971
|
const identityKey = this.identityKey;
|
|
819
|
-
|
|
820
|
-
//
|
|
821
|
-
//
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
: storage.provider)
|
|
829
|
-
: undefined;
|
|
972
|
+
// `'search'`: `network.relayAddrs` contributes the bare `/p2p-circuit` SEARCH
|
|
973
|
+
// entry, which opens no connection — the reservation is driven explicitly at
|
|
974
|
+
// the END of `start()`, so the control database is built while this node holds
|
|
975
|
+
// zero control connections. See `relay-addrs.ts` and `start()`.
|
|
976
|
+
const listenAddrs = resolveListenAddrs(network, 'search');
|
|
977
|
+
const enableRelay = this.relayServerEnabled();
|
|
978
|
+
// The control node's own storage, resolved once per runtime (see
|
|
979
|
+
// {@link resolveControlStorage}) rather than per call.
|
|
980
|
+
const controlStorageProvider = this.resolveControlStorage();
|
|
830
981
|
const nodeOptions = {
|
|
831
982
|
port: 0,
|
|
832
983
|
bootstrapNodes: controlNetwork.bootstrapNodes,
|
|
@@ -845,15 +996,25 @@ export class CadreNode {
|
|
|
845
996
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
846
997
|
...(identityKey && { privateKey: identityKey }),
|
|
847
998
|
...(network?.transports && { transports: network.transports }),
|
|
848
|
-
// Configured `listenAddrs`, plus
|
|
849
|
-
// `network.relayAddrs`
|
|
999
|
+
// Configured `listenAddrs`, plus the bare `/p2p-circuit` search listener when
|
|
1000
|
+
// `network.relayAddrs` is set — that listener registers the pending reservation
|
|
1001
|
+
// `driveControlRelayReservation` fills after bring-up.
|
|
850
1002
|
...(listenAddrs && { listenAddrs }),
|
|
1003
|
+
// What this node ADVERTISES, which is not the same question as what it binds.
|
|
1004
|
+
// Either field is present only when configured non-empty; a non-empty
|
|
1005
|
+
// `announceAddrs` replaces the advertised set (see warnIfAnnounceAddrsDiscardRelay).
|
|
1006
|
+
...resolveAnnounceAddrs(network),
|
|
851
1007
|
// The CONTROL node composes the membership admission gate onto any
|
|
852
|
-
// caller-supplied gater (deny from either wins on inbound
|
|
853
|
-
// hooks pass through). Strand
|
|
854
|
-
//
|
|
1008
|
+
// caller-supplied gater (deny from either wins on the inbound-encrypted
|
|
1009
|
+
// and relay-reservation hooks; all other hooks pass through). Strand
|
|
1010
|
+
// cohort nodes keep the raw configured gater (see
|
|
1011
|
+
// strand-instance-manager.ts) — their peers are legitimately
|
|
855
1012
|
// cross-party, so cadre membership must not gate them.
|
|
856
|
-
connectionGater: createMembershipConnectionGater({
|
|
1013
|
+
connectionGater: createMembershipConnectionGater({
|
|
1014
|
+
admitInbound: (remotePeerId) => this.admitInboundControlConnection(remotePeerId),
|
|
1015
|
+
admitRelayReservation: (remotePeerId) => this.admitControlRelayReservation(remotePeerId),
|
|
1016
|
+
bringUpInFlight: () => this.controlBringUpInFlight
|
|
1017
|
+
}, network?.connectionGater),
|
|
857
1018
|
// Fail-closed per-stream authorization for the four Optimystic control-DB
|
|
858
1019
|
// protocols — the members-only layer the connection gater's stranger
|
|
859
1020
|
// carve-outs cannot express (see authorizeInboundControlStream). Control
|
|
@@ -862,6 +1023,17 @@ export class CadreNode {
|
|
|
862
1023
|
};
|
|
863
1024
|
return nodeOptions;
|
|
864
1025
|
}
|
|
1026
|
+
/**
|
|
1027
|
+
* Does this node's CONTROL libp2p run the circuit-relay server? An explicit
|
|
1028
|
+
* `network.enableRelay` wins; otherwise storage-profile nodes default to on
|
|
1029
|
+
* (better connectivity/uptime). Shared by {@link buildControlNodeOptions}
|
|
1030
|
+
* (which configures the server from it) and
|
|
1031
|
+
* {@link admitInboundControlConnection} (whose deny/admit-for-relay branch
|
|
1032
|
+
* must agree with whether a reservation is even servable here).
|
|
1033
|
+
*/
|
|
1034
|
+
relayServerEnabled() {
|
|
1035
|
+
return this.config.network?.enableRelay ?? (this.config.profile === 'storage');
|
|
1036
|
+
}
|
|
865
1037
|
/**
|
|
866
1038
|
* Decide whether an inbound CONTROL-network connection from `remotePeerId`
|
|
867
1039
|
* should be admitted — the policy behind the connection gater wired in
|
|
@@ -871,7 +1043,7 @@ export class CadreNode {
|
|
|
871
1043
|
* `membership-connection-gater.ts` for the layer's rationale and the
|
|
872
1044
|
* stranger-open protocol allowlist).
|
|
873
1045
|
*
|
|
874
|
-
*
|
|
1046
|
+
* Returns `'admit'` when ANY of:
|
|
875
1047
|
* 1. a shared-baseline check admits ({@link admitControlPeerUnconditionally}
|
|
876
1048
|
* — not running / DB torn down, absent-or-empty trusted-owner anchor, or
|
|
877
1049
|
* configured bootstrap/relay infrastructure);
|
|
@@ -908,44 +1080,117 @@ export class CadreNode {
|
|
|
908
1080
|
* this node yet is denied even though the formation handler would have
|
|
909
1081
|
* accepted it, exactly like the unreplicated-membership-row case below.
|
|
910
1082
|
*
|
|
1083
|
+
* When every check falls through, the verdict depends on whether this node
|
|
1084
|
+
* runs the circuit-relay server ({@link relayServerEnabled}): without one,
|
|
1085
|
+
* `'deny'`; with one, `'admit-for-relay'` — a circuit-relay reservation is
|
|
1086
|
+
* established by the reserving peer DIALING the relay, so a connection deny
|
|
1087
|
+
* here kills the reservation, and that deny is NOT self-healing: an outbound
|
|
1088
|
+
* reconcile re-dial re-establishes a data link, but no outbound dial can
|
|
1089
|
+
* grant the remote peer a reservation, and a relay-only peer has no address
|
|
1090
|
+
* of its own to dial back — the reservation IS its address. The gater admits
|
|
1091
|
+
* such a connection, decides the reservation via
|
|
1092
|
+
* {@link admitControlRelayReservation}, and drops the connection if no
|
|
1093
|
+
* reservation is admitted in time (see `membership-connection-gater.ts` →
|
|
1094
|
+
* "The relay-reservation seam").
|
|
1095
|
+
*
|
|
911
1096
|
* NOTE: check 4/5 runs a control-DB read per inbound connection
|
|
912
1097
|
* (`listAuthorizedMembers`); connections are rare and cadres small, and this
|
|
913
1098
|
* layer is fail-open behind `ADMISSION_DECISION_TIMEOUT_MS`, so the live
|
|
914
1099
|
* read is safe here — unlike the per-stream gate, which must consult the
|
|
915
1100
|
* materialized {@link authorizedControlPeers} snapshot instead.
|
|
916
|
-
* NOTE: a sibling whose membership row has not yet
|
|
917
|
-
*
|
|
918
|
-
* either side's next outbound reconcile dial (outbound is never
|
|
919
|
-
* re-establishes the link — self-healing,
|
|
1101
|
+
* NOTE: on a relay-DISABLED node, a sibling whose membership row has not yet
|
|
1102
|
+
* replicated here is denied until the row converges (typically via the
|
|
1103
|
+
* owner); either side's next outbound reconcile dial (outbound is never
|
|
1104
|
+
* gated) re-establishes the DATA link — self-healing for data, visible as a
|
|
1105
|
+
* transient deny. That self-healing story never covered a reservation, which
|
|
1106
|
+
* is exactly why the relay-enabled path above exists.
|
|
920
1107
|
*/
|
|
921
1108
|
async admitInboundControlConnection(remotePeerId) {
|
|
922
1109
|
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
923
|
-
return
|
|
1110
|
+
return 'admit';
|
|
924
1111
|
}
|
|
925
1112
|
if (Date.now() < this.enrollmentWindowUntil) {
|
|
926
|
-
return
|
|
1113
|
+
return 'admit';
|
|
927
1114
|
}
|
|
928
1115
|
if (this.delegateAdmission.has(remotePeerId)) {
|
|
929
|
-
return
|
|
1116
|
+
return 'admit';
|
|
930
1117
|
}
|
|
931
1118
|
const authorized = await this.listAuthorizedMembers();
|
|
932
1119
|
if (authorized.length === 0) {
|
|
933
|
-
return
|
|
1120
|
+
return 'admit';
|
|
934
1121
|
}
|
|
935
1122
|
if (authorized.some((m) => m.peerId === remotePeerId)) {
|
|
936
|
-
return
|
|
1123
|
+
return 'admit';
|
|
937
1124
|
}
|
|
938
1125
|
try {
|
|
939
1126
|
if (await this.strandSolicitationService?.hasOutstandingInvitation()) {
|
|
940
|
-
return
|
|
1127
|
+
return 'admit';
|
|
941
1128
|
}
|
|
942
1129
|
}
|
|
943
1130
|
catch (error) {
|
|
944
1131
|
log('admitInboundControlConnection: outstanding-invitation check threw for %s — admitting (fail-open): %o', remotePeerId, error);
|
|
945
|
-
return
|
|
1132
|
+
return 'admit';
|
|
1133
|
+
}
|
|
1134
|
+
if (this.relayServerEnabled()) {
|
|
1135
|
+
log('admitInboundControlConnection: admitting %s FOR RELAY ONLY — not an authorized member and no enrollment path open; the gater drops the connection unless a reservation is admitted', remotePeerId);
|
|
1136
|
+
return 'admit-for-relay';
|
|
946
1137
|
}
|
|
947
1138
|
log('admitInboundControlConnection: DENYING inbound from %s — not an authorized member and no enrollment path open', remotePeerId);
|
|
948
|
-
return
|
|
1139
|
+
return 'deny';
|
|
1140
|
+
}
|
|
1141
|
+
/**
|
|
1142
|
+
* Should `remotePeerId` be granted a circuit-relay reservation slot on this
|
|
1143
|
+
* node's relay server? The policy behind the gater's
|
|
1144
|
+
* `denyInboundRelayReservation` hook (see `membership-connection-gater.ts` →
|
|
1145
|
+
* "The relay-reservation seam"); the circuit-relay server consults it per
|
|
1146
|
+
* RESERVE request, so it is never called on a node whose relay server is off.
|
|
1147
|
+
*
|
|
1148
|
+
* Admits when ANY of:
|
|
1149
|
+
* 1. a shared-baseline check admits ({@link admitControlPeerUnconditionally}
|
|
1150
|
+
* — not running / DB torn down, absent-or-empty trusted-owner anchor, or
|
|
1151
|
+
* configured bootstrap/relay infrastructure);
|
|
1152
|
+
* 2. the peer holds a live delegate admission grant — a member's strand
|
|
1153
|
+
* node reserving here (see `delegate-admission.ts`); never counted
|
|
1154
|
+
* against the unauthorized budget;
|
|
1155
|
+
* 3. the authorized-member set is empty (cold start) or the peer IS an
|
|
1156
|
+
* authorized member — members are never counted against the budget; or
|
|
1157
|
+
* 4. the peer takes (or already holds) a slot in the bounded
|
|
1158
|
+
* unauthorized-reservation budget — the boot-ordering window where a
|
|
1159
|
+
* genuine member reserves before its `CadrePeer` row replicates here.
|
|
1160
|
+
* Cap via `network.unauthorizedRelayReservationCap` (0 = refuse every
|
|
1161
|
+
* unauthorized reservation).
|
|
1162
|
+
*
|
|
1163
|
+
* Every uncounted admission also RELEASES the budget slot the peer may hold
|
|
1164
|
+
* from an earlier reservation taken while it was still unplaceable — a member
|
|
1165
|
+
* that boots, reserves, and only then has its row replicate here would
|
|
1166
|
+
* otherwise keep that slot spent for the rest of the entry's TTL.
|
|
1167
|
+
*
|
|
1168
|
+
* The connection gate's stranger carve-outs (an open enrollment window, an
|
|
1169
|
+
* outstanding formation invitation) deliberately do NOT extend here: they
|
|
1170
|
+
* exist so a stranger's SEED or FORMATION stream can ride a connection, and
|
|
1171
|
+
* neither needs relay capacity. A genuine invitee that does need a relay slot
|
|
1172
|
+
* takes one from the budget like any other unplaced peer, so an open window
|
|
1173
|
+
* never becomes an unbounded grant of this node's forwarding capacity.
|
|
1174
|
+
*/
|
|
1175
|
+
async admitControlRelayReservation(remotePeerId) {
|
|
1176
|
+
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1177
|
+
return this.admitReservationUncounted(remotePeerId);
|
|
1178
|
+
}
|
|
1179
|
+
if (this.delegateAdmission.has(remotePeerId)) {
|
|
1180
|
+
return this.admitReservationUncounted(remotePeerId);
|
|
1181
|
+
}
|
|
1182
|
+
const authorized = await this.listAuthorizedMembers();
|
|
1183
|
+
if (authorized.length === 0 || authorized.some((m) => m.peerId === remotePeerId)) {
|
|
1184
|
+
return this.admitReservationUncounted(remotePeerId);
|
|
1185
|
+
}
|
|
1186
|
+
const admitted = this.unauthorizedRelayReservations.tryAdmit(remotePeerId);
|
|
1187
|
+
log('admitControlRelayReservation: %s %s under the unauthorized-reservation budget', admitted ? 'admitting' : 'REFUSING', remotePeerId);
|
|
1188
|
+
return admitted;
|
|
1189
|
+
}
|
|
1190
|
+
/** Admit a reservation on the peer's own merits, giving back any budget slot it still holds. */
|
|
1191
|
+
admitReservationUncounted(remotePeerId) {
|
|
1192
|
+
this.unauthorizedRelayReservations.release(remotePeerId);
|
|
1193
|
+
return true;
|
|
949
1194
|
}
|
|
950
1195
|
/**
|
|
951
1196
|
* Hold the control-network inbound gate open for not-yet-authorized peers
|
|
@@ -961,7 +1206,8 @@ export class CadreNode {
|
|
|
961
1206
|
/**
|
|
962
1207
|
* Record (or refresh) a delegate admission grant: `delegatePeerId` is the
|
|
963
1208
|
* transport peerId of `announcerPeerId`'s strand-`strandId` node, admitted
|
|
964
|
-
* at the CONNECTION
|
|
1209
|
+
* at the CONNECTION and RESERVATION levels (all a circuit-relay reservation
|
|
1210
|
+
* needs, without spending the unauthorized-reservation budget) for
|
|
965
1211
|
* `DELEGATE_GRANT_TTL_MS`. Called by the strand-addr responder after its
|
|
966
1212
|
* authorized-membership gate has passed; public so tests can drive the
|
|
967
1213
|
* admission policy without a full strand launch. A re-announce for the same
|
|
@@ -1064,13 +1310,19 @@ export class CadreNode {
|
|
|
1064
1310
|
*
|
|
1065
1311
|
* The SOLE caller is {@link drainMembershipGate}, which serializes refreshes —
|
|
1066
1312
|
* so two reads can no longer settle out of order.
|
|
1313
|
+
*
|
|
1314
|
+
* Reads WITHOUT the transient-failure retry, deliberately: `ControlDatabase` drives
|
|
1315
|
+
* this listener with its write lock HELD (`notifyMembershipChanged`), so a retrying
|
|
1316
|
+
* read here would sleep its backoff holding the lock and stall every other local
|
|
1317
|
+
* writer. Nothing is lost — this refresh already keeps the previous snapshot on
|
|
1318
|
+
* failure and is re-driven by the next membership write and by the timed reconcile.
|
|
1067
1319
|
*/
|
|
1068
1320
|
async refreshAuthorizedControlPeers(reason) {
|
|
1069
1321
|
if (!this._running || !this.controlDatabase) {
|
|
1070
1322
|
return;
|
|
1071
1323
|
}
|
|
1072
1324
|
try {
|
|
1073
|
-
const members = await this.listAuthorizedMembers();
|
|
1325
|
+
const members = await this.listAuthorizedMembers(false);
|
|
1074
1326
|
this.authorizedControlPeers = new Set(members.map((m) => m.peerId));
|
|
1075
1327
|
log('refreshAuthorizedControlPeers(%s): %d authorized peer(s)', reason, this.authorizedControlPeers.size);
|
|
1076
1328
|
}
|
|
@@ -1203,7 +1455,11 @@ export class CadreNode {
|
|
|
1203
1455
|
const existing = await this.controlDatabase.queryPeerRecord(peerId);
|
|
1204
1456
|
if (!existing) {
|
|
1205
1457
|
if (!this.seedBootstrapService) {
|
|
1206
|
-
|
|
1458
|
+
// "No row" also covers "revoked": queryPeerRecord reads a row whose StampId is
|
|
1459
|
+
// retired in Revocation as absent, so a removed node lands here every heartbeat
|
|
1460
|
+
// and stops refreshing its own record — correct, but say so, or the message reads
|
|
1461
|
+
// as "never added" for a node that was removed.
|
|
1462
|
+
log('registerSelf: no readable CadrePeer row for self (never added, or removed and its stamp retired) and no owner service to self-insert; skipping');
|
|
1207
1463
|
return 'skipped';
|
|
1208
1464
|
}
|
|
1209
1465
|
// First-time row: requires an owner signature (the node is its own
|
|
@@ -1225,9 +1481,10 @@ export class CadreNode {
|
|
|
1225
1481
|
// first publish", add a fourth SelfRegistrationOutcome rather than re-labelling.
|
|
1226
1482
|
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
1227
1483
|
}
|
|
1228
|
-
//
|
|
1484
|
+
// Reached on the fall-through above, where the pre-race read missed the row a
|
|
1229
1485
|
// concurrent authorize then seated — so re-read to get that row's `UpdatedAt`, which
|
|
1230
|
-
// the strictly-greater self-update rule is measured against.
|
|
1486
|
+
// the strictly-greater self-update rule is measured against. The re-read can still
|
|
1487
|
+
// come back null (see the branch below).
|
|
1231
1488
|
//
|
|
1232
1489
|
// NOTE: when `existing` is set it predates this publish, so a removePeer(self) racing
|
|
1233
1490
|
// in would leave the UPDATE below matching no rows while still reporting 'refreshed'.
|
|
@@ -1235,7 +1492,13 @@ export class CadreNode {
|
|
|
1235
1492
|
// if removing self ever becomes something that happens concurrently in practice.
|
|
1236
1493
|
const current = existing ?? await this.controlDatabase.queryPeerRecord(peerId);
|
|
1237
1494
|
if (!current) {
|
|
1238
|
-
|
|
1495
|
+
// Two causes, and the second is a steady state rather than a race: the row really
|
|
1496
|
+
// vanished mid-publish (next refresh re-inserts), or this node was REVOKED — its row
|
|
1497
|
+
// is physically present, so insertCadrePeer's in-lock existence check no-op'd above,
|
|
1498
|
+
// but queryPeerRecord reads it as absent because its stamp is retired. The revoked
|
|
1499
|
+
// case repeats every heartbeat and never re-inserts; that is intended (a revoked node
|
|
1500
|
+
// has nothing to publish), so this stays a skip, not an error.
|
|
1501
|
+
log('registerSelf: own CadrePeer row not readable (vanished mid-publish, or this node is revoked and its stamp retired); skipping');
|
|
1239
1502
|
return 'skipped';
|
|
1240
1503
|
}
|
|
1241
1504
|
const record = this.signSelfRecord(peerId, signingKey, addrs, current);
|
|
@@ -1341,6 +1604,20 @@ export class CadreNode {
|
|
|
1341
1604
|
reconcile('self:peer:update');
|
|
1342
1605
|
};
|
|
1343
1606
|
this.controlNode.addEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
1607
|
+
// NOTE: a node AUTHORIZED AFTER IT BOOTED can wait a whole heartbeat (7.5 min)
|
|
1608
|
+
// to publish its address. `registerSelf` no-ops while a node has no readable
|
|
1609
|
+
// `CadrePeer` row, so its boot-time publish does nothing, and the two triggers
|
|
1610
|
+
// wired here are the only ones after that: `self:peer:update` needs an ADDRESS
|
|
1611
|
+
// CHANGE, and a relay-only node's addresses stop changing the moment its
|
|
1612
|
+
// reservation lands — inside `start()`, before this listener exists. Harmless
|
|
1613
|
+
// where a late-authorized node's addresses keep churning (NAT/relay rotation
|
|
1614
|
+
// fire the event anyway), and invisible while parties authorize members before
|
|
1615
|
+
// they boot. If late enrollment becomes the normal path — an invited phone is
|
|
1616
|
+
// exactly that — republish on the membership-change seam that already exists
|
|
1617
|
+
// (`ControlDatabase.setMembershipChangeListener`) rather than lengthening this
|
|
1618
|
+
// list of triggers. Measured while writing
|
|
1619
|
+
// `relay-only-control-addr.integration.ts` case 4, which drives `registerSelf()`
|
|
1620
|
+
// explicitly because neither trigger is dependable there.
|
|
1344
1621
|
this.recordRefreshTimer = setInterval(() => republish('heartbeat'), DEFAULT_PEER_RECORD_HEARTBEAT_MS);
|
|
1345
1622
|
this.recordRefreshTimer?.unref?.();
|
|
1346
1623
|
const reconcileMs = this.config.network?.controlCohort?.reconcileMs ?? DEFAULT_CONTROL_COHORT_RECONCILE_MS;
|
|
@@ -1415,6 +1692,10 @@ export class CadreNode {
|
|
|
1415
1692
|
* Survivors are returned signaling (`/p2p-circuit`) first, filtered to
|
|
1416
1693
|
* signaling-only when requested, as parsed `Multiaddr`s (unparsable addrs
|
|
1417
1694
|
* dropped). Any gate failure yields an empty array rather than throwing.
|
|
1695
|
+
*
|
|
1696
|
+
* Every returned address is normalized to terminate in `/p2p/<peerId>` — see
|
|
1697
|
+
* {@link normalizeDialAddrs} for why that invariant, not each dial site's own
|
|
1698
|
+
* handling, is what keeps a mixed list from taking a whole peer offline.
|
|
1418
1699
|
*/
|
|
1419
1700
|
async resolvePeerAddrs(peerId, opts = {}) {
|
|
1420
1701
|
if (!this.controlDatabase) {
|
|
@@ -1459,12 +1740,71 @@ export class CadreNode {
|
|
|
1459
1740
|
}
|
|
1460
1741
|
// Order signaling-first (the on-record order was what we verified above),
|
|
1461
1742
|
// optionally restrict to signaling addrs, then parse — dropping any addr
|
|
1462
|
-
// that does not parse as a multiaddr
|
|
1743
|
+
// that does not parse as a multiaddr — and finally normalize every survivor
|
|
1744
|
+
// onto `/p2p/<peerId>` so no caller ever sees a mixed list.
|
|
1463
1745
|
let addrs = orderSignalingFirst(record.addrs);
|
|
1464
1746
|
if (opts.signalingOnly) {
|
|
1465
1747
|
addrs = addrs.filter(isSignalingAddr);
|
|
1466
1748
|
}
|
|
1467
|
-
return this.parseMultiaddrs(addrs);
|
|
1749
|
+
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
1750
|
+
}
|
|
1751
|
+
/**
|
|
1752
|
+
* Guarantee every address handed to a control-network dial terminates in
|
|
1753
|
+
* `/p2p/<peerId>`, with no duplicates — the invariant every dial path
|
|
1754
|
+
* downstream relies on.
|
|
1755
|
+
*
|
|
1756
|
+
* `libp2p.dial(addrs)` requires that the addresses in ONE dial either all name
|
|
1757
|
+
* a peer id or none do; a list mixing a circuit address (which ends in
|
|
1758
|
+
* `/p2p/<target>`) with a bare direct one (which ends in `/tcp/…`) makes that
|
|
1759
|
+
* call throw, and the whole peer is then silently skipped rather than one bad
|
|
1760
|
+
* address.
|
|
1761
|
+
*
|
|
1762
|
+
* Applied by ALL THREE sources of control-dial candidates, so no call site
|
|
1763
|
+
* invents its own rule and none can produce a mixed list: the signed record
|
|
1764
|
+
* ({@link resolvePeerAddrs}), the libp2p address book
|
|
1765
|
+
* ({@link peerStoreAddrs}), and a seed's retained owner addresses
|
|
1766
|
+
* ({@link bootstrapDialAddrs}).
|
|
1767
|
+
*
|
|
1768
|
+
* An address naming a DIFFERENT trailing peer id is dropped: it does not reach
|
|
1769
|
+
* `peerId`, so it does not belong in this peer's candidate list (libp2p's own
|
|
1770
|
+
* dial queue filters the same shape out one layer lower).
|
|
1771
|
+
*/
|
|
1772
|
+
normalizeDialAddrs(addrs, peerId) {
|
|
1773
|
+
// Keyed by the NORMALIZED string, so two entries differing only by the suffix
|
|
1774
|
+
// (`/ip4/…/tcp/4001` and `/ip4/…/tcp/4001/p2p/<peerId>`) collapse into one.
|
|
1775
|
+
// Normalization is what makes them equal; a surviving duplicate would cost a
|
|
1776
|
+
// real dial attempt — a whole slice of `dialWake`'s budget — on an address
|
|
1777
|
+
// already being tried.
|
|
1778
|
+
const out = new Map();
|
|
1779
|
+
for (const addr of addrs) {
|
|
1780
|
+
const bound = this.bindAddrToPeer(addr, peerId);
|
|
1781
|
+
if (bound) {
|
|
1782
|
+
out.set(bound.toString(), bound);
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
return [...out.values()];
|
|
1786
|
+
}
|
|
1787
|
+
/**
|
|
1788
|
+
* One address bound to `peerId`, or `null` when it cannot be — logged either
|
|
1789
|
+
* way, never thrown, so every caller's list-shaping stays total.
|
|
1790
|
+
*
|
|
1791
|
+
* `withTrailingPeerId` encapsulates `/p2p/<peerId>`, which throws on a peer id
|
|
1792
|
+
* that does not parse. Unreachable for a `CadrePeer` row (the binding gate
|
|
1793
|
+
* above parsed it already), reachable for a seed-supplied one
|
|
1794
|
+
* ({@link bootstrapDialAddrs}).
|
|
1795
|
+
*/
|
|
1796
|
+
bindAddrToPeer(addr, peerId) {
|
|
1797
|
+
try {
|
|
1798
|
+
const bound = withTrailingPeerId(addr, peerId);
|
|
1799
|
+
if (!bound) {
|
|
1800
|
+
log('normalizeDialAddrs: dropping addr %s — it names a peer other than %s', addr.toString(), peerId);
|
|
1801
|
+
}
|
|
1802
|
+
return bound;
|
|
1803
|
+
}
|
|
1804
|
+
catch (error) {
|
|
1805
|
+
log('normalizeDialAddrs: cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
1806
|
+
return null;
|
|
1807
|
+
}
|
|
1468
1808
|
}
|
|
1469
1809
|
/** Parse multiaddr strings, dropping (and logging) any that fail to parse. */
|
|
1470
1810
|
parseMultiaddrs(addrs) {
|
|
@@ -1530,8 +1870,11 @@ export class CadreNode {
|
|
|
1530
1870
|
// than a local write) are picked up here, bounding the snapshot's staleness
|
|
1531
1871
|
// to the reconcile interval.
|
|
1532
1872
|
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
1533
|
-
// CadrePeer query (two reads per pass)
|
|
1534
|
-
//
|
|
1873
|
+
// CadrePeer query (two reads per pass), plus a third from
|
|
1874
|
+
// `refreshStrandPeerAddrs` on the passes where a strand is due AND this node
|
|
1875
|
+
// holds a control connection (one read for the whole pass, not one per
|
|
1876
|
+
// strand); if those reads ever get costly, share one row-set across all
|
|
1877
|
+
// three.
|
|
1535
1878
|
await this.refreshMembershipGate('reconcile');
|
|
1536
1879
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1537
1880
|
return;
|
|
@@ -1544,6 +1887,14 @@ export class CadreNode {
|
|
|
1544
1887
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1545
1888
|
return;
|
|
1546
1889
|
}
|
|
1890
|
+
// Then re-warm each running strand's own address book from its siblings, on
|
|
1891
|
+
// its own (much longer) throttle. Same reasoning as warmSiblingAddrBook
|
|
1892
|
+
// below, one layer down: replication runs on the STRAND network, and every
|
|
1893
|
+
// layer under cadre-core dials strand peers by bare peer id.
|
|
1894
|
+
await this.refreshStrandPeerAddrs();
|
|
1895
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1896
|
+
return;
|
|
1897
|
+
}
|
|
1547
1898
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1548
1899
|
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
1549
1900
|
// Revocation tombstone retires — the half of a removal replication cannot carry
|
|
@@ -1622,9 +1973,16 @@ export class CadreNode {
|
|
|
1622
1973
|
// Don't silently bound coverage — surface what the out-degree cap dropped.
|
|
1623
1974
|
log('reconcileControlCohort: capped %d non-owner sibling(s) at targetDegree=%d', cappedNonOwner, targetDegree);
|
|
1624
1975
|
}
|
|
1625
|
-
// 3.
|
|
1976
|
+
// 3. Warm the libp2p address book with EVERY sibling's verified addresses,
|
|
1977
|
+
// keeping what resolved for the dial loop below.
|
|
1978
|
+
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
1979
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1980
|
+
return;
|
|
1981
|
+
}
|
|
1982
|
+
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
1626
1983
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
1627
|
-
//
|
|
1984
|
+
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
1985
|
+
// addresses step 3 already resolved for it.
|
|
1628
1986
|
let dialed = 0;
|
|
1629
1987
|
for (const sibling of dials) {
|
|
1630
1988
|
if (!this._running || !this.controlNode) {
|
|
@@ -1633,34 +1991,109 @@ export class CadreNode {
|
|
|
1633
1991
|
if (connected.has(sibling.peerId)) {
|
|
1634
1992
|
continue;
|
|
1635
1993
|
}
|
|
1636
|
-
if (await this.dialControlSibling(sibling)) {
|
|
1994
|
+
if (await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? [])) {
|
|
1637
1995
|
dialed++;
|
|
1638
1996
|
}
|
|
1639
1997
|
}
|
|
1640
1998
|
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)', siblings.length, dials.length, dialed);
|
|
1641
1999
|
}
|
|
1642
2000
|
/**
|
|
1643
|
-
* Resolve
|
|
1644
|
-
*
|
|
2001
|
+
* Resolve every sibling's signed address record once and merge what resolves
|
|
2002
|
+
* into the control node's libp2p **address book** (peerStore). Returns the
|
|
2003
|
+
* resolved addresses per sibling so the dial loop reuses them rather than
|
|
2004
|
+
* re-resolving — one `queryPeerRecord` per sibling per pass, not two.
|
|
2005
|
+
*
|
|
2006
|
+
* Deliberately every sibling, not the {@link selectControlCohortDials} subset,
|
|
2007
|
+
* and deliberately including already-connected ones: everything below
|
|
2008
|
+
* cadre-core dials by bare peer id (Optimystic's cluster/repo clients, FRET
|
|
2009
|
+
* ping/announce), so the address book has to be warm BEFORE a live connection
|
|
2010
|
+
* drops, and a sibling the out-degree cap declines to dial is still one those
|
|
2011
|
+
* layers may need to reach.
|
|
2012
|
+
*
|
|
2013
|
+
* Only `resolvePeerAddrs` output is merged — never the cold-start
|
|
2014
|
+
* `peerStoreAddrs` fallback, which came out of the address book to begin with
|
|
2015
|
+
* and would restamp unverified seed addresses indefinitely. A sibling that
|
|
2016
|
+
* resolves to nothing (revoked, stale, untrusted) is not written at all, so its
|
|
2017
|
+
* existing entry ages out on its own.
|
|
2018
|
+
*
|
|
2019
|
+
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
2020
|
+
* so it costs one record query per sibling per reconcile pass (~15s) and each
|
|
2021
|
+
* one delays the pass's first dial. A cadre is a handful of devices, so today
|
|
2022
|
+
* that is a few extra local reads; if cadres ever grow large, batch the records
|
|
2023
|
+
* into one query, merge only on change, or move the warm pass after the dials.
|
|
2024
|
+
*/
|
|
2025
|
+
async warmSiblingAddrBook(siblings) {
|
|
2026
|
+
const resolved = new Map();
|
|
2027
|
+
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
2028
|
+
for (const sibling of siblings) {
|
|
2029
|
+
if (!this._running || !this.controlNode) {
|
|
2030
|
+
break;
|
|
2031
|
+
}
|
|
2032
|
+
const addrs = await this.resolveSiblingAddrs(sibling.peerId);
|
|
2033
|
+
resolved.set(sibling.peerId, addrs);
|
|
2034
|
+
// Re-guarded after the resolve await as well as before it: the write is
|
|
2035
|
+
// the half that matters, and a torn-down node must never be written to.
|
|
2036
|
+
const controlNode = this.controlNode;
|
|
2037
|
+
if (!this._running || !controlNode) {
|
|
2038
|
+
break;
|
|
2039
|
+
}
|
|
2040
|
+
// `mergePeerAddrs` owns the whole best-effort contract — including parsing
|
|
2041
|
+
// a malformed `CadrePeer.PeerId` — so one bad row cannot abort the pass.
|
|
2042
|
+
counts[await mergePeerAddrs(controlNode, sibling.peerId, addrs)]++;
|
|
2043
|
+
}
|
|
2044
|
+
log('reconcileControlCohort: address book warmed (siblings=%d, merged=%d, restamped=%d, skipped=%d, failed=%d)', resolved.size, counts.merged, counts.restamped, counts.skipped, counts.failed);
|
|
2045
|
+
return resolved;
|
|
2046
|
+
}
|
|
2047
|
+
/**
|
|
2048
|
+
* {@link resolvePeerAddrs} for one sibling, best-effort: a control-DB read
|
|
2049
|
+
* failure yields `[]` (and the cold-start fallback then gets its turn) rather
|
|
2050
|
+
* than aborting the whole pass, like every other step here.
|
|
2051
|
+
*/
|
|
2052
|
+
async resolveSiblingAddrs(peerId) {
|
|
2053
|
+
try {
|
|
2054
|
+
return await this.resolvePeerAddrs(peerId);
|
|
2055
|
+
}
|
|
2056
|
+
catch (error) {
|
|
2057
|
+
log('reconcileControlCohort: resolving addrs for %s failed (continuing): %o', peerId, error);
|
|
2058
|
+
return [];
|
|
2059
|
+
}
|
|
2060
|
+
}
|
|
2061
|
+
/**
|
|
2062
|
+
* Budget for ONE proactive dial of the reconcile pass — steady-state sibling
|
|
2063
|
+
* ({@link dialControlSibling}) and cold-start bootstrap peer
|
|
2064
|
+
* ({@link dialBootstrapPeer}) alike, since both feed a whole address list to
|
|
2065
|
+
* one `dial()` and both run in a sequential loop that the next entry waits on.
|
|
2066
|
+
*
|
|
2067
|
+
* See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why the pass bounds
|
|
2068
|
+
* this itself rather than inheriting each address's libp2p attempt timeout.
|
|
2069
|
+
*/
|
|
2070
|
+
controlDialTimeoutMs() {
|
|
2071
|
+
return this.config.network?.controlCohort?.dialTimeoutMs
|
|
2072
|
+
?? DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS;
|
|
2073
|
+
}
|
|
2074
|
+
/**
|
|
2075
|
+
* Dial one sibling from its already-resolved addresses, best-effort. Returns
|
|
2076
|
+
* whether a dial was attempted (false when no address resolves).
|
|
1645
2077
|
*
|
|
1646
|
-
* A per-peer failure (NAT, offline, relay down, connection-gater denial
|
|
1647
|
-
*
|
|
1648
|
-
*
|
|
1649
|
-
*
|
|
2078
|
+
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
2079
|
+
* the {@link controlDialTimeoutMs} budget expiring) is logged and swallowed so
|
|
2080
|
+
* one unreachable sibling never aborts the pass — exactly like
|
|
2081
|
+
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
2082
|
+
* simply retried on the next pass.
|
|
1650
2083
|
*/
|
|
1651
|
-
async dialControlSibling(sibling) {
|
|
2084
|
+
async dialControlSibling(sibling, resolved) {
|
|
1652
2085
|
const controlNode = this.controlNode;
|
|
1653
2086
|
if (!controlNode) {
|
|
1654
2087
|
return false;
|
|
1655
2088
|
}
|
|
1656
|
-
const addrs = await this.resolveControlDialAddrs(sibling.peerId);
|
|
2089
|
+
const addrs = await this.resolveControlDialAddrs(sibling.peerId, resolved);
|
|
1657
2090
|
if (addrs.length === 0) {
|
|
1658
2091
|
log('reconcileControlCohort: no dialable control address for sibling %s; skipping', sibling.peerId);
|
|
1659
2092
|
return false;
|
|
1660
2093
|
}
|
|
1661
2094
|
try {
|
|
1662
2095
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
1663
|
-
await controlNode.dial(addrs);
|
|
2096
|
+
await withDeadline(this.controlDialTimeoutMs(), `reconcileControlCohort dial of sibling ${sibling.peerId}`, (signal) => controlNode.dial(addrs, { signal }));
|
|
1664
2097
|
return true;
|
|
1665
2098
|
}
|
|
1666
2099
|
catch (error) {
|
|
@@ -1669,16 +2102,16 @@ export class CadreNode {
|
|
|
1669
2102
|
}
|
|
1670
2103
|
}
|
|
1671
2104
|
/**
|
|
1672
|
-
*
|
|
2105
|
+
* A sibling's control-network dial addresses for the reconcile pass.
|
|
1673
2106
|
*
|
|
1674
|
-
* Primary (steady state): the signed, fresh, trust-gated control addresses
|
|
1675
|
-
*
|
|
1676
|
-
*
|
|
1677
|
-
*
|
|
1678
|
-
*
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
2107
|
+
* Primary (steady state): the signed, fresh, trust-gated control addresses
|
|
2108
|
+
* `resolved` for it by {@link warmSiblingAddrBook} — passed in rather than
|
|
2109
|
+
* re-resolved, so the pass makes one record query per sibling. Fallback (cold
|
|
2110
|
+
* start): the libp2p peerStore entries `applySeed` populated, used only while
|
|
2111
|
+
* the record is not yet resolvable. Returns `[]` (never throws) when neither
|
|
2112
|
+
* yields an address — that sibling is skipped this pass.
|
|
2113
|
+
*/
|
|
2114
|
+
async resolveControlDialAddrs(peerId, resolved) {
|
|
1682
2115
|
if (resolved.length > 0) {
|
|
1683
2116
|
return resolved;
|
|
1684
2117
|
}
|
|
@@ -1688,6 +2121,16 @@ export class CadreNode {
|
|
|
1688
2121
|
* Cold-start fallback: the libp2p peerStore multiaddrs for `peerId` (seeded by
|
|
1689
2122
|
* {@link SeedBootstrapService.applySeed}). Returns `[]` on a missing entry or any
|
|
1690
2123
|
* parse/lookup failure — never throws.
|
|
2124
|
+
*
|
|
2125
|
+
* Normalized through {@link normalizeDialAddrs} for the same reason
|
|
2126
|
+
* {@link resolvePeerAddrs} is, and with more cause: the address book hands back
|
|
2127
|
+
* a list that is inherently MIXED. `@libp2p/peer-store`'s
|
|
2128
|
+
* `dedupeFilterAndSortAddresses` strips a trailing `/p2p/<peerId>` only when
|
|
2129
|
+
* that id is the address's FIRST `/p2p/` component, so a direct address
|
|
2130
|
+
* round-trips bare while a relayed one — whose first `/p2p/` names the relay —
|
|
2131
|
+
* keeps its suffix (the same asymmetry `peer-addr-book.ts`'s `addrKey` exists
|
|
2132
|
+
* to absorb). Feeding both to one `dial()` is exactly the
|
|
2133
|
+
* `InvalidParametersError` that skipped the whole peer.
|
|
1691
2134
|
*/
|
|
1692
2135
|
async peerStoreAddrs(peerId) {
|
|
1693
2136
|
if (!this.controlNode) {
|
|
@@ -1697,7 +2140,7 @@ export class CadreNode {
|
|
|
1697
2140
|
const peer = await this.controlNode.peerStore.get(peerIdFromString(peerId));
|
|
1698
2141
|
// Re-parse through the top-level multiaddr parser so the returned type matches
|
|
1699
2142
|
// resolvePeerAddrs (the peerStore bundles its own @multiformats/multiaddr copy).
|
|
1700
|
-
return this.parseMultiaddrs(peer.addresses.map((a) => a.multiaddr.toString()));
|
|
2143
|
+
return this.normalizeDialAddrs(this.parseMultiaddrs(peer.addresses.map((a) => a.multiaddr.toString())), peerId);
|
|
1701
2144
|
}
|
|
1702
2145
|
catch (error) {
|
|
1703
2146
|
log('reconcileControlCohort: peerStore lookup for %s failed: %o', peerId, error);
|
|
@@ -1786,6 +2229,13 @@ export class CadreNode {
|
|
|
1786
2229
|
if (!this._running || !this.controlNode) {
|
|
1787
2230
|
return;
|
|
1788
2231
|
}
|
|
2232
|
+
// NOTE: a connection this node still holds in `status: 'open'` after the remote
|
|
2233
|
+
// has already aborted it counts as connected here, so the cold-start retry stays
|
|
2234
|
+
// suppressed for that peer until the connection monitor's next ping notices the
|
|
2235
|
+
// abort — measured at ~9 s in `control-cohort-cold-start-retry.integration.ts`.
|
|
2236
|
+
// Bounded and self-healing today; it would start to matter if the connection
|
|
2237
|
+
// monitor's ping interval were lengthened, or if recovery ever had to meet a
|
|
2238
|
+
// deadline tighter than one ping interval.
|
|
1789
2239
|
if (connected.has(peerId)) {
|
|
1790
2240
|
continue;
|
|
1791
2241
|
}
|
|
@@ -1799,32 +2249,15 @@ export class CadreNode {
|
|
|
1799
2249
|
* Bind a bootstrap peer's seed addresses to its peer id, so the dial
|
|
1800
2250
|
* authenticates the peer it is aiming at rather than trusting whoever answers.
|
|
1801
2251
|
*
|
|
1802
|
-
*
|
|
1803
|
-
*
|
|
1804
|
-
*
|
|
1805
|
-
*
|
|
1806
|
-
* bare
|
|
2252
|
+
* The same {@link normalizeDialAddrs} rule the resolved and address-book paths
|
|
2253
|
+
* use, on the third and last source of control-dial candidates: an address that
|
|
2254
|
+
* already terminates in `/p2p/<id>` must carry THIS peer's id or it is dropped
|
|
2255
|
+
* (a seed that disagrees with itself is not a dial target); one that does not —
|
|
2256
|
+
* a bare listen addr, or a relay hop with the destination missing — gets the id
|
|
2257
|
+
* encapsulated; an unencapsulatable one is dropped rather than dialed bare.
|
|
1807
2258
|
*/
|
|
1808
2259
|
bootstrapDialAddrs(peerId, addrs) {
|
|
1809
|
-
|
|
1810
|
-
for (const addr of this.parseMultiaddrs(addrs)) {
|
|
1811
|
-
const embedded = trailingPeerId(addr);
|
|
1812
|
-
if (embedded === peerId) {
|
|
1813
|
-
out.push(addr);
|
|
1814
|
-
continue;
|
|
1815
|
-
}
|
|
1816
|
-
if (embedded !== null) {
|
|
1817
|
-
log('reconcileControlCohort(cold-start): addr %s names peer %s, not %s; dropping', addr.toString(), embedded, peerId);
|
|
1818
|
-
continue;
|
|
1819
|
-
}
|
|
1820
|
-
try {
|
|
1821
|
-
out.push(addr.encapsulate(`/p2p/${peerId}`));
|
|
1822
|
-
}
|
|
1823
|
-
catch (error) {
|
|
1824
|
-
log('reconcileControlCohort(cold-start): cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
1825
|
-
}
|
|
1826
|
-
}
|
|
1827
|
-
return out;
|
|
2260
|
+
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
1828
2261
|
}
|
|
1829
2262
|
/** Dial one cold-start bootstrap peer, best-effort. Returns whether it connected. */
|
|
1830
2263
|
async dialBootstrapPeer(peerId, addrs) {
|
|
@@ -1839,7 +2272,7 @@ export class CadreNode {
|
|
|
1839
2272
|
}
|
|
1840
2273
|
try {
|
|
1841
2274
|
log('reconcileControlCohort(cold-start): dialing bootstrap peer %s (%d addr(s))', peerId, parsed.length);
|
|
1842
|
-
await controlNode.dial(parsed);
|
|
2275
|
+
await withDeadline(this.controlDialTimeoutMs(), `reconcileControlCohort cold-start dial of bootstrap peer ${peerId}`, (signal) => controlNode.dial(parsed, { signal }));
|
|
1843
2276
|
return true;
|
|
1844
2277
|
}
|
|
1845
2278
|
catch (error) {
|
|
@@ -2510,7 +2943,6 @@ export class CadreNode {
|
|
|
2510
2943
|
return;
|
|
2511
2944
|
}
|
|
2512
2945
|
try {
|
|
2513
|
-
// Discovery path: no explicit mode — infer from cohort membership.
|
|
2514
2946
|
await this.launchStrand(strand, sAppConfig);
|
|
2515
2947
|
}
|
|
2516
2948
|
catch (error) {
|
|
@@ -2543,7 +2975,30 @@ export class CadreNode {
|
|
|
2543
2975
|
});
|
|
2544
2976
|
}
|
|
2545
2977
|
}
|
|
2978
|
+
/**
|
|
2979
|
+
* Return this node to its pre-{@link start} state. The ONE teardown — `stop()` and
|
|
2980
|
+
* a failed `start()` both come here, so anything a stopped node must not still be
|
|
2981
|
+
* doing belongs in this method and not in `stop()`. `stop()` adds only the
|
|
2982
|
+
* `control:disconnected` emit, which a never-connected start has no edge for.
|
|
2983
|
+
*/
|
|
2546
2984
|
async cleanup() {
|
|
2985
|
+
// Open the connection gate's bring-up quiet period unconditionally: teardown
|
|
2986
|
+
// must never be gated, and this is the path a FAILED start() takes.
|
|
2987
|
+
this.controlBringUpInFlight = false;
|
|
2988
|
+
// Stop the relay retry loop BEFORE tearing the node down, not after: the control
|
|
2989
|
+
// node is stopped partway through below, and a tick that fires during it would
|
|
2990
|
+
// dial and then poll a half-torn-down node for the whole reserve timeout —
|
|
2991
|
+
// logging spurious dial failures on a `.unref()`'d timer that nothing ever
|
|
2992
|
+
// clears. `driveControlRelayReservation` starts this supervisor from inside
|
|
2993
|
+
// `start()`, so a start that fails ON the reservation leaves one running unless
|
|
2994
|
+
// it is stopped here.
|
|
2995
|
+
this.relayReserveSupervisor?.stop();
|
|
2996
|
+
this.relayReserveSupervisor = null;
|
|
2997
|
+
// Drop the rest of the relay-reservation posture: a torn-down node holds no
|
|
2998
|
+
// reservation, so neither a restarted instance nor a caller inspecting a failed
|
|
2999
|
+
// start reports the previous attempt's `reserved`.
|
|
3000
|
+
this.relayReserveAddrs = [];
|
|
3001
|
+
this.relayReserveError = null;
|
|
2547
3002
|
// Resolve + clear any in-flight wake windows first, so an in-flight check-in
|
|
2548
3003
|
// or serviceWake unblocks and tears down cleanly rather than firing a stale
|
|
2549
3004
|
// window timer (or hanging) after the strand manager is stopped below.
|
|
@@ -2611,6 +3066,23 @@ export class CadreNode {
|
|
|
2611
3066
|
await this.controlNode.stop();
|
|
2612
3067
|
this.controlNode = null;
|
|
2613
3068
|
}
|
|
3069
|
+
// Release the control store's claim on its cache LAST — after the database and
|
|
3070
|
+
// node that write through it are down. The wrapper counts holders, so this empties
|
|
3071
|
+
// and unregisters the cache only if no other scope still holds it. Dropping the
|
|
3072
|
+
// field is what lets a stop()/start() cycle re-resolve the provider against a live
|
|
3073
|
+
// cache instead of handing the restarted node a retired wrapper. Logged, never
|
|
3074
|
+
// thrown: a cache-bookkeeping failure must not abort a teardown.
|
|
3075
|
+
if (this.controlStorage) {
|
|
3076
|
+
const controlStorage = this.controlStorage;
|
|
3077
|
+
this.controlStorage = null;
|
|
3078
|
+
await disposeStorageCache(controlStorage).catch((err) => log('Failed to dispose control storage cache: %o', err));
|
|
3079
|
+
}
|
|
3080
|
+
// LAST, so every step above still ran against a node that considered itself up
|
|
3081
|
+
// (several of them early-return while `_running` is false). A failed `start()`
|
|
3082
|
+
// that had already flipped it — the reservation drive throws after that point —
|
|
3083
|
+
// must not leave `isRunning` true over a torn-down node, which would report
|
|
3084
|
+
// healthy and make the embedder's retry a no-op ("already running").
|
|
3085
|
+
this._running = false;
|
|
2614
3086
|
}
|
|
2615
3087
|
// Hibernation callbacks
|
|
2616
3088
|
async handleStrandIdle(strandId) {
|
|
@@ -2649,7 +3121,7 @@ export class CadreNode {
|
|
|
2649
3121
|
}
|
|
2650
3122
|
/**
|
|
2651
3123
|
* Wake a strand. If it was hibernating (quiesced — no libp2p node), re-resolve
|
|
2652
|
-
* the cohort discovery seed
|
|
3124
|
+
* the cohort discovery seed exactly as `launchStrand` does and rebuild
|
|
2653
3125
|
* its runtime via the strand manager. If it is still live (e.g. waking an idle
|
|
2654
3126
|
* strand, which retains its resources), just flip the status. Overlapping wake
|
|
2655
3127
|
* triggers are coalesced upstream by `HibernationManager`, so this runs once
|
|
@@ -2669,8 +3141,8 @@ export class CadreNode {
|
|
|
2669
3141
|
this.emit('strand:waking', { strandId });
|
|
2670
3142
|
return;
|
|
2671
3143
|
}
|
|
2672
|
-
// Quiesced: re-resolve volatile cohort
|
|
2673
|
-
//
|
|
3144
|
+
// Quiesced: re-resolve the volatile cohort input (the seed may have grown)
|
|
3145
|
+
// and rebuild the runtime.
|
|
2674
3146
|
log('Waking strand %s — rebuilding strand-network resources', strandId);
|
|
2675
3147
|
await this.resumeStrandRuntime(strandId);
|
|
2676
3148
|
instance.lastActivity = new Date();
|
|
@@ -2678,11 +3150,10 @@ export class CadreNode {
|
|
|
2678
3150
|
log('Strand %s awake (resources rebuilt)', strandId);
|
|
2679
3151
|
}
|
|
2680
3152
|
/**
|
|
2681
|
-
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort
|
|
2682
|
-
* first: the discovery seed may have grown
|
|
2683
|
-
* shifted the mode `bootstrap → networked` since the strand last ran. Shared by
|
|
3153
|
+
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort input
|
|
3154
|
+
* first: the discovery seed may have grown since the strand last ran. Shared by
|
|
2684
3155
|
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so
|
|
2685
|
-
* both apply the same fresh seed
|
|
3156
|
+
* both apply the same fresh seed resolution. `resumeStrand` is idempotent
|
|
2686
3157
|
* (returns the live instance unchanged) as a backstop against double-resume.
|
|
2687
3158
|
*/
|
|
2688
3159
|
async resumeStrandRuntime(strandId) {
|
|
@@ -2692,12 +3163,13 @@ export class CadreNode {
|
|
|
2692
3163
|
const delegatePeerId = this.identityKey
|
|
2693
3164
|
? peerIdFromPrivateKey(await strandTransportKey(this.identityKey, strandId)).toString()
|
|
2694
3165
|
: undefined;
|
|
2695
|
-
const
|
|
2696
|
-
const
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
3166
|
+
const bootstrapNodes = await this.resolveCohortSeed(strandId, delegatePeerId);
|
|
3167
|
+
const instance = await this.strandManager.resumeStrand(strandId, { bootstrapNodes });
|
|
3168
|
+
// Same reason as the launch path: `bootstrapNodes` only reaches the address
|
|
3169
|
+
// book through @libp2p/bootstrap discovery, so merge it directly as well.
|
|
3170
|
+
if (instance.libp2pNode) {
|
|
3171
|
+
await this.mergeStrandPeerAddrs(instance.libp2pNode, bootstrapNodes, strandId);
|
|
3172
|
+
}
|
|
2701
3173
|
}
|
|
2702
3174
|
/**
|
|
2703
3175
|
* Real cohort check-in for a hibernating strand (the `onCheckIn` callback).
|
|
@@ -2708,7 +3180,7 @@ export class CadreNode {
|
|
|
2708
3180
|
* realized as a resume → bounded window → re-hibernate-if-idle cycle that
|
|
2709
3181
|
* reuses the existing quiesce/resume primitives rather than a bespoke probe:
|
|
2710
3182
|
*
|
|
2711
|
-
* 1. Resume the strand (rebuild node + db, re-resolve cohort seed
|
|
3183
|
+
* 1. Resume the strand (rebuild node + db, re-resolve the cohort seed) so
|
|
2712
3184
|
* its strand network can reach cohort peers — exactly as a wake does.
|
|
2713
3185
|
* 2. Hold it resumed for a bounded window, during which the app may drive
|
|
2714
3186
|
* reads (pull-on-read) and record activity.
|
|
@@ -2732,7 +3204,7 @@ export class CadreNode {
|
|
|
2732
3204
|
}
|
|
2733
3205
|
try {
|
|
2734
3206
|
// 1. Resume exactly as a wake does: re-resolve the (possibly grown) cohort
|
|
2735
|
-
// seed
|
|
3207
|
+
// seed, then rebuild the runtime.
|
|
2736
3208
|
log('Check-in: resuming strand %s to probe the cohort for pending activity', strandId);
|
|
2737
3209
|
await this.resumeStrandRuntime(strandId);
|
|
2738
3210
|
// 2-3. Bounded window for the strand network to connect + the app to act,
|
|
@@ -2837,15 +3309,13 @@ export class CadreNode {
|
|
|
2837
3309
|
if (!this._running) {
|
|
2838
3310
|
throw new Error('CadreNode not running');
|
|
2839
3311
|
}
|
|
2840
|
-
const { strandRow, sAppConfig,
|
|
3312
|
+
const { strandRow, sAppConfig, founder } = config;
|
|
2841
3313
|
// Store sApp config for this strand
|
|
2842
3314
|
this.sAppConfigs.set(strandRow.Id, sAppConfig);
|
|
2843
|
-
log('Registered sAppConfig for strand %s (sApp: %s,
|
|
2844
|
-
// Pass `mode` (possibly undefined) through: an explicit mode wins, while a
|
|
2845
|
-
// caller that omits it gets the same cohort-inferred mode as the discovery path.
|
|
3315
|
+
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)', strandRow.Id, sAppConfig.id, founder ?? false);
|
|
2846
3316
|
// `founder` only flows from the explicit addStrand path — the control-discovered
|
|
2847
3317
|
// join path never founds, so its rows arrive via sync (see handleStrandAdded).
|
|
2848
|
-
return await this.launchStrand(strandRow, sAppConfig,
|
|
3318
|
+
return await this.launchStrand(strandRow, sAppConfig, founder);
|
|
2849
3319
|
}
|
|
2850
3320
|
/**
|
|
2851
3321
|
* Publish a strand row to the shared control database under this node's own
|
|
@@ -3081,8 +3551,8 @@ export class CadreNode {
|
|
|
3081
3551
|
/**
|
|
3082
3552
|
* Shared strand launch path for both the explicit (`addStrand`) and the
|
|
3083
3553
|
* control-discovered (`handleStrandAdded`) entry points. Resolves the cohort
|
|
3084
|
-
* seed,
|
|
3085
|
-
*
|
|
3554
|
+
* seed, starts the strand, and registers it with the hibernation manager
|
|
3555
|
+
* before emitting `strand:started`.
|
|
3086
3556
|
*
|
|
3087
3557
|
* Idempotent no-op when the strand manager already tracks `strand.Id` — the
|
|
3088
3558
|
* ordinary founding sequence is `addStrand` (starts it locally) followed by
|
|
@@ -3093,7 +3563,7 @@ export class CadreNode {
|
|
|
3093
3563
|
* instance that never stopped. The guard runs before the cohort-seed RPC
|
|
3094
3564
|
* fan-out, so a rediscovery costs nothing beyond the map lookup.
|
|
3095
3565
|
*/
|
|
3096
|
-
async launchStrand(strand, sAppConfig,
|
|
3566
|
+
async launchStrand(strand, sAppConfig, founder) {
|
|
3097
3567
|
const existing = this.strandManager.getInstance(strand.Id);
|
|
3098
3568
|
if (existing) {
|
|
3099
3569
|
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
@@ -3122,8 +3592,7 @@ export class CadreNode {
|
|
|
3122
3592
|
// `libp2p.start()` (the responder records the grant before replying; the
|
|
3123
3593
|
// client awaits the replies).
|
|
3124
3594
|
const delegatePeerId = transportKey ? peerIdFromPrivateKey(transportKey).toString() : undefined;
|
|
3125
|
-
const
|
|
3126
|
-
const mode = selectStrandMode(explicitMode, seed.hasOtherPeers);
|
|
3595
|
+
const bootstrapNodes = await this.resolveCohortSeed(strand.Id, delegatePeerId);
|
|
3127
3596
|
const instance = await this.strandManager.startStrand({
|
|
3128
3597
|
strandRow: strand,
|
|
3129
3598
|
sAppConfig,
|
|
@@ -3132,30 +3601,37 @@ export class CadreNode {
|
|
|
3132
3601
|
profile: this.config.profile,
|
|
3133
3602
|
defaultLatencyHint: this.config.hibernation?.defaultLatencyHint ?? 'interactive',
|
|
3134
3603
|
privateKey: transportKey,
|
|
3135
|
-
bootstrapNodes
|
|
3136
|
-
mode,
|
|
3604
|
+
bootstrapNodes,
|
|
3137
3605
|
requireSignedSchemas: this.config.requireSignedSchemas,
|
|
3138
3606
|
clusterSize: this.config.strandClusterSize,
|
|
3139
3607
|
backfill: this.config.strandBackfill,
|
|
3140
3608
|
founder
|
|
3141
3609
|
});
|
|
3610
|
+
// The seed reached the node as `bootstrapNodes`, which only enters the
|
|
3611
|
+
// address book via @libp2p/bootstrap discovery. Merge it directly too, so a
|
|
3612
|
+
// sibling is dialable by bare peer id from the first moment (see
|
|
3613
|
+
// mergeStrandPeerAddrs; refreshStrandPeerAddrs keeps it warm from here on).
|
|
3614
|
+
if (instance.libp2pNode) {
|
|
3615
|
+
await this.mergeStrandPeerAddrs(instance.libp2pNode, bootstrapNodes, strand.Id);
|
|
3616
|
+
}
|
|
3142
3617
|
this.hibernationManager.trackStrand(instance);
|
|
3143
3618
|
this.emit('strand:started', { strandId: strand.Id });
|
|
3144
3619
|
return instance;
|
|
3145
3620
|
}
|
|
3146
3621
|
/**
|
|
3147
|
-
* Resolve a strand's discovery seed
|
|
3148
|
-
*
|
|
3149
|
-
*
|
|
3150
|
-
*
|
|
3151
|
-
* addresses that must not
|
|
3622
|
+
* Resolve a strand's discovery seed — the dialable strand-network multiaddr
|
|
3623
|
+
* strings for cohort siblings. Membership comes from the control network's
|
|
3624
|
+
* CadrePeer rows; the **strand-network** bootstrap addresses are resolved on
|
|
3625
|
+
* demand over the control mesh via the strand-addr RPC — deliberately NOT
|
|
3626
|
+
* from `CadrePeer.Multiaddr`, which carries *control* addresses that must not
|
|
3627
|
+
* seed the strand mesh.
|
|
3152
3628
|
*
|
|
3153
3629
|
* Only siblings we already hold an open control connection to are RPC'd: they
|
|
3154
3630
|
* are the ones that can answer right now, and dialing them by peerId reuses the
|
|
3155
3631
|
* live connection. When no connected sibling yet runs the strand the seed is
|
|
3156
|
-
* empty (`[]`) —
|
|
3157
|
-
*
|
|
3158
|
-
*
|
|
3632
|
+
* empty (`[]`) — the empty seed self-heals on the next resume / check-in pass.
|
|
3633
|
+
* Returns an empty seed when the control DB or node is absent (not yet
|
|
3634
|
+
* started / torn down).
|
|
3159
3635
|
*
|
|
3160
3636
|
* When `delegatePeerId` is given (a strand launch/resume is imminent), the
|
|
3161
3637
|
* pass doubles as the delegate ANNOUNCEMENT: our circuit relays are merged
|
|
@@ -3170,16 +3646,9 @@ export class CadreNode {
|
|
|
3170
3646
|
*/
|
|
3171
3647
|
async resolveCohortSeed(strandId, delegatePeerId) {
|
|
3172
3648
|
if (!this.controlDatabase || !this.controlNode) {
|
|
3173
|
-
return
|
|
3649
|
+
return [];
|
|
3174
3650
|
}
|
|
3175
|
-
const
|
|
3176
|
-
const { otherPeerIds, hasOtherPeers } = deriveCohortMembers(peers, this.controlNode.peerId.toString());
|
|
3177
|
-
// RPC only siblings we already have a control connection to — they can answer
|
|
3178
|
-
// now, and `dialProtocol` by peerId reuses the open connection.
|
|
3179
|
-
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3180
|
-
const targets = otherPeerIds
|
|
3181
|
-
.filter((id) => connected.has(id))
|
|
3182
|
-
.map((peerId) => ({ peerId }));
|
|
3651
|
+
const targets = await this.connectedSiblingTargets();
|
|
3183
3652
|
const relays = delegatePeerId === undefined ? [] : this.circuitRelayTargets();
|
|
3184
3653
|
for (const relay of relays) {
|
|
3185
3654
|
if (!targets.some((t) => t.peerId === relay.relayPeerId)) {
|
|
@@ -3192,15 +3661,70 @@ export class CadreNode {
|
|
|
3192
3661
|
// Throttle state for the RELAY targets only — refreshDelegateGrants never
|
|
3193
3662
|
// looks up a sibling key, so recording one would only be dead weight.
|
|
3194
3663
|
this.recordDelegateAnnounces(relays.map((r) => r.relayPeerId), strandId);
|
|
3195
|
-
return
|
|
3664
|
+
return bootstrapNodes;
|
|
3665
|
+
}
|
|
3666
|
+
/**
|
|
3667
|
+
* The co-cadre siblings worth sending a strand-addr RPC to right now: cohort
|
|
3668
|
+
* members (self excluded) we already hold an open control connection to. They
|
|
3669
|
+
* are the ones that can answer immediately, and dialing them by peerId reuses
|
|
3670
|
+
* the live connection instead of opening a second one.
|
|
3671
|
+
*
|
|
3672
|
+
* Shared by {@link resolveCohortSeed} (launch/resume) and
|
|
3673
|
+
* {@link refreshStrandPeerAddrs} (the periodic pass) so both ask the same set.
|
|
3674
|
+
* Empty when the control DB or node is absent (not yet started / torn down).
|
|
3675
|
+
*
|
|
3676
|
+
* The read below is issued even when the node holds no control connection and
|
|
3677
|
+
* the answer is therefore empty by construction. That is deliberate on the
|
|
3678
|
+
* launch path: `control-database-solo-warm-start.spec.ts` exists to prove this
|
|
3679
|
+
* exact read does not stall as an embedder's FIRST awaited control operation,
|
|
3680
|
+
* on a warm cohort no one can reach. A caller that re-enters often enough for
|
|
3681
|
+
* the read to matter should skip the call itself, as
|
|
3682
|
+
* {@link refreshStrandPeerAddrs} does.
|
|
3683
|
+
*
|
|
3684
|
+
* NOTE: this read is UNBOUNDED and sits on the critical path an embedding app
|
|
3685
|
+
* awaits during startup — `addStrand` cannot resolve until it does, and it is
|
|
3686
|
+
* the first control operation an embedder's boot order issues (before genesis,
|
|
3687
|
+
* before seed bootstrap). Measured fine today: the warm-start-alone shape a
|
|
3688
|
+
* report pointed at — a `CadrePeer` list naming peers that are all gone, read
|
|
3689
|
+
* off real files after a restart — completes in milliseconds, and
|
|
3690
|
+
* `control-database-solo-warm-start.spec.ts` covers it under a deadline. If a
|
|
3691
|
+
* control read ever CAN stall (a transactor change that consults the network
|
|
3692
|
+
* for a local row, a storage backend with blocking I/O), give this one a
|
|
3693
|
+
* timeout — and decide then whether the breach fails `addStrand` or degrades
|
|
3694
|
+
* to an empty seed, because those promise callers different things.
|
|
3695
|
+
*/
|
|
3696
|
+
async connectedSiblingTargets() {
|
|
3697
|
+
const controlNode = this.controlNode;
|
|
3698
|
+
if (!this.controlDatabase || !controlNode) {
|
|
3699
|
+
return [];
|
|
3700
|
+
}
|
|
3701
|
+
const peers = await this.controlDatabase.queryCadrePeers();
|
|
3702
|
+
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3703
|
+
return deriveCohortMembers(peers, controlNode.peerId.toString())
|
|
3704
|
+
.filter((id) => connected.has(id))
|
|
3705
|
+
.map((peerId) => ({ peerId }));
|
|
3196
3706
|
}
|
|
3197
3707
|
/**
|
|
3198
3708
|
* The circuit relays this node's strand nodes would reserve through: the
|
|
3199
|
-
* union of
|
|
3200
|
-
* `network.listenAddrs` circuit entry
|
|
3201
|
-
*
|
|
3202
|
-
*
|
|
3203
|
-
*
|
|
3709
|
+
* union of CONFIGURED relays — `network.relayAddrs`, plus any hand-written
|
|
3710
|
+
* `network.listenAddrs` circuit entry, both of which a strand node inherits
|
|
3711
|
+
* verbatim — and the control node's own live `/p2p-circuit` multiaddrs (a
|
|
3712
|
+
* reservation this node discovered rather than configured).
|
|
3713
|
+
*
|
|
3714
|
+
* The CONFIGURED half reads `network.relayAddrs` and `network.listenAddrs`
|
|
3715
|
+
* directly rather than going through {@link resolveListenAddrs}: the control
|
|
3716
|
+
* node's resolution takes the `'search'` route, whose bare `/p2p-circuit` entry
|
|
3717
|
+
* names no relay, so a configured relay would be unannounceable here until this
|
|
3718
|
+
* node's own reservation landed — and the announce must be in place BEFORE a
|
|
3719
|
+
* strand node dials its reservation, since the relay's membership gate does not
|
|
3720
|
+
* know the strand's derived peerId (see delegate-admission.ts). Configuration is
|
|
3721
|
+
* authoritative; the live multiaddrs only ADD relays nobody configured.
|
|
3722
|
+
*
|
|
3723
|
+
* The `listenAddrs` half is now unreachable on a control node — `relay-addrs.ts`
|
|
3724
|
+
* rejects a hand-written `<relay>/p2p-circuit` listen entry outright, because the
|
|
3725
|
+
* bring-up quiet period denies the dial that listener makes from inside
|
|
3726
|
+
* `libp2p.start()`. Kept because it costs one spread and this is the one place
|
|
3727
|
+
* that would silently stop announcing if that rejection is ever relaxed.
|
|
3204
3728
|
*
|
|
3205
3729
|
* NOTE: a relay the STRAND node discovers on its own (autorelay — in neither
|
|
3206
3730
|
* source) gets no announcement, and a membership-gated one will deny it; fine
|
|
@@ -3208,7 +3732,8 @@ export class CadreNode {
|
|
|
3208
3732
|
*/
|
|
3209
3733
|
circuitRelayTargets() {
|
|
3210
3734
|
return extractCircuitRelayTargets([
|
|
3211
|
-
...(
|
|
3735
|
+
...relayCircuitAddrs(this.config.network?.relayAddrs ?? []),
|
|
3736
|
+
...(this.config.network?.listenAddrs ?? []),
|
|
3212
3737
|
...(this.controlNode?.getMultiaddrs().map(String) ?? [])
|
|
3213
3738
|
]);
|
|
3214
3739
|
}
|
|
@@ -3277,6 +3802,157 @@ export class CadreNode {
|
|
|
3277
3802
|
await collectStrandAddrs(controlNode, due.map(relayStrandAddrPeer), strandId, { delegatePeerId });
|
|
3278
3803
|
this.recordDelegateAnnounces(due.map((relay) => relay.relayPeerId), strandId, now);
|
|
3279
3804
|
}
|
|
3805
|
+
/**
|
|
3806
|
+
* Re-resolve each running strand's SIBLING addresses over the control mesh and
|
|
3807
|
+
* merge them into that strand's own libp2p address book, throttled to once per
|
|
3808
|
+
* {@link STRAND_PEER_ADDR_REFRESH_MS} per strand.
|
|
3809
|
+
*
|
|
3810
|
+
* Without this, a strand's address book is written exactly once — the
|
|
3811
|
+
* launch/resume seed — and everything below cadre-core that dials a strand peer
|
|
3812
|
+
* by bare peer id (Optimystic's cluster and repo clients, FRET ping/announce)
|
|
3813
|
+
* loses its address for any sibling it is not currently connected to: a sibling
|
|
3814
|
+
* that restarted its strand node or rotated its relay reservation is never
|
|
3815
|
+
* re-resolved, and even the original seed addresses fall off at the peerStore's
|
|
3816
|
+
* one-hour expiry (see `peer-addr-book.ts`).
|
|
3817
|
+
*
|
|
3818
|
+
* Distinct from {@link refreshDelegateGrants}, deliberately: that pass covers
|
|
3819
|
+
* RELAYS on a `DELEGATE_GRANT_TTL_MS / 2` throttle to keep circuit-relay
|
|
3820
|
+
* admission grants alive, this one covers SIBLINGS on an address-expiry
|
|
3821
|
+
* throttle to keep the address book warm. They overlap only in that both carry
|
|
3822
|
+
* `delegatePeerId`, so a sibling that also runs a relay gets its grant
|
|
3823
|
+
* refreshed here as a side effect. Merging them would tie an admission-grant
|
|
3824
|
+
* TTL to an address-expiry window that has nothing to do with it.
|
|
3825
|
+
*
|
|
3826
|
+
* Strands refresh CONCURRENTLY, like `refreshDelegateGrants`, so one
|
|
3827
|
+
* unreachable sibling's dial timeout does not stack up per strand.
|
|
3828
|
+
*/
|
|
3829
|
+
async refreshStrandPeerAddrs(now = Date.now()) {
|
|
3830
|
+
if (!this._running || !this.controlNode) {
|
|
3831
|
+
return;
|
|
3832
|
+
}
|
|
3833
|
+
// A hibernating / quiescing strand has no node to seed, so it is not running
|
|
3834
|
+
// for this pass's purposes even though its instance is still tracked.
|
|
3835
|
+
const running = new Set();
|
|
3836
|
+
for (const [strandId, instance] of this.strandManager.getInstances()) {
|
|
3837
|
+
if (instance.libp2pNode) {
|
|
3838
|
+
running.add(strandId);
|
|
3839
|
+
}
|
|
3840
|
+
}
|
|
3841
|
+
// Drop stamps for strands no longer running: the map must not grow for the
|
|
3842
|
+
// node's lifetime, and a resumed strand should refresh immediately rather
|
|
3843
|
+
// than inherit the stamp its previous incarnation left.
|
|
3844
|
+
for (const strandId of this.strandPeerAddrRefreshAt.keys()) {
|
|
3845
|
+
if (!running.has(strandId)) {
|
|
3846
|
+
this.strandPeerAddrRefreshAt.delete(strandId);
|
|
3847
|
+
}
|
|
3848
|
+
}
|
|
3849
|
+
const refreshMs = this.config.network?.controlCohort?.strandAddrRefreshMs ?? STRAND_PEER_ADDR_REFRESH_MS;
|
|
3850
|
+
const due = [...running].filter((strandId) => now - (this.strandPeerAddrRefreshAt.get(strandId) ?? 0) >= refreshMs);
|
|
3851
|
+
if (due.length === 0) {
|
|
3852
|
+
return;
|
|
3853
|
+
}
|
|
3854
|
+
// A due strand with nobody to ask leaves its throttle unstamped (see below),
|
|
3855
|
+
// so this pass re-enters on EVERY 15 s reconcile tick for as long as the node
|
|
3856
|
+
// is alone — which is the steady state of a solo node running a strand.
|
|
3857
|
+
// `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
3858
|
+
// connections its answer is empty whatever the table holds, so decide it here
|
|
3859
|
+
// from the connection list instead of paying for the read once a tick.
|
|
3860
|
+
if (this.controlNode.getConnections().length === 0) {
|
|
3861
|
+
return;
|
|
3862
|
+
}
|
|
3863
|
+
// NOTE: one strand-addr RPC per (due strand × connected sibling) per refresh
|
|
3864
|
+
// interval — each a tiny request/response on an already-open control
|
|
3865
|
+
// connection. If a node ever runs MANY strands at once, batch the RPC to
|
|
3866
|
+
// carry several strand ids per request rather than one fan-out per strand.
|
|
3867
|
+
const targets = await this.connectedSiblingTargets().catch((error) => {
|
|
3868
|
+
log('refreshStrandPeerAddrs: sibling enumeration failed (skipping pass): %o', error);
|
|
3869
|
+
return [];
|
|
3870
|
+
});
|
|
3871
|
+
// No sibling to ask (or a shutdown landed mid-enumeration): leave every stamp
|
|
3872
|
+
// untouched so the next reconcile tick retries, rather than sitting out the
|
|
3873
|
+
// whole refresh interval having asked nobody.
|
|
3874
|
+
if (targets.length === 0 || !this._running || !this.controlNode) {
|
|
3875
|
+
return;
|
|
3876
|
+
}
|
|
3877
|
+
await Promise.all(due.map((strandId) => this.refreshOneStrandPeerAddrs(strandId, targets, now)));
|
|
3878
|
+
}
|
|
3879
|
+
/**
|
|
3880
|
+
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings, then
|
|
3881
|
+
* merge their answers into this strand's address book. Errors are logged and
|
|
3882
|
+
* swallowed so one strand's failure never costs the others their refresh.
|
|
3883
|
+
*/
|
|
3884
|
+
async refreshOneStrandPeerAddrs(strandId, targets, now) {
|
|
3885
|
+
const controlNode = this.controlNode;
|
|
3886
|
+
const strandNode = this.strandManager.getInstance(strandId)?.libp2pNode;
|
|
3887
|
+
if (!controlNode || !strandNode) {
|
|
3888
|
+
return;
|
|
3889
|
+
}
|
|
3890
|
+
try {
|
|
3891
|
+
// The running strand node's own peerId is the delegate to announce — see
|
|
3892
|
+
// the relationship note on refreshStrandPeerAddrs.
|
|
3893
|
+
const addrs = await collectStrandAddrs(controlNode, [...targets], strandId, {
|
|
3894
|
+
delegatePeerId: strandNode.peerId.toString()
|
|
3895
|
+
});
|
|
3896
|
+
// Stamped on the RPC having happened, not on its answer: the fan-out is
|
|
3897
|
+
// what the throttle exists to bound, and an all-empty round is a normal
|
|
3898
|
+
// steady state for a strand no connected sibling currently runs.
|
|
3899
|
+
this.strandPeerAddrRefreshAt.set(strandId, now);
|
|
3900
|
+
// Re-read the instance after the await — a strand stopped mid-pass (or one
|
|
3901
|
+
// already restarted onto a new node) must never have its store written to.
|
|
3902
|
+
if (!this._running || this.strandManager.getInstance(strandId)?.libp2pNode !== strandNode) {
|
|
3903
|
+
return;
|
|
3904
|
+
}
|
|
3905
|
+
await this.mergeStrandPeerAddrs(strandNode, addrs, strandId);
|
|
3906
|
+
}
|
|
3907
|
+
catch (error) {
|
|
3908
|
+
log('refreshStrandPeerAddrs: strand %s refresh failed (continuing): %o', strandId, error);
|
|
3909
|
+
}
|
|
3910
|
+
}
|
|
3911
|
+
/**
|
|
3912
|
+
* Merge strand-network addresses into ONE strand node's libp2p address book,
|
|
3913
|
+
* attributed per peer. Best-effort throughout: an address-book write must never
|
|
3914
|
+
* fail a launch, a resume, or a reconcile pass.
|
|
3915
|
+
*
|
|
3916
|
+
* `addrs` is the peer-agnostic union the strand-addr RPC returns, so
|
|
3917
|
+
* {@link groupAddrsByPeerId} attributes each entry to the **strand transport**
|
|
3918
|
+
* peerId in its final `/p2p/` component — never the sibling's control peerId,
|
|
3919
|
+
* which names a different libp2p node entirely. Entries that name no peer are
|
|
3920
|
+
* dropped there and counted here, once per pass.
|
|
3921
|
+
*
|
|
3922
|
+
* NOTE: a member could answer with arbitrary multiaddrs bound to arbitrary peer
|
|
3923
|
+
* ids and poison this address book. That is the same exposure the launch-time
|
|
3924
|
+
* `bootstrapNodes` seeding already accepts, and the cost is bounded: an address
|
|
3925
|
+
* grants no authority, the dialed peer authenticates by peer id at the
|
|
3926
|
+
* handshake, and a bad entry costs one failed dial that ages out at the
|
|
3927
|
+
* peerStore's one-hour expiry. No new gating here — cross-party strand trust is
|
|
3928
|
+
* `backlog/strand-network-nat-relay-reachability`.
|
|
3929
|
+
*/
|
|
3930
|
+
async mergeStrandPeerAddrs(strandNode, addrs, strandId) {
|
|
3931
|
+
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
3932
|
+
let grouped = 0;
|
|
3933
|
+
let peers = 0;
|
|
3934
|
+
try {
|
|
3935
|
+
const groups = groupAddrsByPeerId(addrs);
|
|
3936
|
+
peers = groups.size;
|
|
3937
|
+
const selfId = strandNode.peerId.toString();
|
|
3938
|
+
for (const [peerId, peerAddrs] of groups) {
|
|
3939
|
+
grouped += peerAddrs.length;
|
|
3940
|
+
// We never RPC ourselves, so self should never appear — belt and braces:
|
|
3941
|
+
// a node must not write its own addresses into its own address book.
|
|
3942
|
+
if (peerId === selfId) {
|
|
3943
|
+
continue;
|
|
3944
|
+
}
|
|
3945
|
+
counts[await mergePeerAddrs(strandNode, peerId, peerAddrs)]++;
|
|
3946
|
+
}
|
|
3947
|
+
}
|
|
3948
|
+
catch (error) {
|
|
3949
|
+
// `mergePeerAddrs` folds its own failures, so reaching here means the node
|
|
3950
|
+
// itself is unusable (torn down mid-merge). Never fail a launch/resume or a
|
|
3951
|
+
// reconcile pass over an address book.
|
|
3952
|
+
log('strand %s address book merge failed (continuing): %o', strandId, error);
|
|
3953
|
+
}
|
|
3954
|
+
log('strand %s address book merged (peers=%d, merged=%d, restamped=%d, skipped=%d, failed=%d, dropped=%d)', strandId, peers, counts.merged, counts.restamped, counts.skipped, counts.failed, addrs.length - grouped);
|
|
3955
|
+
}
|
|
3280
3956
|
/**
|
|
3281
3957
|
* The local strand instance's dialable strand-network multiaddrs for the
|
|
3282
3958
|
* strand-addr RPC, ordered signaling-first (reusing the control node's
|
|
@@ -3508,16 +4184,60 @@ export class CadreNode {
|
|
|
3508
4184
|
getControlNode() {
|
|
3509
4185
|
return this.controlNode;
|
|
3510
4186
|
}
|
|
4187
|
+
/**
|
|
4188
|
+
* The boot-path half of {@link reserveRelays}: reserve through every
|
|
4189
|
+
* `network.relayAddrs` entry, or fail `start()`.
|
|
4190
|
+
*
|
|
4191
|
+
* WHY IT RUNS HERE, AT THE END. `network.relayAddrs` used to resolve to a
|
|
4192
|
+
* CONFIGURED circuit listener, which libp2p dials from inside `libp2p.start()`
|
|
4193
|
+
* — so the relay was already a connected peer, and therefore already in this
|
|
4194
|
+
* node's Optimystic cohort, before `ControlDatabase.initialize()` ran. Building
|
|
4195
|
+
* that database is a long chain of cohort-consulting block probes (catalog
|
|
4196
|
+
* hydration, then one per table, then the indexes), and a sibling that has not
|
|
4197
|
+
* yet replicated this node's `CadrePeer` row correctly refuses every one of
|
|
4198
|
+
* them, so bring-up died on `BlockUnavailableError` — deterministically, on
|
|
4199
|
+
* every boot of a relay-only node. Retrying could not converge either: the
|
|
4200
|
+
* condition that clears the refusal is this node's own row reaching the
|
|
4201
|
+
* sibling, and writing that row needs the database the retry is building.
|
|
4202
|
+
*
|
|
4203
|
+
* So the invariant is ordering, not retrying: the control database is built
|
|
4204
|
+
* while this node holds ZERO control connections (a cohort of one, entirely
|
|
4205
|
+
* local), and only then does it reach out. Everything above in {@link start}
|
|
4206
|
+
* has completed by the time this runs.
|
|
4207
|
+
*
|
|
4208
|
+
* BUDGET: the supervisor's first attempt — `relay-reservation.ts`'s
|
|
4209
|
+
* `DEFAULT_RELAY_RESERVE_TIMEOUT_MS`, 10 s — is what `start()` waits on — deliberately the module default rather
|
|
4210
|
+
* than a boot-specific one. A healthy dial-plus-reserve is sub-second even over
|
|
4211
|
+
* a WAN, so 10 s is slack for a slow link; going much longer would make a dead
|
|
4212
|
+
* relay indistinguishable from a hung start, and much shorter would fail nodes
|
|
4213
|
+
* on links that were merely slow. The retries carry on in the background after
|
|
4214
|
+
* this resolves, exactly as they do for a {@link reserveRelays} caller.
|
|
4215
|
+
*/
|
|
4216
|
+
async driveControlRelayReservation() {
|
|
4217
|
+
const relayAddrs = this.config.network?.relayAddrs ?? [];
|
|
4218
|
+
if (relayAddrs.length === 0) {
|
|
4219
|
+
return;
|
|
4220
|
+
}
|
|
4221
|
+
const t0 = performance.now();
|
|
4222
|
+
const state = await this.reserveRelays([...relayAddrs]);
|
|
4223
|
+
timing('[start] reserveRelays: %dms', Math.round(performance.now() - t0));
|
|
4224
|
+
if (state.status !== 'reserved') {
|
|
4225
|
+
throw new RelayReservationFailedError(relayAddrs, state);
|
|
4226
|
+
}
|
|
4227
|
+
log('Reserved a relay slot: %o', state.circuitAddrs);
|
|
4228
|
+
}
|
|
3511
4229
|
/**
|
|
3512
4230
|
* Start keeping a relay reservation: dial the given relay(s) from the control
|
|
3513
4231
|
* node, ask the first one that answers for a reservation slot, wait until the
|
|
3514
4232
|
* resulting `/p2p-circuit` address makes this node dialable — and keep a
|
|
3515
4233
|
* supervisor running that re-drives whenever the reservation is later lost.
|
|
3516
4234
|
*
|
|
3517
|
-
*
|
|
3518
|
-
*
|
|
3519
|
-
*
|
|
3520
|
-
*
|
|
4235
|
+
* The FAIL-SOFT entry point, for a caller that discovers its relay at runtime
|
|
4236
|
+
* (a browser tab, a host UI). A node that names `network.relayAddrs` gets the
|
|
4237
|
+
* same drive from {@link driveControlRelayReservation} at the end of
|
|
4238
|
+
* {@link start}, which is fail-fast instead. Both fill the pending reservation
|
|
4239
|
+
* the bare `/p2p-circuit` search listener registered; calling this afterwards
|
|
4240
|
+
* simply replaces the supervisor with one over the new list.
|
|
3521
4241
|
*
|
|
3522
4242
|
* Resolves once the FIRST attempt has settled, exactly as it did when it drove
|
|
3523
4243
|
* once; the retries continue in the background until {@link stop} or the next
|
|
@@ -3650,6 +4370,10 @@ export class CadreNode {
|
|
|
3650
4370
|
* Push the multiaddrs that future invites should advertise. Pass `null` to
|
|
3651
4371
|
* revert to the libp2p-reported addresses (the default). The host calls this
|
|
3652
4372
|
* at spawn and on every NAT change.
|
|
4373
|
+
*
|
|
4374
|
+
* Entries need NOT carry a `/p2p/<peerId>` suffix — {@link resolveInviteAddresses}
|
|
4375
|
+
* appends this node's own before anything publishes or dials them (see
|
|
4376
|
+
* {@link normalizeSelfAddrs}). Passing them suffixed is equally fine.
|
|
3653
4377
|
*/
|
|
3654
4378
|
setInviteAddresses(addresses) {
|
|
3655
4379
|
this.latestInviteAddresses = addresses;
|
|
@@ -3657,17 +4381,59 @@ export class CadreNode {
|
|
|
3657
4381
|
}
|
|
3658
4382
|
/**
|
|
3659
4383
|
* Resolve the addresses to embed in invites. Prefers pushed addresses, then
|
|
3660
|
-
* any config-supplied resolver, then the libp2p-observed multiaddrs.
|
|
4384
|
+
* any config-supplied resolver, then the libp2p-observed multiaddrs. The two
|
|
4385
|
+
* app-supplied sources are normalized onto `/p2p/<self>` (see
|
|
4386
|
+
* {@link normalizeSelfAddrs}) so neither hook has to remember the suffix;
|
|
4387
|
+
* libp2p's own addresses already carry it.
|
|
3661
4388
|
*/
|
|
3662
4389
|
async resolveInviteAddresses() {
|
|
3663
4390
|
if (this.latestInviteAddresses !== null) {
|
|
3664
|
-
return this.latestInviteAddresses;
|
|
4391
|
+
return this.normalizeSelfAddrs(this.latestInviteAddresses);
|
|
3665
4392
|
}
|
|
3666
4393
|
if (this.config.network?.inviteAddressResolver) {
|
|
3667
|
-
return this.config.network.inviteAddressResolver();
|
|
4394
|
+
return this.normalizeSelfAddrs(await this.config.network.inviteAddressResolver());
|
|
3668
4395
|
}
|
|
4396
|
+
// Not normalized: this branch is libp2p's own output, which already
|
|
4397
|
+
// encapsulates the peer id into every address it reports. Only the two app
|
|
4398
|
+
// hooks above take arbitrary strings and therefore need the guarantee.
|
|
3669
4399
|
return this.getMultiaddrs();
|
|
3670
4400
|
}
|
|
4401
|
+
/**
|
|
4402
|
+
* Normalize addresses an app handed us for THIS node onto `/p2p/<self>`, the
|
|
4403
|
+
* same invariant {@link normalizeDialAddrs} enforces on the way out.
|
|
4404
|
+
*
|
|
4405
|
+
* `setInviteAddresses` (the admin API `PUT /admin/invite-addresses`) and
|
|
4406
|
+
* `network.inviteAddressResolver` both take arbitrary strings, and whatever
|
|
4407
|
+
* they return ends up in this node's published `CadrePeer` record — so ONE
|
|
4408
|
+
* unsuffixed entry from either hook is enough to give every sibling in the
|
|
4409
|
+
* party a mixed candidate list for this peer. Establishing the suffix here,
|
|
4410
|
+
* where those strings enter the system, means neither hook has to know the
|
|
4411
|
+
* rule; `cadre-host`'s `buildInviteAddresses` already appends it, and
|
|
4412
|
+
* re-normalizing an already-suffixed address is a no-op.
|
|
4413
|
+
*
|
|
4414
|
+
* Unparsable and other-peer-addressed entries are passed through untouched
|
|
4415
|
+
* rather than dropped: publication is not the place to police an address's
|
|
4416
|
+
* validity, and {@link resolvePeerAddrs} already drops both on the read side.
|
|
4417
|
+
* Before `start()` resolves an identity there is no peer id to append, so the
|
|
4418
|
+
* list is returned as given.
|
|
4419
|
+
*/
|
|
4420
|
+
normalizeSelfAddrs(addrs) {
|
|
4421
|
+
const selfPeerId = this.peerId?.toString();
|
|
4422
|
+
if (!selfPeerId) {
|
|
4423
|
+
return addrs;
|
|
4424
|
+
}
|
|
4425
|
+
return addrs.map((addr) => {
|
|
4426
|
+
let parsed;
|
|
4427
|
+
try {
|
|
4428
|
+
parsed = multiaddr(addr);
|
|
4429
|
+
}
|
|
4430
|
+
catch (error) {
|
|
4431
|
+
log('normalizeSelfAddrs: leaving unparsable multiaddr %s as-is: %o', addr, error);
|
|
4432
|
+
return addr;
|
|
4433
|
+
}
|
|
4434
|
+
return withTrailingPeerId(parsed, selfPeerId)?.toString() ?? addr;
|
|
4435
|
+
});
|
|
4436
|
+
}
|
|
3671
4437
|
/**
|
|
3672
4438
|
* Enumerate the cadre's `CadrePeer` membership — the ADDRESSABLE surface (see
|
|
3673
4439
|
* {@link isMember}). Rows whose stamp is retired in `CadreControl.Revocation`
|
|
@@ -3729,13 +4495,18 @@ export class CadreNode {
|
|
|
3729
4495
|
* legit member goes un-authorized on readers that only pin the new key. Full
|
|
3730
4496
|
* rotation handling (re-vouch on rotate) is the
|
|
3731
4497
|
* `flip-strand-membership-rotation-known-gap` work, not this predicate's.
|
|
4498
|
+
*
|
|
4499
|
+
* @param retry - Whether the underlying membership read may retry a transient cluster
|
|
4500
|
+
* failure. Only {@link refreshAuthorizedControlPeers} passes `false`, because it runs
|
|
4501
|
+
* as the control database's membership listener with that database's write lock held;
|
|
4502
|
+
* see its comment.
|
|
3732
4503
|
*/
|
|
3733
|
-
async listAuthorizedMembers() {
|
|
4504
|
+
async listAuthorizedMembers(retry = true) {
|
|
3734
4505
|
if (!this.controlDatabase) {
|
|
3735
4506
|
throw new Error('CadreNode must be started before listing members');
|
|
3736
4507
|
}
|
|
3737
4508
|
const selfPeerId = this.peerId?.toString();
|
|
3738
|
-
const rows = await this.controlDatabase.queryCadrePeers();
|
|
4509
|
+
const rows = await this.controlDatabase.queryCadrePeers(retry);
|
|
3739
4510
|
const authorized = rows
|
|
3740
4511
|
.filter(row => row.peerId !== selfPeerId && this.hasAnchoredVoucher(row))
|
|
3741
4512
|
.map(({ peerId, multiaddr }) => ({ peerId, multiaddr }));
|