@serfab/cadre-core 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -255
- package/dist/announce-addrs.d.ts +49 -0
- package/dist/announce-addrs.js +78 -0
- package/dist/announce-addrs.js.map +1 -0
- package/dist/cadre-node.d.ts +484 -58
- package/dist/cadre-node.js +953 -182
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +26 -0
- package/dist/control-cohort.js +26 -0
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +222 -125
- package/dist/control-database.js +356 -266
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +8 -7
- package/dist/control-formation-recorder.js +8 -7
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +103 -0
- package/dist/control-read-retry.js +145 -0
- package/dist/control-read-retry.js.map +1 -0
- package/dist/control-retry.d.ts +101 -0
- package/dist/control-retry.js +108 -0
- package/dist/control-retry.js.map +1 -0
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +805 -774
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +65 -31
- package/dist/control-write-retry.js +33 -103
- package/dist/control-write-retry.js.map +1 -1
- package/dist/delegate-admission.d.ts +17 -15
- package/dist/delegate-admission.js +17 -15
- package/dist/delegate-admission.js.map +1 -1
- package/dist/index.d.ts +7 -5
- package/dist/index.js +18 -9
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +211 -25
- package/dist/membership-connection-gater.js +309 -29
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/peer-addr-book.d.ts +92 -0
- package/dist/peer-addr-book.js +220 -0
- package/dist/peer-addr-book.js.map +1 -0
- package/dist/peer-record.d.ts +26 -0
- package/dist/peer-record.js +32 -0
- package/dist/peer-record.js.map +1 -1
- package/dist/relay-addrs.d.ts +76 -26
- package/dist/relay-addrs.js +115 -27
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +20 -15
- package/dist/relay-reservation.js +18 -14
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.d.ts +7 -0
- package/dist/seed-bootstrap.js +15 -13
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-addr-protocol.js +16 -0
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-backfill.js +13 -4
- package/dist/strand-backfill.js.map +1 -1
- package/dist/strand-cohort.d.ts +4 -37
- package/dist/strand-cohort.js +4 -15
- package/dist/strand-cohort.js.map +1 -1
- package/dist/strand-database.d.ts +11 -16
- package/dist/strand-database.js +19 -7
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +18 -17
- package/dist/strand-formation-manager.js +25 -27
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +7 -7
- package/dist/strand-formation-protocol.js +8 -8
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +53 -21
- package/dist/strand-instance-manager.js +128 -37
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +113 -19
- package/dist/strand-membership-writer.js +219 -51
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-solicitation.d.ts +3 -3
- package/dist/strand-solicitation.js +4 -5
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +47 -2
- package/dist/strand-wake-protocol.js +76 -6
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/types.d.ts +107 -54
- package/dist/types.js.map +1 -1
- package/package.json +8 -7
- package/src/announce-addrs.ts +93 -0
- package/src/arachnode-stub.ts +159 -159
- package/src/bootstrap-peer-store-file.ts +44 -44
- package/src/bootstrap-peer-store.ts +172 -172
- package/src/cadre-node.ts +1026 -194
- package/src/canonical-json.ts +30 -30
- package/src/control-authorization.ts +114 -114
- package/src/control-cohort.ts +130 -103
- package/src/control-database.ts +376 -298
- package/src/control-formation-recorder.ts +305 -304
- package/src/control-read-retry.ts +167 -0
- package/src/control-retry.ts +190 -0
- package/src/control-schema.ts +817 -786
- package/src/control-stream.ts +205 -205
- package/src/control-write-retry.ts +383 -473
- package/src/delegate-admission.ts +304 -302
- package/src/device-token.ts +94 -94
- package/src/diagnostics/connection-path.ts +277 -277
- package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
- package/src/ed25519-key.ts +134 -134
- package/src/enrollment.ts +199 -199
- package/src/file-durable-slot.ts +61 -61
- package/src/formation-approval.ts +604 -604
- package/src/fs-atomic.ts +107 -107
- package/src/hibernation-manager.ts +437 -437
- package/src/identity-key.ts +109 -109
- package/src/index.ts +435 -406
- package/src/key-store-file.ts +132 -132
- package/src/key-store.ts +99 -99
- package/src/membership-connection-gater.ts +368 -39
- package/src/node-local-snapshot.ts +260 -260
- package/src/peer-addr-book.ts +250 -0
- package/src/peer-authorization.ts +259 -259
- package/src/peer-record.ts +197 -164
- package/src/push-credentials.ts +84 -84
- package/src/push-node.ts +64 -64
- package/src/push-notifier-apns.ts +247 -247
- package/src/push-notifier-fcm.ts +220 -220
- package/src/push-notifier-shared.ts +28 -28
- package/src/push-notifier.ts +52 -52
- package/src/relay-addrs.ts +134 -27
- package/src/relay-reservation.ts +22 -17
- package/src/seed-bootstrap.ts +15 -15
- package/src/seed-trust-policy.ts +140 -140
- package/src/strand-addr-protocol.ts +458 -441
- package/src/strand-backfill.ts +28 -5
- package/src/strand-cohort.ts +38 -77
- package/src/strand-database.ts +202 -204
- package/src/strand-formation-manager.ts +26 -28
- package/src/strand-formation-protocol.ts +692 -692
- package/src/strand-instance-manager.ts +636 -553
- package/src/strand-member-key.ts +39 -39
- package/src/strand-member-registry.ts +182 -182
- package/src/strand-membership-writer.ts +1627 -1430
- package/src/strand-solicitation.ts +471 -472
- package/src/strand-transport-key.ts +58 -58
- package/src/strand-wake-payload.ts +32 -32
- package/src/strand-wake-protocol.ts +400 -314
- package/src/strand-watcher.ts +314 -314
- package/src/trusted-owner-store-file.ts +41 -41
- package/src/trusted-owner-store.ts +171 -171
- package/src/types.ts +1354 -1303
package/src/cadre-node.ts
CHANGED
|
@@ -2,7 +2,8 @@ import debug from 'debug';
|
|
|
2
2
|
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
3
3
|
import type { Libp2p, PeerId, PrivateKey, Connection } from '@libp2p/interface';
|
|
4
4
|
import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
5
|
-
import { createLibp2pNode } from '@optimystic/db-p2p';
|
|
5
|
+
import { createLibp2pNode, type IRawStorage } from '@optimystic/db-p2p';
|
|
6
|
+
import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
|
|
6
7
|
import { multiaddr } from '@multiformats/multiaddr';
|
|
7
8
|
import type { Multiaddr } from '@multiformats/multiaddr';
|
|
8
9
|
import type {
|
|
@@ -22,7 +23,6 @@ import type {
|
|
|
22
23
|
OpenInvitation,
|
|
23
24
|
FormStrandResult,
|
|
24
25
|
StrandFormationDisclosure,
|
|
25
|
-
StrandMode,
|
|
26
26
|
ResolveOpts,
|
|
27
27
|
SelfRegistrationOutcome,
|
|
28
28
|
ServiceWakeResult,
|
|
@@ -42,6 +42,7 @@ import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
|
42
42
|
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
43
43
|
import { MemoryTrustedOwnerStore, type TrustedOwnerStore, type TrustSource } from './trusted-owner-store.js';
|
|
44
44
|
import { MemoryBootstrapPeerStore, type BootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
45
|
+
import { mergePeerAddrs, groupAddrsByPeerId, type MergeAddrsResult } from './peer-addr-book.js';
|
|
45
46
|
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
46
47
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
47
48
|
import {
|
|
@@ -50,7 +51,7 @@ import {
|
|
|
50
51
|
isPeerRecordFresh,
|
|
51
52
|
orderSignalingFirst,
|
|
52
53
|
isSignalingAddr,
|
|
53
|
-
|
|
54
|
+
withTrailingPeerId,
|
|
54
55
|
currentMemberTrustPolicy,
|
|
55
56
|
DEFAULT_PEER_RECORD_MAX_AGE_MS,
|
|
56
57
|
DEFAULT_PEER_RECORD_HEARTBEAT_MS
|
|
@@ -62,13 +63,15 @@ import {
|
|
|
62
63
|
} from './device-token.js';
|
|
63
64
|
import { StrandWatcher, type StrandQueryable, type SAppIdLookup } from './strand-watcher.js';
|
|
64
65
|
import { StrandInstanceManager } from './strand-instance-manager.js';
|
|
65
|
-
import { deriveCohortMembers
|
|
66
|
+
import { deriveCohortMembers } from './strand-cohort.js';
|
|
66
67
|
import type { CohortPeerRow } from './strand-cohort.js';
|
|
67
68
|
import {
|
|
68
69
|
selectControlCohortDials,
|
|
70
|
+
DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
|
|
69
71
|
DEFAULT_CONTROL_COHORT_RECONCILE_MS,
|
|
70
72
|
DEFAULT_CONTROL_COHORT_TARGET_DEGREE
|
|
71
73
|
} from './control-cohort.js';
|
|
74
|
+
import { withDeadline } from './control-stream.js';
|
|
72
75
|
import { EnrollmentService } from './enrollment.js';
|
|
73
76
|
import { HibernationManager, type HibernationCallbacks } from './hibernation-manager.js';
|
|
74
77
|
import { ControlDatabase, type RevokedRowRef } from './control-database.js';
|
|
@@ -78,7 +81,12 @@ import {
|
|
|
78
81
|
StrandSolicitationService,
|
|
79
82
|
type StrandSolicitationServiceOptions
|
|
80
83
|
} from './strand-solicitation.js';
|
|
81
|
-
import {
|
|
84
|
+
import {
|
|
85
|
+
createMembershipConnectionGater,
|
|
86
|
+
DEFAULT_ENROLLMENT_WINDOW_MS,
|
|
87
|
+
UnauthorizedReservationBudget,
|
|
88
|
+
type InboundConnectionVerdict
|
|
89
|
+
} from './membership-connection-gater.js';
|
|
82
90
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
83
91
|
import { StrandAddrService, collectStrandAddrs, type StrandAddrPeer } from './strand-addr-protocol.js';
|
|
84
92
|
import {
|
|
@@ -89,7 +97,8 @@ import {
|
|
|
89
97
|
peerStrandKey,
|
|
90
98
|
type CircuitRelayTarget
|
|
91
99
|
} from './delegate-admission.js';
|
|
92
|
-
import { resolveListenAddrs } from './relay-addrs.js';
|
|
100
|
+
import { relayCircuitAddrs, resolveListenAddrs, RelayReservationFailedError } from './relay-addrs.js';
|
|
101
|
+
import { replacesAdvertisedAddrs, resolveAnnounceAddrs } from './announce-addrs.js';
|
|
93
102
|
import {
|
|
94
103
|
superviseRelayReservation,
|
|
95
104
|
resolveRelayReservationState,
|
|
@@ -117,6 +126,19 @@ const timing = debug('sereus:cadre:timing');
|
|
|
117
126
|
*/
|
|
118
127
|
const TURN_CONSUME_WINDOW_MS = 1000;
|
|
119
128
|
|
|
129
|
+
/**
|
|
130
|
+
* How often a running strand re-resolves its siblings' strand-network addresses
|
|
131
|
+
* and re-merges them into its own libp2p address book
|
|
132
|
+
* ({@link CadreNode.refreshStrandPeerAddrs}). Overridable per node via
|
|
133
|
+
* `network.controlCohort.strandAddrRefreshMs`.
|
|
134
|
+
*
|
|
135
|
+
* Ten minutes sits comfortably inside the peerStore's one-hour address expiry
|
|
136
|
+
* (`MAX_ADDRESS_AGE`, see `peer-addr-book.ts`) with headroom for several missed
|
|
137
|
+
* passes, and far above the 15 s reconcile cadence the refresh rides on, so the
|
|
138
|
+
* strand-addr RPC fan-out stays cheap.
|
|
139
|
+
*/
|
|
140
|
+
export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
141
|
+
|
|
120
142
|
type EventHandler<T> = (data: T) => void;
|
|
121
143
|
|
|
122
144
|
/**
|
|
@@ -204,6 +226,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
204
226
|
private pushFanoutService: PushFanoutService | null = null;
|
|
205
227
|
/** Backing field for the {@link running} / {@link isRunning} getters. */
|
|
206
228
|
private _running = false;
|
|
229
|
+
/**
|
|
230
|
+
* The control database's own raw storage (cache-wrapped), resolved once per
|
|
231
|
+
* `start()` and released in {@link cleanup}. Owning it is what keeps
|
|
232
|
+
* `config.storage.provider('control')` a once-per-runtime call: a `stop()` then
|
|
233
|
+
* `start()` cycle on this object re-resolves against a live cache, and never
|
|
234
|
+
* orphans the previous wrapper's registration in the shared cache pool.
|
|
235
|
+
*/
|
|
236
|
+
private controlStorage: IRawStorage | null = null;
|
|
207
237
|
/**
|
|
208
238
|
* In-flight {@link serviceWake} operations keyed by strandId. Coalesces
|
|
209
239
|
* concurrent on-demand wakes for the same strand into one runtime build + one
|
|
@@ -220,10 +250,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
220
250
|
private eventHandlers: Map<keyof CadreNodeEvents, Set<EventHandler<never>>> = new Map();
|
|
221
251
|
|
|
222
252
|
/**
|
|
223
|
-
* Relay multiaddrs the caller asked {@link reserveRelays} to reserve through
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
*
|
|
253
|
+
* Relay multiaddrs the caller asked {@link reserveRelays} to reserve through —
|
|
254
|
+
* or, on a node that names `network.relayAddrs`, the ones
|
|
255
|
+
* {@link driveControlRelayReservation} asked for at the end of {@link start}.
|
|
256
|
+
* Empty (the default) means neither happened, so {@link getRelayReservationState}
|
|
257
|
+
* reports `none`.
|
|
227
258
|
*/
|
|
228
259
|
private relayReserveAddrs: string[] = [];
|
|
229
260
|
/**
|
|
@@ -239,6 +270,24 @@ export class CadreNode implements SAppIdLookup {
|
|
|
239
270
|
*/
|
|
240
271
|
private relayReserveError: string | null = null;
|
|
241
272
|
|
|
273
|
+
/**
|
|
274
|
+
* True from just before the control libp2p node is created until
|
|
275
|
+
* {@link ControlDatabase.initialize} has settled — the connection gate's
|
|
276
|
+
* BRING-UP QUIET PERIOD (`membership-connection-gater.ts`), during which this
|
|
277
|
+
* node refuses every control connection in both directions.
|
|
278
|
+
*
|
|
279
|
+
* The invariant: the control database is built while this node holds zero
|
|
280
|
+
* control connections. Every same-party sibling connected in that window joins
|
|
281
|
+
* the Optimystic cohort the bring-up's block probes consult, and a sibling that
|
|
282
|
+
* has not yet replicated this node's `CadrePeer` row refuses them all — which
|
|
283
|
+
* fails `start()` outright and cannot be retried into convergence (writing the
|
|
284
|
+
* row that would clear the refusal needs the database being built).
|
|
285
|
+
*
|
|
286
|
+
* Cleared on FAILURE as well as success ({@link cleanup} clears it), so
|
|
287
|
+
* teardown is never gated.
|
|
288
|
+
*/
|
|
289
|
+
private controlBringUpInFlight = false;
|
|
290
|
+
|
|
242
291
|
/** Map of strandId -> sAppConfig for sAppId filtering and management */
|
|
243
292
|
private sAppConfigs: Map<string, SAppConfig> = new Map();
|
|
244
293
|
|
|
@@ -305,13 +354,27 @@ export class CadreNode implements SAppIdLookup {
|
|
|
305
354
|
/**
|
|
306
355
|
* In-memory delegate admission grants (see `delegate-admission.ts`): the
|
|
307
356
|
* strand-node transport peerIds this party's members have announced over the
|
|
308
|
-
* strand-addr RPC, admitted at the CONNECTION
|
|
309
|
-
* strand node can hold a circuit-relay reservation on this
|
|
310
|
-
*
|
|
311
|
-
*
|
|
357
|
+
* strand-addr RPC, admitted at the CONNECTION and RESERVATION levels so a
|
|
358
|
+
* member's NAT'd strand node can hold a circuit-relay reservation on this
|
|
359
|
+
* node (without spending the unauthorized-reservation budget). Consulted
|
|
360
|
+
* ONLY by {@link admitInboundControlConnection} and
|
|
361
|
+
* {@link admitControlRelayReservation}; the fail-closed per-stream gate
|
|
362
|
+
* ({@link authorizeInboundControlStream}) never honors it.
|
|
312
363
|
*/
|
|
313
364
|
private readonly delegateAdmission = new DelegateAdmissionStore();
|
|
314
365
|
|
|
366
|
+
/**
|
|
367
|
+
* Bounded budget of concurrent circuit-relay reservations granted to peers
|
|
368
|
+
* this node cannot (yet) place as members — the boot-ordering window where a
|
|
369
|
+
* genuine sibling reserves before its `CadrePeer` row has replicated here.
|
|
370
|
+
* Consulted only by {@link admitControlRelayReservation} (the gater's
|
|
371
|
+
* `denyInboundRelayReservation` policy); authorized members and announced
|
|
372
|
+
* delegates are admitted before it and never counted. Cap from
|
|
373
|
+
* `network.unauthorizedRelayReservationCap` (default
|
|
374
|
+
* `MAX_UNAUTHORIZED_RELAY_RESERVATIONS`).
|
|
375
|
+
*/
|
|
376
|
+
private readonly unauthorizedRelayReservations: UnauthorizedReservationBudget;
|
|
377
|
+
|
|
315
378
|
/**
|
|
316
379
|
* When this node last ANNOUNCED a delegate, keyed `<targetPeerId>\n<strandId>`
|
|
317
380
|
* (the peer announced TO, not the delegate). Throttles
|
|
@@ -322,6 +385,17 @@ export class CadreNode implements SAppIdLookup {
|
|
|
322
385
|
*/
|
|
323
386
|
private readonly delegateAnnounceAt = new Map<string, number>();
|
|
324
387
|
|
|
388
|
+
/**
|
|
389
|
+
* When each running strand last had its siblings' strand addresses re-resolved
|
|
390
|
+
* into its own address book, keyed by strandId. Throttles
|
|
391
|
+
* {@link refreshStrandPeerAddrs} to once per {@link STRAND_PEER_ADDR_REFRESH_MS}
|
|
392
|
+
* per strand so the 15 s reconcile tick never becomes per-tick RPC chatter.
|
|
393
|
+
* Entries for strands that are no longer running are pruned on each pass, so a
|
|
394
|
+
* hibernated-then-resumed strand refreshes immediately rather than inheriting a
|
|
395
|
+
* stale stamp.
|
|
396
|
+
*/
|
|
397
|
+
private readonly strandPeerAddrRefreshAt = new Map<string, number>();
|
|
398
|
+
|
|
325
399
|
/**
|
|
326
400
|
* Cold-start bootstrap dial targets: the owner-flagged peers of every seed
|
|
327
401
|
* this node has applied, keyed by peer id, valued by the seed's multiaddr
|
|
@@ -466,6 +540,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
466
540
|
this.config = config;
|
|
467
541
|
this.strandManager = new StrandInstanceManager();
|
|
468
542
|
this.enrollmentService = new EnrollmentService();
|
|
543
|
+
this.unauthorizedRelayReservations = new UnauthorizedReservationBudget(
|
|
544
|
+
config.network?.unauthorizedRelayReservationCap
|
|
545
|
+
);
|
|
469
546
|
|
|
470
547
|
// Create hibernation manager with callbacks
|
|
471
548
|
const hibernationCallbacks: HibernationCallbacks = {
|
|
@@ -595,20 +672,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
595
672
|
|
|
596
673
|
log('Starting CadreNode for party: %s', this.config.controlNetwork.partyId);
|
|
597
674
|
|
|
598
|
-
// NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
599
|
-
// diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
600
|
-
// second such warning ever appears here, route both through a `CadreNodeEvents` entry the
|
|
601
|
-
// embedder surfaces instead of growing a console surface inside a library.
|
|
602
|
-
if (this.config.network?.announceAddrs && this.config.network.announceAddrs.length > 0) {
|
|
603
|
-
console.warn(
|
|
604
|
-
'network.announceAddrs is set but not yet supported (no upstream db-p2p option to apply it) ' +
|
|
605
|
-
'— this node will keep advertising its listen/relay addresses instead.'
|
|
606
|
-
);
|
|
607
|
-
}
|
|
608
|
-
|
|
609
675
|
try {
|
|
610
676
|
const tTotal = performance.now();
|
|
611
677
|
|
|
678
|
+
// Config pre-flight, inside the guarded region: resolving the listen addrs it reads
|
|
679
|
+
// throws on a malformed `relayAddrs` entry, and that failure belongs on the same
|
|
680
|
+
// logged-and-cleaned-up path as every other one in start().
|
|
681
|
+
this.warnIfAnnounceAddrsDiscardRelay();
|
|
682
|
+
|
|
612
683
|
// Install the WebRTC TURN-relay tracker BEFORE any libp2p bring-up, so it
|
|
613
684
|
// wraps globalThis.RTCPeerConnection before the control node can create one.
|
|
614
685
|
// Inert on Node.js (no RTCPeerConnection); disposed in cleanup().
|
|
@@ -631,6 +702,25 @@ export class CadreNode implements SAppIdLookup {
|
|
|
631
702
|
// loaded before the first reconcile pass could consult them.
|
|
632
703
|
this.initializeBootstrapPeerStore();
|
|
633
704
|
|
|
705
|
+
// Arm the connection gate's BRING-UP QUIET PERIOD before the libp2p node
|
|
706
|
+
// exists, so no connection can form ahead of it: `@libp2p/bootstrap` emits
|
|
707
|
+
// its discovery events ~1 s after `libp2p.start()` and the connection
|
|
708
|
+
// manager auto-dials from there, which is INSIDE createControlNode below.
|
|
709
|
+
// The invariant being protected is "zero control connections during
|
|
710
|
+
// control-database bring-up" — see the field's doc and
|
|
711
|
+
// `membership-connection-gater.ts` → "The bring-up quiet period".
|
|
712
|
+
//
|
|
713
|
+
// NOTE: without this gate the bootstrap arm is a RACE, not a guarantee — its
|
|
714
|
+
// safety margin is bring-up duration versus @libp2p/bootstrap's 1 s discovery
|
|
715
|
+
// fuse, and bring-up duration is (raw-storage operations) × per-operation
|
|
716
|
+
// latency. The cold-start operation count and the 50-90 ms/op figure a loaded
|
|
717
|
+
// phone actually sees live in `control-database.ts`'s `loadSchema` note (and
|
|
718
|
+
// are pinned by `control-start-storage-op-budget.spec.ts`): at the top of that
|
|
719
|
+
// range bring-up is 9-15 s, far past the fuse. So if this gate is ever removed
|
|
720
|
+
// or bypassed, that arm reopens for exactly the nodes most likely to hit it —
|
|
721
|
+
// phones joining a party, whose membership row has not replicated yet.
|
|
722
|
+
this.controlBringUpInFlight = true;
|
|
723
|
+
|
|
634
724
|
// Create the control network libp2p node
|
|
635
725
|
let t0 = performance.now();
|
|
636
726
|
this.controlNode = await this.createControlNode();
|
|
@@ -653,6 +743,21 @@ export class CadreNode implements SAppIdLookup {
|
|
|
653
743
|
});
|
|
654
744
|
await this.controlDatabase.initialize();
|
|
655
745
|
timing('[start] controlDatabase.initialize: %dms', Math.round(performance.now() - t0));
|
|
746
|
+
// The database exists locally now, so the quiet period has done its job and
|
|
747
|
+
// this node can be in the conversation. A failure above never reaches here —
|
|
748
|
+
// `cleanup()` on the catch path opens the gate instead, so teardown (and a
|
|
749
|
+
// retry by the embedder) is not gated.
|
|
750
|
+
//
|
|
751
|
+
// NOTE: the window ends HERE, not at the end of start(), so the awaited control-DB
|
|
752
|
+
// work that follows — `strandWatcher.start()`'s first poll — can run against a
|
|
753
|
+
// cohort a connection just joined. Fine now: every block that poll reads was
|
|
754
|
+
// created locally by the initialize() above, so a refusing sibling has nothing to
|
|
755
|
+
// refuse. If a boot ever fails with `BlockUnavailableError` from BELOW this line,
|
|
756
|
+
// that assumption broke. Do not simply move this clear later without measuring:
|
|
757
|
+
// `handleStrandAdded` runs inside `strandWatcher.start()` and seeds each strand
|
|
758
|
+
// node from CONNECTED siblings (`resolveCohortSeed`), so a longer window trades a
|
|
759
|
+
// bring-up hazard for boot-time strand seeds that are always empty.
|
|
760
|
+
this.controlBringUpInFlight = false;
|
|
656
761
|
log('Control database initialized');
|
|
657
762
|
|
|
658
763
|
// Attach the per-stream gate to the control DB's membership hub, so every
|
|
@@ -736,12 +841,28 @@ export class CadreNode implements SAppIdLookup {
|
|
|
736
841
|
// start and admits.
|
|
737
842
|
void this.refreshMembershipGate('start');
|
|
738
843
|
|
|
844
|
+
// Reserve the configured relays LAST, so every step above ran against a
|
|
845
|
+
// cohort of one (see driveControlRelayReservation). Ahead of
|
|
846
|
+
// scheduleSelfRegistration, so this node's first published `CadrePeer` row
|
|
847
|
+
// already carries the `/p2p-circuit` address the reservation earns it.
|
|
848
|
+
// Throws on a relay that will not have us — start() is fail-fast for a node
|
|
849
|
+
// whose operator named a relay.
|
|
850
|
+
await this.driveControlRelayReservation();
|
|
851
|
+
|
|
739
852
|
// Schedule self-registration in background
|
|
740
853
|
this.scheduleSelfRegistration();
|
|
741
854
|
|
|
742
855
|
} catch (error) {
|
|
743
856
|
log('Failed to start CadreNode: %o', error);
|
|
857
|
+
// `driveControlRelayReservation` throws AFTER `control:connected` was emitted,
|
|
858
|
+
// so a subscriber can have seen this node come up. Balance the edge rather than
|
|
859
|
+
// leaving it dangling — `cleanup()` has torn the node down either way, and the
|
|
860
|
+
// events are what an embedder drives its "am I connected" UI from.
|
|
861
|
+
const wasConnected = this._running;
|
|
744
862
|
await this.cleanup();
|
|
863
|
+
if (wasConnected) {
|
|
864
|
+
this.emit('control:disconnected', undefined);
|
|
865
|
+
}
|
|
745
866
|
throw error;
|
|
746
867
|
}
|
|
747
868
|
}
|
|
@@ -782,19 +903,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
782
903
|
}
|
|
783
904
|
|
|
784
905
|
log('Stopping CadreNode');
|
|
785
|
-
// Stop the retry loop BEFORE tearing the node down, not after: `cleanup()`
|
|
786
|
-
// stops the control node partway through, and a tick that fires during it
|
|
787
|
-
// would dial and then poll a half-torn-down node for the whole reserve
|
|
788
|
-
// timeout — logging spurious dial failures and holding the process open.
|
|
789
|
-
this.relayReserveSupervisor?.stop();
|
|
790
|
-
this.relayReserveSupervisor = null;
|
|
791
906
|
await this.cleanup();
|
|
792
|
-
// Drop the rest of the relay-reservation posture: a stopped node holds no
|
|
793
|
-
// reservation, so a restarted instance must not report the previous run's
|
|
794
|
-
// `reserved`.
|
|
795
|
-
this.relayReserveAddrs = [];
|
|
796
|
-
this.relayReserveError = null;
|
|
797
|
-
this._running = false;
|
|
798
907
|
this.emit('control:disconnected', undefined);
|
|
799
908
|
log('CadreNode stopped');
|
|
800
909
|
}
|
|
@@ -989,6 +1098,76 @@ export class CadreNode implements SAppIdLookup {
|
|
|
989
1098
|
return await createLibp2pNode(this.buildControlNodeOptions());
|
|
990
1099
|
}
|
|
991
1100
|
|
|
1101
|
+
/**
|
|
1102
|
+
* The control database's cache-wrapped raw storage, resolved from
|
|
1103
|
+
* `config.storage.provider` on first use and then held in {@link controlStorage}
|
|
1104
|
+
* for the rest of this runtime.
|
|
1105
|
+
*
|
|
1106
|
+
* Resolving once is the contract `RawStorageProvider` (types.ts) states: a provider is
|
|
1107
|
+
* called once per scope per runtime lifetime. Re-entering it per call would mint a
|
|
1108
|
+
* second store over one backend — a fresh, cold cache, with the previous wrapper's
|
|
1109
|
+
* registration orphaned in the process-wide pool.
|
|
1110
|
+
*
|
|
1111
|
+
* Lazy rather than eager in `start()` so the pure-unit call path
|
|
1112
|
+
* (`cadre-node-control-node-options.spec.ts` calls `buildControlNodeOptions` on a
|
|
1113
|
+
* bare `new CadreNode`) still resolves — and, on that path too, only once.
|
|
1114
|
+
*
|
|
1115
|
+
* Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's
|
|
1116
|
+
* `cached-storage.ts`) because a control start's cost is its raw-storage
|
|
1117
|
+
* operation count.
|
|
1118
|
+
*/
|
|
1119
|
+
private resolveControlStorage(): IRawStorage | undefined {
|
|
1120
|
+
if (this.controlStorage) {
|
|
1121
|
+
return this.controlStorage;
|
|
1122
|
+
}
|
|
1123
|
+
const provider = this.config.storage?.provider;
|
|
1124
|
+
if (!provider) {
|
|
1125
|
+
return undefined;
|
|
1126
|
+
}
|
|
1127
|
+
const resolved = typeof provider === 'function' ? provider('control') : provider;
|
|
1128
|
+
this.controlStorage = wrapStorageWithCache(resolved, 'control');
|
|
1129
|
+
return this.controlStorage;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
/**
|
|
1133
|
+
* Warn the operator when `network.announceAddrs` will silently cost this node the
|
|
1134
|
+
* relay reachability it also configured.
|
|
1135
|
+
*
|
|
1136
|
+
* A non-empty announce set REPLACES everything libp2p advertises, so the
|
|
1137
|
+
* `/p2p-circuit` address earned by a relay reservation is dropped from the node's
|
|
1138
|
+
* advertised addresses even though the reservation itself is still held — peers
|
|
1139
|
+
* behind NAT stop being able to reach it through that relay. Not an error: an
|
|
1140
|
+
* operator whose relay slot is decorative may genuinely want only the announced
|
|
1141
|
+
* address, so this reports and proceeds.
|
|
1142
|
+
*
|
|
1143
|
+
* Keyed off the RESOLVED listen addrs rather than `relayAddrs` alone, so a
|
|
1144
|
+
* hand-written `/p2p-circuit` entry in `listenAddrs` — the same reservation by the
|
|
1145
|
+
* longer route — is caught too.
|
|
1146
|
+
*
|
|
1147
|
+
* NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
1148
|
+
* diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
1149
|
+
* second such warning ever appears here, route both through a `CadreNodeEvents` entry the
|
|
1150
|
+
* embedder surfaces instead of growing a console surface inside a library.
|
|
1151
|
+
*/
|
|
1152
|
+
private warnIfAnnounceAddrsDiscardRelay(): void {
|
|
1153
|
+
const network = this.config.network;
|
|
1154
|
+
if (!replacesAdvertisedAddrs(network)) {
|
|
1155
|
+
return;
|
|
1156
|
+
}
|
|
1157
|
+
const listensOnCircuit = (resolveListenAddrs(network) ?? [])
|
|
1158
|
+
.some((addr) => addr.includes('/p2p-circuit'));
|
|
1159
|
+
if (!listensOnCircuit) {
|
|
1160
|
+
return;
|
|
1161
|
+
}
|
|
1162
|
+
console.warn(
|
|
1163
|
+
'network.announceAddrs is set alongside a circuit-relay listener (from network.relayAddrs, ' +
|
|
1164
|
+
'or a /p2p-circuit entry in network.listenAddrs). A non-empty announce set REPLACES every ' +
|
|
1165
|
+
'address this node advertises, so the /p2p-circuit address earned from the relay reservation ' +
|
|
1166
|
+
'will not be advertised and peers will stop reaching this node through that relay. ' +
|
|
1167
|
+
'Use network.appendAnnounceAddrs instead to advertise an extra address without discarding the rest.'
|
|
1168
|
+
);
|
|
1169
|
+
}
|
|
1170
|
+
|
|
992
1171
|
/**
|
|
993
1172
|
* Map this node's config onto the control network's libp2p node options.
|
|
994
1173
|
*
|
|
@@ -998,21 +1177,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
998
1177
|
* only caller in production is `createControlNode`.
|
|
999
1178
|
*/
|
|
1000
1179
|
private buildControlNodeOptions(): Parameters<typeof createLibp2pNode>[0] {
|
|
1001
|
-
const { controlNetwork, network,
|
|
1180
|
+
const { controlNetwork, network, profile } = this.config;
|
|
1002
1181
|
const identityKey = this.identityKey;
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
//
|
|
1006
|
-
//
|
|
1007
|
-
const
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
: storage.provider)
|
|
1015
|
-
: undefined;
|
|
1182
|
+
// `'search'`: `network.relayAddrs` contributes the bare `/p2p-circuit` SEARCH
|
|
1183
|
+
// entry, which opens no connection — the reservation is driven explicitly at
|
|
1184
|
+
// the END of `start()`, so the control database is built while this node holds
|
|
1185
|
+
// zero control connections. See `relay-addrs.ts` and `start()`.
|
|
1186
|
+
const listenAddrs = resolveListenAddrs(network, 'search');
|
|
1187
|
+
|
|
1188
|
+
const enableRelay = this.relayServerEnabled();
|
|
1189
|
+
|
|
1190
|
+
// The control node's own storage, resolved once per runtime (see
|
|
1191
|
+
// {@link resolveControlStorage}) rather than per call.
|
|
1192
|
+
const controlStorageProvider = this.resolveControlStorage();
|
|
1016
1193
|
|
|
1017
1194
|
const nodeOptions: Parameters<typeof createLibp2pNode>[0] = {
|
|
1018
1195
|
port: 0,
|
|
@@ -1032,16 +1209,26 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1032
1209
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
1033
1210
|
...(identityKey && { privateKey: identityKey }),
|
|
1034
1211
|
...(network?.transports && { transports: network.transports }),
|
|
1035
|
-
// Configured `listenAddrs`, plus
|
|
1036
|
-
// `network.relayAddrs`
|
|
1212
|
+
// Configured `listenAddrs`, plus the bare `/p2p-circuit` search listener when
|
|
1213
|
+
// `network.relayAddrs` is set — that listener registers the pending reservation
|
|
1214
|
+
// `driveControlRelayReservation` fills after bring-up.
|
|
1037
1215
|
...(listenAddrs && { listenAddrs }),
|
|
1216
|
+
// What this node ADVERTISES, which is not the same question as what it binds.
|
|
1217
|
+
// Either field is present only when configured non-empty; a non-empty
|
|
1218
|
+
// `announceAddrs` replaces the advertised set (see warnIfAnnounceAddrsDiscardRelay).
|
|
1219
|
+
...resolveAnnounceAddrs(network),
|
|
1038
1220
|
// The CONTROL node composes the membership admission gate onto any
|
|
1039
|
-
// caller-supplied gater (deny from either wins on inbound
|
|
1040
|
-
// hooks pass through). Strand
|
|
1041
|
-
//
|
|
1221
|
+
// caller-supplied gater (deny from either wins on the inbound-encrypted
|
|
1222
|
+
// and relay-reservation hooks; all other hooks pass through). Strand
|
|
1223
|
+
// cohort nodes keep the raw configured gater (see
|
|
1224
|
+
// strand-instance-manager.ts) — their peers are legitimately
|
|
1042
1225
|
// cross-party, so cadre membership must not gate them.
|
|
1043
1226
|
connectionGater: createMembershipConnectionGater(
|
|
1044
|
-
{
|
|
1227
|
+
{
|
|
1228
|
+
admitInbound: (remotePeerId) => this.admitInboundControlConnection(remotePeerId),
|
|
1229
|
+
admitRelayReservation: (remotePeerId) => this.admitControlRelayReservation(remotePeerId),
|
|
1230
|
+
bringUpInFlight: () => this.controlBringUpInFlight
|
|
1231
|
+
},
|
|
1045
1232
|
network?.connectionGater
|
|
1046
1233
|
),
|
|
1047
1234
|
// Fail-closed per-stream authorization for the four Optimystic control-DB
|
|
@@ -1055,6 +1242,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1055
1242
|
return nodeOptions;
|
|
1056
1243
|
}
|
|
1057
1244
|
|
|
1245
|
+
/**
|
|
1246
|
+
* Does this node's CONTROL libp2p run the circuit-relay server? An explicit
|
|
1247
|
+
* `network.enableRelay` wins; otherwise storage-profile nodes default to on
|
|
1248
|
+
* (better connectivity/uptime). Shared by {@link buildControlNodeOptions}
|
|
1249
|
+
* (which configures the server from it) and
|
|
1250
|
+
* {@link admitInboundControlConnection} (whose deny/admit-for-relay branch
|
|
1251
|
+
* must agree with whether a reservation is even servable here).
|
|
1252
|
+
*/
|
|
1253
|
+
private relayServerEnabled(): boolean {
|
|
1254
|
+
return this.config.network?.enableRelay ?? (this.config.profile === 'storage');
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1058
1257
|
/**
|
|
1059
1258
|
* Decide whether an inbound CONTROL-network connection from `remotePeerId`
|
|
1060
1259
|
* should be admitted — the policy behind the connection gater wired in
|
|
@@ -1064,7 +1263,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1064
1263
|
* `membership-connection-gater.ts` for the layer's rationale and the
|
|
1065
1264
|
* stranger-open protocol allowlist).
|
|
1066
1265
|
*
|
|
1067
|
-
*
|
|
1266
|
+
* Returns `'admit'` when ANY of:
|
|
1068
1267
|
* 1. a shared-baseline check admits ({@link admitControlPeerUnconditionally}
|
|
1069
1268
|
* — not running / DB torn down, absent-or-empty trusted-owner anchor, or
|
|
1070
1269
|
* configured bootstrap/relay infrastructure);
|
|
@@ -1101,43 +1300,119 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1101
1300
|
* this node yet is denied even though the formation handler would have
|
|
1102
1301
|
* accepted it, exactly like the unreplicated-membership-row case below.
|
|
1103
1302
|
*
|
|
1303
|
+
* When every check falls through, the verdict depends on whether this node
|
|
1304
|
+
* runs the circuit-relay server ({@link relayServerEnabled}): without one,
|
|
1305
|
+
* `'deny'`; with one, `'admit-for-relay'` — a circuit-relay reservation is
|
|
1306
|
+
* established by the reserving peer DIALING the relay, so a connection deny
|
|
1307
|
+
* here kills the reservation, and that deny is NOT self-healing: an outbound
|
|
1308
|
+
* reconcile re-dial re-establishes a data link, but no outbound dial can
|
|
1309
|
+
* grant the remote peer a reservation, and a relay-only peer has no address
|
|
1310
|
+
* of its own to dial back — the reservation IS its address. The gater admits
|
|
1311
|
+
* such a connection, decides the reservation via
|
|
1312
|
+
* {@link admitControlRelayReservation}, and drops the connection if no
|
|
1313
|
+
* reservation is admitted in time (see `membership-connection-gater.ts` →
|
|
1314
|
+
* "The relay-reservation seam").
|
|
1315
|
+
*
|
|
1104
1316
|
* NOTE: check 4/5 runs a control-DB read per inbound connection
|
|
1105
1317
|
* (`listAuthorizedMembers`); connections are rare and cadres small, and this
|
|
1106
1318
|
* layer is fail-open behind `ADMISSION_DECISION_TIMEOUT_MS`, so the live
|
|
1107
1319
|
* read is safe here — unlike the per-stream gate, which must consult the
|
|
1108
1320
|
* materialized {@link authorizedControlPeers} snapshot instead.
|
|
1109
|
-
* NOTE: a sibling whose membership row has not yet
|
|
1110
|
-
*
|
|
1111
|
-
* either side's next outbound reconcile dial (outbound is never
|
|
1112
|
-
* re-establishes the link — self-healing,
|
|
1113
|
-
|
|
1114
|
-
|
|
1321
|
+
* NOTE: on a relay-DISABLED node, a sibling whose membership row has not yet
|
|
1322
|
+
* replicated here is denied until the row converges (typically via the
|
|
1323
|
+
* owner); either side's next outbound reconcile dial (outbound is never
|
|
1324
|
+
* gated) re-establishes the DATA link — self-healing for data, visible as a
|
|
1325
|
+
* transient deny. That self-healing story never covered a reservation, which
|
|
1326
|
+
* is exactly why the relay-enabled path above exists.
|
|
1327
|
+
*/
|
|
1328
|
+
private async admitInboundControlConnection(remotePeerId: string): Promise<InboundConnectionVerdict> {
|
|
1115
1329
|
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1116
|
-
return
|
|
1330
|
+
return 'admit';
|
|
1117
1331
|
}
|
|
1118
1332
|
if (Date.now() < this.enrollmentWindowUntil) {
|
|
1119
|
-
return
|
|
1333
|
+
return 'admit';
|
|
1120
1334
|
}
|
|
1121
1335
|
if (this.delegateAdmission.has(remotePeerId)) {
|
|
1122
|
-
return
|
|
1336
|
+
return 'admit';
|
|
1123
1337
|
}
|
|
1124
1338
|
const authorized = await this.listAuthorizedMembers();
|
|
1125
1339
|
if (authorized.length === 0) {
|
|
1126
|
-
return
|
|
1340
|
+
return 'admit';
|
|
1127
1341
|
}
|
|
1128
1342
|
if (authorized.some((m) => m.peerId === remotePeerId)) {
|
|
1129
|
-
return
|
|
1343
|
+
return 'admit';
|
|
1130
1344
|
}
|
|
1131
1345
|
try {
|
|
1132
1346
|
if (await this.strandSolicitationService?.hasOutstandingInvitation()) {
|
|
1133
|
-
return
|
|
1347
|
+
return 'admit';
|
|
1134
1348
|
}
|
|
1135
1349
|
} catch (error) {
|
|
1136
1350
|
log('admitInboundControlConnection: outstanding-invitation check threw for %s — admitting (fail-open): %o', remotePeerId, error);
|
|
1137
|
-
return
|
|
1351
|
+
return 'admit';
|
|
1352
|
+
}
|
|
1353
|
+
if (this.relayServerEnabled()) {
|
|
1354
|
+
log('admitInboundControlConnection: admitting %s FOR RELAY ONLY — not an authorized member and no enrollment path open; the gater drops the connection unless a reservation is admitted', remotePeerId);
|
|
1355
|
+
return 'admit-for-relay';
|
|
1138
1356
|
}
|
|
1139
1357
|
log('admitInboundControlConnection: DENYING inbound from %s — not an authorized member and no enrollment path open', remotePeerId);
|
|
1140
|
-
return
|
|
1358
|
+
return 'deny';
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
/**
|
|
1362
|
+
* Should `remotePeerId` be granted a circuit-relay reservation slot on this
|
|
1363
|
+
* node's relay server? The policy behind the gater's
|
|
1364
|
+
* `denyInboundRelayReservation` hook (see `membership-connection-gater.ts` →
|
|
1365
|
+
* "The relay-reservation seam"); the circuit-relay server consults it per
|
|
1366
|
+
* RESERVE request, so it is never called on a node whose relay server is off.
|
|
1367
|
+
*
|
|
1368
|
+
* Admits when ANY of:
|
|
1369
|
+
* 1. a shared-baseline check admits ({@link admitControlPeerUnconditionally}
|
|
1370
|
+
* — not running / DB torn down, absent-or-empty trusted-owner anchor, or
|
|
1371
|
+
* configured bootstrap/relay infrastructure);
|
|
1372
|
+
* 2. the peer holds a live delegate admission grant — a member's strand
|
|
1373
|
+
* node reserving here (see `delegate-admission.ts`); never counted
|
|
1374
|
+
* against the unauthorized budget;
|
|
1375
|
+
* 3. the authorized-member set is empty (cold start) or the peer IS an
|
|
1376
|
+
* authorized member — members are never counted against the budget; or
|
|
1377
|
+
* 4. the peer takes (or already holds) a slot in the bounded
|
|
1378
|
+
* unauthorized-reservation budget — the boot-ordering window where a
|
|
1379
|
+
* genuine member reserves before its `CadrePeer` row replicates here.
|
|
1380
|
+
* Cap via `network.unauthorizedRelayReservationCap` (0 = refuse every
|
|
1381
|
+
* unauthorized reservation).
|
|
1382
|
+
*
|
|
1383
|
+
* Every uncounted admission also RELEASES the budget slot the peer may hold
|
|
1384
|
+
* from an earlier reservation taken while it was still unplaceable — a member
|
|
1385
|
+
* that boots, reserves, and only then has its row replicate here would
|
|
1386
|
+
* otherwise keep that slot spent for the rest of the entry's TTL.
|
|
1387
|
+
*
|
|
1388
|
+
* The connection gate's stranger carve-outs (an open enrollment window, an
|
|
1389
|
+
* outstanding formation invitation) deliberately do NOT extend here: they
|
|
1390
|
+
* exist so a stranger's SEED or FORMATION stream can ride a connection, and
|
|
1391
|
+
* neither needs relay capacity. A genuine invitee that does need a relay slot
|
|
1392
|
+
* takes one from the budget like any other unplaced peer, so an open window
|
|
1393
|
+
* never becomes an unbounded grant of this node's forwarding capacity.
|
|
1394
|
+
*/
|
|
1395
|
+
private async admitControlRelayReservation(remotePeerId: string): Promise<boolean> {
|
|
1396
|
+
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1397
|
+
return this.admitReservationUncounted(remotePeerId);
|
|
1398
|
+
}
|
|
1399
|
+
if (this.delegateAdmission.has(remotePeerId)) {
|
|
1400
|
+
return this.admitReservationUncounted(remotePeerId);
|
|
1401
|
+
}
|
|
1402
|
+
const authorized = await this.listAuthorizedMembers();
|
|
1403
|
+
if (authorized.length === 0 || authorized.some((m) => m.peerId === remotePeerId)) {
|
|
1404
|
+
return this.admitReservationUncounted(remotePeerId);
|
|
1405
|
+
}
|
|
1406
|
+
const admitted = this.unauthorizedRelayReservations.tryAdmit(remotePeerId);
|
|
1407
|
+
log('admitControlRelayReservation: %s %s under the unauthorized-reservation budget',
|
|
1408
|
+
admitted ? 'admitting' : 'REFUSING', remotePeerId);
|
|
1409
|
+
return admitted;
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
/** Admit a reservation on the peer's own merits, giving back any budget slot it still holds. */
|
|
1413
|
+
private admitReservationUncounted(remotePeerId: string): true {
|
|
1414
|
+
this.unauthorizedRelayReservations.release(remotePeerId);
|
|
1415
|
+
return true;
|
|
1141
1416
|
}
|
|
1142
1417
|
|
|
1143
1418
|
/**
|
|
@@ -1155,7 +1430,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1155
1430
|
/**
|
|
1156
1431
|
* Record (or refresh) a delegate admission grant: `delegatePeerId` is the
|
|
1157
1432
|
* transport peerId of `announcerPeerId`'s strand-`strandId` node, admitted
|
|
1158
|
-
* at the CONNECTION
|
|
1433
|
+
* at the CONNECTION and RESERVATION levels (all a circuit-relay reservation
|
|
1434
|
+
* needs, without spending the unauthorized-reservation budget) for
|
|
1159
1435
|
* `DELEGATE_GRANT_TTL_MS`. Called by the strand-addr responder after its
|
|
1160
1436
|
* authorized-membership gate has passed; public so tests can drive the
|
|
1161
1437
|
* admission policy without a full strand launch. A re-announce for the same
|
|
@@ -1263,13 +1539,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1263
1539
|
*
|
|
1264
1540
|
* The SOLE caller is {@link drainMembershipGate}, which serializes refreshes —
|
|
1265
1541
|
* so two reads can no longer settle out of order.
|
|
1542
|
+
*
|
|
1543
|
+
* Reads WITHOUT the transient-failure retry, deliberately: `ControlDatabase` drives
|
|
1544
|
+
* this listener with its write lock HELD (`notifyMembershipChanged`), so a retrying
|
|
1545
|
+
* read here would sleep its backoff holding the lock and stall every other local
|
|
1546
|
+
* writer. Nothing is lost — this refresh already keeps the previous snapshot on
|
|
1547
|
+
* failure and is re-driven by the next membership write and by the timed reconcile.
|
|
1266
1548
|
*/
|
|
1267
1549
|
private async refreshAuthorizedControlPeers(reason: string): Promise<void> {
|
|
1268
1550
|
if (!this._running || !this.controlDatabase) {
|
|
1269
1551
|
return;
|
|
1270
1552
|
}
|
|
1271
1553
|
try {
|
|
1272
|
-
const members = await this.listAuthorizedMembers();
|
|
1554
|
+
const members = await this.listAuthorizedMembers(false);
|
|
1273
1555
|
this.authorizedControlPeers = new Set(members.map((m) => m.peerId));
|
|
1274
1556
|
log('refreshAuthorizedControlPeers(%s): %d authorized peer(s)', reason, this.authorizedControlPeers.size);
|
|
1275
1557
|
} catch (error) {
|
|
@@ -1407,7 +1689,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1407
1689
|
|
|
1408
1690
|
if (!existing) {
|
|
1409
1691
|
if (!this.seedBootstrapService) {
|
|
1410
|
-
|
|
1692
|
+
// "No row" also covers "revoked": queryPeerRecord reads a row whose StampId is
|
|
1693
|
+
// retired in Revocation as absent, so a removed node lands here every heartbeat
|
|
1694
|
+
// and stops refreshing its own record — correct, but say so, or the message reads
|
|
1695
|
+
// as "never added" for a node that was removed.
|
|
1696
|
+
log('registerSelf: no readable CadrePeer row for self (never added, or removed and its stamp retired) and no owner service to self-insert; skipping');
|
|
1411
1697
|
return 'skipped';
|
|
1412
1698
|
}
|
|
1413
1699
|
// First-time row: requires an owner signature (the node is its own
|
|
@@ -1429,9 +1715,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1429
1715
|
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
1430
1716
|
}
|
|
1431
1717
|
|
|
1432
|
-
//
|
|
1718
|
+
// Reached on the fall-through above, where the pre-race read missed the row a
|
|
1433
1719
|
// concurrent authorize then seated — so re-read to get that row's `UpdatedAt`, which
|
|
1434
|
-
// the strictly-greater self-update rule is measured against.
|
|
1720
|
+
// the strictly-greater self-update rule is measured against. The re-read can still
|
|
1721
|
+
// come back null (see the branch below).
|
|
1435
1722
|
//
|
|
1436
1723
|
// NOTE: when `existing` is set it predates this publish, so a removePeer(self) racing
|
|
1437
1724
|
// in would leave the UPDATE below matching no rows while still reporting 'refreshed'.
|
|
@@ -1439,7 +1726,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1439
1726
|
// if removing self ever becomes something that happens concurrently in practice.
|
|
1440
1727
|
const current = existing ?? await this.controlDatabase.queryPeerRecord(peerId);
|
|
1441
1728
|
if (!current) {
|
|
1442
|
-
|
|
1729
|
+
// Two causes, and the second is a steady state rather than a race: the row really
|
|
1730
|
+
// vanished mid-publish (next refresh re-inserts), or this node was REVOKED — its row
|
|
1731
|
+
// is physically present, so insertCadrePeer's in-lock existence check no-op'd above,
|
|
1732
|
+
// but queryPeerRecord reads it as absent because its stamp is retired. The revoked
|
|
1733
|
+
// case repeats every heartbeat and never re-inserts; that is intended (a revoked node
|
|
1734
|
+
// has nothing to publish), so this stays a skip, not an error.
|
|
1735
|
+
log('registerSelf: own CadrePeer row not readable (vanished mid-publish, or this node is revoked and its stamp retired); skipping');
|
|
1443
1736
|
return 'skipped';
|
|
1444
1737
|
}
|
|
1445
1738
|
const record = this.signSelfRecord(peerId, signingKey, addrs, current);
|
|
@@ -1562,6 +1855,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1562
1855
|
};
|
|
1563
1856
|
this.controlNode.addEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
1564
1857
|
|
|
1858
|
+
// NOTE: a node AUTHORIZED AFTER IT BOOTED can wait a whole heartbeat (7.5 min)
|
|
1859
|
+
// to publish its address. `registerSelf` no-ops while a node has no readable
|
|
1860
|
+
// `CadrePeer` row, so its boot-time publish does nothing, and the two triggers
|
|
1861
|
+
// wired here are the only ones after that: `self:peer:update` needs an ADDRESS
|
|
1862
|
+
// CHANGE, and a relay-only node's addresses stop changing the moment its
|
|
1863
|
+
// reservation lands — inside `start()`, before this listener exists. Harmless
|
|
1864
|
+
// where a late-authorized node's addresses keep churning (NAT/relay rotation
|
|
1865
|
+
// fire the event anyway), and invisible while parties authorize members before
|
|
1866
|
+
// they boot. If late enrollment becomes the normal path — an invited phone is
|
|
1867
|
+
// exactly that — republish on the membership-change seam that already exists
|
|
1868
|
+
// (`ControlDatabase.setMembershipChangeListener`) rather than lengthening this
|
|
1869
|
+
// list of triggers. Measured while writing
|
|
1870
|
+
// `relay-only-control-addr.integration.ts` case 4, which drives `registerSelf()`
|
|
1871
|
+
// explicitly because neither trigger is dependable there.
|
|
1565
1872
|
this.recordRefreshTimer = setInterval(() => republish('heartbeat'), DEFAULT_PEER_RECORD_HEARTBEAT_MS);
|
|
1566
1873
|
(this.recordRefreshTimer as { unref?: () => void } | null)?.unref?.();
|
|
1567
1874
|
|
|
@@ -1642,6 +1949,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1642
1949
|
* Survivors are returned signaling (`/p2p-circuit`) first, filtered to
|
|
1643
1950
|
* signaling-only when requested, as parsed `Multiaddr`s (unparsable addrs
|
|
1644
1951
|
* dropped). Any gate failure yields an empty array rather than throwing.
|
|
1952
|
+
*
|
|
1953
|
+
* Every returned address is normalized to terminate in `/p2p/<peerId>` — see
|
|
1954
|
+
* {@link normalizeDialAddrs} for why that invariant, not each dial site's own
|
|
1955
|
+
* handling, is what keeps a mixed list from taking a whole peer offline.
|
|
1645
1956
|
*/
|
|
1646
1957
|
async resolvePeerAddrs(peerId: string, opts: ResolveOpts = {}): Promise<Multiaddr[]> {
|
|
1647
1958
|
if (!this.controlDatabase) {
|
|
@@ -1694,12 +2005,72 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1694
2005
|
|
|
1695
2006
|
// Order signaling-first (the on-record order was what we verified above),
|
|
1696
2007
|
// optionally restrict to signaling addrs, then parse — dropping any addr
|
|
1697
|
-
// that does not parse as a multiaddr
|
|
2008
|
+
// that does not parse as a multiaddr — and finally normalize every survivor
|
|
2009
|
+
// onto `/p2p/<peerId>` so no caller ever sees a mixed list.
|
|
1698
2010
|
let addrs = orderSignalingFirst(record.addrs);
|
|
1699
2011
|
if (opts.signalingOnly) {
|
|
1700
2012
|
addrs = addrs.filter(isSignalingAddr);
|
|
1701
2013
|
}
|
|
1702
|
-
return this.parseMultiaddrs(addrs);
|
|
2014
|
+
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
2015
|
+
}
|
|
2016
|
+
|
|
2017
|
+
/**
|
|
2018
|
+
* Guarantee every address handed to a control-network dial terminates in
|
|
2019
|
+
* `/p2p/<peerId>`, with no duplicates — the invariant every dial path
|
|
2020
|
+
* downstream relies on.
|
|
2021
|
+
*
|
|
2022
|
+
* `libp2p.dial(addrs)` requires that the addresses in ONE dial either all name
|
|
2023
|
+
* a peer id or none do; a list mixing a circuit address (which ends in
|
|
2024
|
+
* `/p2p/<target>`) with a bare direct one (which ends in `/tcp/…`) makes that
|
|
2025
|
+
* call throw, and the whole peer is then silently skipped rather than one bad
|
|
2026
|
+
* address.
|
|
2027
|
+
*
|
|
2028
|
+
* Applied by ALL THREE sources of control-dial candidates, so no call site
|
|
2029
|
+
* invents its own rule and none can produce a mixed list: the signed record
|
|
2030
|
+
* ({@link resolvePeerAddrs}), the libp2p address book
|
|
2031
|
+
* ({@link peerStoreAddrs}), and a seed's retained owner addresses
|
|
2032
|
+
* ({@link bootstrapDialAddrs}).
|
|
2033
|
+
*
|
|
2034
|
+
* An address naming a DIFFERENT trailing peer id is dropped: it does not reach
|
|
2035
|
+
* `peerId`, so it does not belong in this peer's candidate list (libp2p's own
|
|
2036
|
+
* dial queue filters the same shape out one layer lower).
|
|
2037
|
+
*/
|
|
2038
|
+
private normalizeDialAddrs(addrs: Multiaddr[], peerId: string): Multiaddr[] {
|
|
2039
|
+
// Keyed by the NORMALIZED string, so two entries differing only by the suffix
|
|
2040
|
+
// (`/ip4/…/tcp/4001` and `/ip4/…/tcp/4001/p2p/<peerId>`) collapse into one.
|
|
2041
|
+
// Normalization is what makes them equal; a surviving duplicate would cost a
|
|
2042
|
+
// real dial attempt — a whole slice of `dialWake`'s budget — on an address
|
|
2043
|
+
// already being tried.
|
|
2044
|
+
const out = new Map<string, Multiaddr>();
|
|
2045
|
+
for (const addr of addrs) {
|
|
2046
|
+
const bound = this.bindAddrToPeer(addr, peerId);
|
|
2047
|
+
if (bound) {
|
|
2048
|
+
out.set(bound.toString(), bound);
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
return [...out.values()];
|
|
2052
|
+
}
|
|
2053
|
+
|
|
2054
|
+
/**
|
|
2055
|
+
* One address bound to `peerId`, or `null` when it cannot be — logged either
|
|
2056
|
+
* way, never thrown, so every caller's list-shaping stays total.
|
|
2057
|
+
*
|
|
2058
|
+
* `withTrailingPeerId` encapsulates `/p2p/<peerId>`, which throws on a peer id
|
|
2059
|
+
* that does not parse. Unreachable for a `CadrePeer` row (the binding gate
|
|
2060
|
+
* above parsed it already), reachable for a seed-supplied one
|
|
2061
|
+
* ({@link bootstrapDialAddrs}).
|
|
2062
|
+
*/
|
|
2063
|
+
private bindAddrToPeer(addr: Multiaddr, peerId: string): Multiaddr | null {
|
|
2064
|
+
try {
|
|
2065
|
+
const bound = withTrailingPeerId(addr, peerId);
|
|
2066
|
+
if (!bound) {
|
|
2067
|
+
log('normalizeDialAddrs: dropping addr %s — it names a peer other than %s', addr.toString(), peerId);
|
|
2068
|
+
}
|
|
2069
|
+
return bound;
|
|
2070
|
+
} catch (error) {
|
|
2071
|
+
log('normalizeDialAddrs: cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
2072
|
+
return null;
|
|
2073
|
+
}
|
|
1703
2074
|
}
|
|
1704
2075
|
|
|
1705
2076
|
/** Parse multiaddr strings, dropping (and logging) any that fail to parse. */
|
|
@@ -1767,8 +2138,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1767
2138
|
// than a local write) are picked up here, bounding the snapshot's staleness
|
|
1768
2139
|
// to the reconcile interval.
|
|
1769
2140
|
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
1770
|
-
// CadrePeer query (two reads per pass)
|
|
1771
|
-
//
|
|
2141
|
+
// CadrePeer query (two reads per pass), plus a third from
|
|
2142
|
+
// `refreshStrandPeerAddrs` on the passes where a strand is due AND this node
|
|
2143
|
+
// holds a control connection (one read for the whole pass, not one per
|
|
2144
|
+
// strand); if those reads ever get costly, share one row-set across all
|
|
2145
|
+
// three.
|
|
1772
2146
|
await this.refreshMembershipGate('reconcile');
|
|
1773
2147
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1774
2148
|
return;
|
|
@@ -1781,6 +2155,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1781
2155
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1782
2156
|
return;
|
|
1783
2157
|
}
|
|
2158
|
+
// Then re-warm each running strand's own address book from its siblings, on
|
|
2159
|
+
// its own (much longer) throttle. Same reasoning as warmSiblingAddrBook
|
|
2160
|
+
// below, one layer down: replication runs on the STRAND network, and every
|
|
2161
|
+
// layer under cadre-core dials strand peers by bare peer id.
|
|
2162
|
+
await this.refreshStrandPeerAddrs();
|
|
2163
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2164
|
+
return;
|
|
2165
|
+
}
|
|
1784
2166
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1785
2167
|
|
|
1786
2168
|
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
@@ -1863,10 +2245,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1863
2245
|
cappedNonOwner, targetDegree);
|
|
1864
2246
|
}
|
|
1865
2247
|
|
|
1866
|
-
// 3.
|
|
2248
|
+
// 3. Warm the libp2p address book with EVERY sibling's verified addresses,
|
|
2249
|
+
// keeping what resolved for the dial loop below.
|
|
2250
|
+
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
2251
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
2252
|
+
return;
|
|
2253
|
+
}
|
|
2254
|
+
|
|
2255
|
+
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
1867
2256
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
1868
2257
|
|
|
1869
|
-
//
|
|
2258
|
+
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
2259
|
+
// addresses step 3 already resolved for it.
|
|
1870
2260
|
let dialed = 0;
|
|
1871
2261
|
for (const sibling of dials) {
|
|
1872
2262
|
if (!this._running || !this.controlNode) {
|
|
@@ -1875,7 +2265,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1875
2265
|
if (connected.has(sibling.peerId)) {
|
|
1876
2266
|
continue;
|
|
1877
2267
|
}
|
|
1878
|
-
if (await this.dialControlSibling(sibling)) {
|
|
2268
|
+
if (await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? [])) {
|
|
1879
2269
|
dialed++;
|
|
1880
2270
|
}
|
|
1881
2271
|
}
|
|
@@ -1884,27 +2274,109 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1884
2274
|
}
|
|
1885
2275
|
|
|
1886
2276
|
/**
|
|
1887
|
-
* Resolve
|
|
1888
|
-
*
|
|
2277
|
+
* Resolve every sibling's signed address record once and merge what resolves
|
|
2278
|
+
* into the control node's libp2p **address book** (peerStore). Returns the
|
|
2279
|
+
* resolved addresses per sibling so the dial loop reuses them rather than
|
|
2280
|
+
* re-resolving — one `queryPeerRecord` per sibling per pass, not two.
|
|
2281
|
+
*
|
|
2282
|
+
* Deliberately every sibling, not the {@link selectControlCohortDials} subset,
|
|
2283
|
+
* and deliberately including already-connected ones: everything below
|
|
2284
|
+
* cadre-core dials by bare peer id (Optimystic's cluster/repo clients, FRET
|
|
2285
|
+
* ping/announce), so the address book has to be warm BEFORE a live connection
|
|
2286
|
+
* drops, and a sibling the out-degree cap declines to dial is still one those
|
|
2287
|
+
* layers may need to reach.
|
|
2288
|
+
*
|
|
2289
|
+
* Only `resolvePeerAddrs` output is merged — never the cold-start
|
|
2290
|
+
* `peerStoreAddrs` fallback, which came out of the address book to begin with
|
|
2291
|
+
* and would restamp unverified seed addresses indefinitely. A sibling that
|
|
2292
|
+
* resolves to nothing (revoked, stale, untrusted) is not written at all, so its
|
|
2293
|
+
* existing entry ages out on its own.
|
|
2294
|
+
*
|
|
2295
|
+
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
2296
|
+
* so it costs one record query per sibling per reconcile pass (~15s) and each
|
|
2297
|
+
* one delays the pass's first dial. A cadre is a handful of devices, so today
|
|
2298
|
+
* that is a few extra local reads; if cadres ever grow large, batch the records
|
|
2299
|
+
* into one query, merge only on change, or move the warm pass after the dials.
|
|
2300
|
+
*/
|
|
2301
|
+
private async warmSiblingAddrBook(siblings: CohortPeerRow[]): Promise<Map<string, Multiaddr[]>> {
|
|
2302
|
+
const resolved = new Map<string, Multiaddr[]>();
|
|
2303
|
+
const counts: Record<MergeAddrsResult, number> = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
2304
|
+
for (const sibling of siblings) {
|
|
2305
|
+
if (!this._running || !this.controlNode) {
|
|
2306
|
+
break;
|
|
2307
|
+
}
|
|
2308
|
+
const addrs = await this.resolveSiblingAddrs(sibling.peerId);
|
|
2309
|
+
resolved.set(sibling.peerId, addrs);
|
|
2310
|
+
// Re-guarded after the resolve await as well as before it: the write is
|
|
2311
|
+
// the half that matters, and a torn-down node must never be written to.
|
|
2312
|
+
const controlNode = this.controlNode;
|
|
2313
|
+
if (!this._running || !controlNode) {
|
|
2314
|
+
break;
|
|
2315
|
+
}
|
|
2316
|
+
// `mergePeerAddrs` owns the whole best-effort contract — including parsing
|
|
2317
|
+
// a malformed `CadrePeer.PeerId` — so one bad row cannot abort the pass.
|
|
2318
|
+
counts[await mergePeerAddrs(controlNode, sibling.peerId, addrs)]++;
|
|
2319
|
+
}
|
|
2320
|
+
log('reconcileControlCohort: address book warmed (siblings=%d, merged=%d, restamped=%d, skipped=%d, failed=%d)',
|
|
2321
|
+
resolved.size, counts.merged, counts.restamped, counts.skipped, counts.failed);
|
|
2322
|
+
return resolved;
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
/**
|
|
2326
|
+
* {@link resolvePeerAddrs} for one sibling, best-effort: a control-DB read
|
|
2327
|
+
* failure yields `[]` (and the cold-start fallback then gets its turn) rather
|
|
2328
|
+
* than aborting the whole pass, like every other step here.
|
|
2329
|
+
*/
|
|
2330
|
+
private async resolveSiblingAddrs(peerId: string): Promise<Multiaddr[]> {
|
|
2331
|
+
try {
|
|
2332
|
+
return await this.resolvePeerAddrs(peerId);
|
|
2333
|
+
} catch (error) {
|
|
2334
|
+
log('reconcileControlCohort: resolving addrs for %s failed (continuing): %o', peerId, error);
|
|
2335
|
+
return [];
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
2338
|
+
|
|
2339
|
+
/**
|
|
2340
|
+
* Budget for ONE proactive dial of the reconcile pass — steady-state sibling
|
|
2341
|
+
* ({@link dialControlSibling}) and cold-start bootstrap peer
|
|
2342
|
+
* ({@link dialBootstrapPeer}) alike, since both feed a whole address list to
|
|
2343
|
+
* one `dial()` and both run in a sequential loop that the next entry waits on.
|
|
1889
2344
|
*
|
|
1890
|
-
*
|
|
1891
|
-
*
|
|
1892
|
-
* exactly like {@link SeedBootstrapService.applySeed}'s owner-dial loop. A
|
|
1893
|
-
* failed dial is simply retried on the next pass.
|
|
2345
|
+
* See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why the pass bounds
|
|
2346
|
+
* this itself rather than inheriting each address's libp2p attempt timeout.
|
|
1894
2347
|
*/
|
|
1895
|
-
private
|
|
2348
|
+
private controlDialTimeoutMs(): number {
|
|
2349
|
+
return this.config.network?.controlCohort?.dialTimeoutMs
|
|
2350
|
+
?? DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS;
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
/**
|
|
2354
|
+
* Dial one sibling from its already-resolved addresses, best-effort. Returns
|
|
2355
|
+
* whether a dial was attempted (false when no address resolves).
|
|
2356
|
+
*
|
|
2357
|
+
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
2358
|
+
* the {@link controlDialTimeoutMs} budget expiring) is logged and swallowed so
|
|
2359
|
+
* one unreachable sibling never aborts the pass — exactly like
|
|
2360
|
+
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
2361
|
+
* simply retried on the next pass.
|
|
2362
|
+
*/
|
|
2363
|
+
private async dialControlSibling(sibling: CohortPeerRow, resolved: Multiaddr[]): Promise<boolean> {
|
|
1896
2364
|
const controlNode = this.controlNode;
|
|
1897
2365
|
if (!controlNode) {
|
|
1898
2366
|
return false;
|
|
1899
2367
|
}
|
|
1900
|
-
const addrs = await this.resolveControlDialAddrs(sibling.peerId);
|
|
2368
|
+
const addrs = await this.resolveControlDialAddrs(sibling.peerId, resolved);
|
|
1901
2369
|
if (addrs.length === 0) {
|
|
1902
2370
|
log('reconcileControlCohort: no dialable control address for sibling %s; skipping', sibling.peerId);
|
|
1903
2371
|
return false;
|
|
1904
2372
|
}
|
|
1905
2373
|
try {
|
|
1906
2374
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
1907
|
-
await
|
|
2375
|
+
await withDeadline(
|
|
2376
|
+
this.controlDialTimeoutMs(),
|
|
2377
|
+
`reconcileControlCohort dial of sibling ${sibling.peerId}`,
|
|
2378
|
+
(signal) => controlNode.dial(addrs, { signal })
|
|
2379
|
+
);
|
|
1908
2380
|
return true;
|
|
1909
2381
|
} catch (error) {
|
|
1910
2382
|
log('reconcileControlCohort: dial of sibling %s failed (continuing): %o', sibling.peerId, error);
|
|
@@ -1913,16 +2385,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1913
2385
|
}
|
|
1914
2386
|
|
|
1915
2387
|
/**
|
|
1916
|
-
*
|
|
2388
|
+
* A sibling's control-network dial addresses for the reconcile pass.
|
|
1917
2389
|
*
|
|
1918
|
-
* Primary (steady state): the signed, fresh, trust-gated control addresses
|
|
1919
|
-
*
|
|
1920
|
-
*
|
|
1921
|
-
*
|
|
1922
|
-
*
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
2390
|
+
* Primary (steady state): the signed, fresh, trust-gated control addresses
|
|
2391
|
+
* `resolved` for it by {@link warmSiblingAddrBook} — passed in rather than
|
|
2392
|
+
* re-resolved, so the pass makes one record query per sibling. Fallback (cold
|
|
2393
|
+
* start): the libp2p peerStore entries `applySeed` populated, used only while
|
|
2394
|
+
* the record is not yet resolvable. Returns `[]` (never throws) when neither
|
|
2395
|
+
* yields an address — that sibling is skipped this pass.
|
|
2396
|
+
*/
|
|
2397
|
+
private async resolveControlDialAddrs(peerId: string, resolved: Multiaddr[]): Promise<Multiaddr[]> {
|
|
1926
2398
|
if (resolved.length > 0) {
|
|
1927
2399
|
return resolved;
|
|
1928
2400
|
}
|
|
@@ -1933,6 +2405,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1933
2405
|
* Cold-start fallback: the libp2p peerStore multiaddrs for `peerId` (seeded by
|
|
1934
2406
|
* {@link SeedBootstrapService.applySeed}). Returns `[]` on a missing entry or any
|
|
1935
2407
|
* parse/lookup failure — never throws.
|
|
2408
|
+
*
|
|
2409
|
+
* Normalized through {@link normalizeDialAddrs} for the same reason
|
|
2410
|
+
* {@link resolvePeerAddrs} is, and with more cause: the address book hands back
|
|
2411
|
+
* a list that is inherently MIXED. `@libp2p/peer-store`'s
|
|
2412
|
+
* `dedupeFilterAndSortAddresses` strips a trailing `/p2p/<peerId>` only when
|
|
2413
|
+
* that id is the address's FIRST `/p2p/` component, so a direct address
|
|
2414
|
+
* round-trips bare while a relayed one — whose first `/p2p/` names the relay —
|
|
2415
|
+
* keeps its suffix (the same asymmetry `peer-addr-book.ts`'s `addrKey` exists
|
|
2416
|
+
* to absorb). Feeding both to one `dial()` is exactly the
|
|
2417
|
+
* `InvalidParametersError` that skipped the whole peer.
|
|
1936
2418
|
*/
|
|
1937
2419
|
private async peerStoreAddrs(peerId: string): Promise<Multiaddr[]> {
|
|
1938
2420
|
if (!this.controlNode) {
|
|
@@ -1942,7 +2424,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1942
2424
|
const peer = await this.controlNode.peerStore.get(peerIdFromString(peerId));
|
|
1943
2425
|
// Re-parse through the top-level multiaddr parser so the returned type matches
|
|
1944
2426
|
// resolvePeerAddrs (the peerStore bundles its own @multiformats/multiaddr copy).
|
|
1945
|
-
return this.
|
|
2427
|
+
return this.normalizeDialAddrs(
|
|
2428
|
+
this.parseMultiaddrs(peer.addresses.map((a) => a.multiaddr.toString())),
|
|
2429
|
+
peerId
|
|
2430
|
+
);
|
|
1946
2431
|
} catch (error) {
|
|
1947
2432
|
log('reconcileControlCohort: peerStore lookup for %s failed: %o', peerId, error);
|
|
1948
2433
|
return [];
|
|
@@ -2033,6 +2518,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2033
2518
|
if (!this._running || !this.controlNode) {
|
|
2034
2519
|
return;
|
|
2035
2520
|
}
|
|
2521
|
+
// NOTE: a connection this node still holds in `status: 'open'` after the remote
|
|
2522
|
+
// has already aborted it counts as connected here, so the cold-start retry stays
|
|
2523
|
+
// suppressed for that peer until the connection monitor's next ping notices the
|
|
2524
|
+
// abort — measured at ~9 s in `control-cohort-cold-start-retry.integration.ts`.
|
|
2525
|
+
// Bounded and self-healing today; it would start to matter if the connection
|
|
2526
|
+
// monitor's ping interval were lengthened, or if recovery ever had to meet a
|
|
2527
|
+
// deadline tighter than one ping interval.
|
|
2036
2528
|
if (connected.has(peerId)) {
|
|
2037
2529
|
continue;
|
|
2038
2530
|
}
|
|
@@ -2048,32 +2540,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2048
2540
|
* Bind a bootstrap peer's seed addresses to its peer id, so the dial
|
|
2049
2541
|
* authenticates the peer it is aiming at rather than trusting whoever answers.
|
|
2050
2542
|
*
|
|
2051
|
-
*
|
|
2052
|
-
*
|
|
2053
|
-
*
|
|
2054
|
-
*
|
|
2055
|
-
* bare
|
|
2543
|
+
* The same {@link normalizeDialAddrs} rule the resolved and address-book paths
|
|
2544
|
+
* use, on the third and last source of control-dial candidates: an address that
|
|
2545
|
+
* already terminates in `/p2p/<id>` must carry THIS peer's id or it is dropped
|
|
2546
|
+
* (a seed that disagrees with itself is not a dial target); one that does not —
|
|
2547
|
+
* a bare listen addr, or a relay hop with the destination missing — gets the id
|
|
2548
|
+
* encapsulated; an unencapsulatable one is dropped rather than dialed bare.
|
|
2056
2549
|
*/
|
|
2057
2550
|
private bootstrapDialAddrs(peerId: string, addrs: string[]): Multiaddr[] {
|
|
2058
|
-
|
|
2059
|
-
for (const addr of this.parseMultiaddrs(addrs)) {
|
|
2060
|
-
const embedded = trailingPeerId(addr);
|
|
2061
|
-
if (embedded === peerId) {
|
|
2062
|
-
out.push(addr);
|
|
2063
|
-
continue;
|
|
2064
|
-
}
|
|
2065
|
-
if (embedded !== null) {
|
|
2066
|
-
log('reconcileControlCohort(cold-start): addr %s names peer %s, not %s; dropping',
|
|
2067
|
-
addr.toString(), embedded, peerId);
|
|
2068
|
-
continue;
|
|
2069
|
-
}
|
|
2070
|
-
try {
|
|
2071
|
-
out.push(addr.encapsulate(`/p2p/${peerId}`));
|
|
2072
|
-
} catch (error) {
|
|
2073
|
-
log('reconcileControlCohort(cold-start): cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
2074
|
-
}
|
|
2075
|
-
}
|
|
2076
|
-
return out;
|
|
2551
|
+
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
2077
2552
|
}
|
|
2078
2553
|
|
|
2079
2554
|
/** Dial one cold-start bootstrap peer, best-effort. Returns whether it connected. */
|
|
@@ -2089,7 +2564,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2089
2564
|
}
|
|
2090
2565
|
try {
|
|
2091
2566
|
log('reconcileControlCohort(cold-start): dialing bootstrap peer %s (%d addr(s))', peerId, parsed.length);
|
|
2092
|
-
await
|
|
2567
|
+
await withDeadline(
|
|
2568
|
+
this.controlDialTimeoutMs(),
|
|
2569
|
+
`reconcileControlCohort cold-start dial of bootstrap peer ${peerId}`,
|
|
2570
|
+
(signal) => controlNode.dial(parsed, { signal })
|
|
2571
|
+
);
|
|
2093
2572
|
return true;
|
|
2094
2573
|
} catch (error) {
|
|
2095
2574
|
log('reconcileControlCohort(cold-start): dial of bootstrap peer %s failed (continuing): %o', peerId, error);
|
|
@@ -2802,7 +3281,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2802
3281
|
}
|
|
2803
3282
|
|
|
2804
3283
|
try {
|
|
2805
|
-
// Discovery path: no explicit mode — infer from cohort membership.
|
|
2806
3284
|
await this.launchStrand(strand, sAppConfig);
|
|
2807
3285
|
} catch (error) {
|
|
2808
3286
|
log('Error starting strand %s: %o', strand.Id, error);
|
|
@@ -2836,7 +3314,32 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2836
3314
|
}
|
|
2837
3315
|
}
|
|
2838
3316
|
|
|
3317
|
+
/**
|
|
3318
|
+
* Return this node to its pre-{@link start} state. The ONE teardown — `stop()` and
|
|
3319
|
+
* a failed `start()` both come here, so anything a stopped node must not still be
|
|
3320
|
+
* doing belongs in this method and not in `stop()`. `stop()` adds only the
|
|
3321
|
+
* `control:disconnected` emit, which a never-connected start has no edge for.
|
|
3322
|
+
*/
|
|
2839
3323
|
private async cleanup(): Promise<void> {
|
|
3324
|
+
// Open the connection gate's bring-up quiet period unconditionally: teardown
|
|
3325
|
+
// must never be gated, and this is the path a FAILED start() takes.
|
|
3326
|
+
this.controlBringUpInFlight = false;
|
|
3327
|
+
|
|
3328
|
+
// Stop the relay retry loop BEFORE tearing the node down, not after: the control
|
|
3329
|
+
// node is stopped partway through below, and a tick that fires during it would
|
|
3330
|
+
// dial and then poll a half-torn-down node for the whole reserve timeout —
|
|
3331
|
+
// logging spurious dial failures on a `.unref()`'d timer that nothing ever
|
|
3332
|
+
// clears. `driveControlRelayReservation` starts this supervisor from inside
|
|
3333
|
+
// `start()`, so a start that fails ON the reservation leaves one running unless
|
|
3334
|
+
// it is stopped here.
|
|
3335
|
+
this.relayReserveSupervisor?.stop();
|
|
3336
|
+
this.relayReserveSupervisor = null;
|
|
3337
|
+
// Drop the rest of the relay-reservation posture: a torn-down node holds no
|
|
3338
|
+
// reservation, so neither a restarted instance nor a caller inspecting a failed
|
|
3339
|
+
// start reports the previous attempt's `reserved`.
|
|
3340
|
+
this.relayReserveAddrs = [];
|
|
3341
|
+
this.relayReserveError = null;
|
|
3342
|
+
|
|
2840
3343
|
// Resolve + clear any in-flight wake windows first, so an in-flight check-in
|
|
2841
3344
|
// or serviceWake unblocks and tears down cleanly rather than firing a stale
|
|
2842
3345
|
// window timer (or hanging) after the strand manager is stopped below.
|
|
@@ -2918,6 +3421,27 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2918
3421
|
await this.controlNode.stop();
|
|
2919
3422
|
this.controlNode = null;
|
|
2920
3423
|
}
|
|
3424
|
+
|
|
3425
|
+
// Release the control store's claim on its cache LAST — after the database and
|
|
3426
|
+
// node that write through it are down. The wrapper counts holders, so this empties
|
|
3427
|
+
// and unregisters the cache only if no other scope still holds it. Dropping the
|
|
3428
|
+
// field is what lets a stop()/start() cycle re-resolve the provider against a live
|
|
3429
|
+
// cache instead of handing the restarted node a retired wrapper. Logged, never
|
|
3430
|
+
// thrown: a cache-bookkeeping failure must not abort a teardown.
|
|
3431
|
+
if (this.controlStorage) {
|
|
3432
|
+
const controlStorage = this.controlStorage;
|
|
3433
|
+
this.controlStorage = null;
|
|
3434
|
+
await disposeStorageCache(controlStorage).catch(
|
|
3435
|
+
(err) => log('Failed to dispose control storage cache: %o', err)
|
|
3436
|
+
);
|
|
3437
|
+
}
|
|
3438
|
+
|
|
3439
|
+
// LAST, so every step above still ran against a node that considered itself up
|
|
3440
|
+
// (several of them early-return while `_running` is false). A failed `start()`
|
|
3441
|
+
// that had already flipped it — the reservation drive throws after that point —
|
|
3442
|
+
// must not leave `isRunning` true over a torn-down node, which would report
|
|
3443
|
+
// healthy and make the embedder's retry a no-op ("already running").
|
|
3444
|
+
this._running = false;
|
|
2921
3445
|
}
|
|
2922
3446
|
|
|
2923
3447
|
// Hibernation callbacks
|
|
@@ -2961,7 +3485,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2961
3485
|
|
|
2962
3486
|
/**
|
|
2963
3487
|
* Wake a strand. If it was hibernating (quiesced — no libp2p node), re-resolve
|
|
2964
|
-
* the cohort discovery seed
|
|
3488
|
+
* the cohort discovery seed exactly as `launchStrand` does and rebuild
|
|
2965
3489
|
* its runtime via the strand manager. If it is still live (e.g. waking an idle
|
|
2966
3490
|
* strand, which retains its resources), just flip the status. Overlapping wake
|
|
2967
3491
|
* triggers are coalesced upstream by `HibernationManager`, so this runs once
|
|
@@ -2983,8 +3507,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2983
3507
|
return;
|
|
2984
3508
|
}
|
|
2985
3509
|
|
|
2986
|
-
// Quiesced: re-resolve volatile cohort
|
|
2987
|
-
//
|
|
3510
|
+
// Quiesced: re-resolve the volatile cohort input (the seed may have grown)
|
|
3511
|
+
// and rebuild the runtime.
|
|
2988
3512
|
log('Waking strand %s — rebuilding strand-network resources', strandId);
|
|
2989
3513
|
await this.resumeStrandRuntime(strandId);
|
|
2990
3514
|
instance.lastActivity = new Date();
|
|
@@ -2993,11 +3517,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2993
3517
|
}
|
|
2994
3518
|
|
|
2995
3519
|
/**
|
|
2996
|
-
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort
|
|
2997
|
-
* first: the discovery seed may have grown
|
|
2998
|
-
* shifted the mode `bootstrap → networked` since the strand last ran. Shared by
|
|
3520
|
+
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort input
|
|
3521
|
+
* first: the discovery seed may have grown since the strand last ran. Shared by
|
|
2999
3522
|
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so
|
|
3000
|
-
* both apply the same fresh seed
|
|
3523
|
+
* both apply the same fresh seed resolution. `resumeStrand` is idempotent
|
|
3001
3524
|
* (returns the live instance unchanged) as a backstop against double-resume.
|
|
3002
3525
|
*/
|
|
3003
3526
|
private async resumeStrandRuntime(strandId: string): Promise<void> {
|
|
@@ -3007,12 +3530,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3007
3530
|
const delegatePeerId = this.identityKey
|
|
3008
3531
|
? peerIdFromPrivateKey(await strandTransportKey(this.identityKey, strandId)).toString()
|
|
3009
3532
|
: undefined;
|
|
3010
|
-
const
|
|
3011
|
-
const
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3533
|
+
const bootstrapNodes = await this.resolveCohortSeed(strandId, delegatePeerId);
|
|
3534
|
+
const instance = await this.strandManager.resumeStrand(strandId, { bootstrapNodes });
|
|
3535
|
+
// Same reason as the launch path: `bootstrapNodes` only reaches the address
|
|
3536
|
+
// book through @libp2p/bootstrap discovery, so merge it directly as well.
|
|
3537
|
+
if (instance.libp2pNode) {
|
|
3538
|
+
await this.mergeStrandPeerAddrs(instance.libp2pNode, bootstrapNodes, strandId);
|
|
3539
|
+
}
|
|
3016
3540
|
}
|
|
3017
3541
|
|
|
3018
3542
|
/**
|
|
@@ -3024,7 +3548,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3024
3548
|
* realized as a resume → bounded window → re-hibernate-if-idle cycle that
|
|
3025
3549
|
* reuses the existing quiesce/resume primitives rather than a bespoke probe:
|
|
3026
3550
|
*
|
|
3027
|
-
* 1. Resume the strand (rebuild node + db, re-resolve cohort seed
|
|
3551
|
+
* 1. Resume the strand (rebuild node + db, re-resolve the cohort seed) so
|
|
3028
3552
|
* its strand network can reach cohort peers — exactly as a wake does.
|
|
3029
3553
|
* 2. Hold it resumed for a bounded window, during which the app may drive
|
|
3030
3554
|
* reads (pull-on-read) and record activity.
|
|
@@ -3049,7 +3573,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3049
3573
|
|
|
3050
3574
|
try {
|
|
3051
3575
|
// 1. Resume exactly as a wake does: re-resolve the (possibly grown) cohort
|
|
3052
|
-
// seed
|
|
3576
|
+
// seed, then rebuild the runtime.
|
|
3053
3577
|
log('Check-in: resuming strand %s to probe the cohort for pending activity', strandId);
|
|
3054
3578
|
await this.resumeStrandRuntime(strandId);
|
|
3055
3579
|
|
|
@@ -3161,18 +3685,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3161
3685
|
throw new Error('CadreNode not running');
|
|
3162
3686
|
}
|
|
3163
3687
|
|
|
3164
|
-
const { strandRow, sAppConfig,
|
|
3688
|
+
const { strandRow, sAppConfig, founder } = config;
|
|
3165
3689
|
|
|
3166
3690
|
// Store sApp config for this strand
|
|
3167
3691
|
this.sAppConfigs.set(strandRow.Id, sAppConfig);
|
|
3168
|
-
log('Registered sAppConfig for strand %s (sApp: %s,
|
|
3169
|
-
strandRow.Id, sAppConfig.id,
|
|
3692
|
+
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)',
|
|
3693
|
+
strandRow.Id, sAppConfig.id, founder ?? false);
|
|
3170
3694
|
|
|
3171
|
-
// Pass `mode` (possibly undefined) through: an explicit mode wins, while a
|
|
3172
|
-
// caller that omits it gets the same cohort-inferred mode as the discovery path.
|
|
3173
3695
|
// `founder` only flows from the explicit addStrand path — the control-discovered
|
|
3174
3696
|
// join path never founds, so its rows arrive via sync (see handleStrandAdded).
|
|
3175
|
-
return await this.launchStrand(strandRow, sAppConfig,
|
|
3697
|
+
return await this.launchStrand(strandRow, sAppConfig, founder);
|
|
3176
3698
|
}
|
|
3177
3699
|
|
|
3178
3700
|
/**
|
|
@@ -3449,8 +3971,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3449
3971
|
/**
|
|
3450
3972
|
* Shared strand launch path for both the explicit (`addStrand`) and the
|
|
3451
3973
|
* control-discovered (`handleStrandAdded`) entry points. Resolves the cohort
|
|
3452
|
-
* seed,
|
|
3453
|
-
*
|
|
3974
|
+
* seed, starts the strand, and registers it with the hibernation manager
|
|
3975
|
+
* before emitting `strand:started`.
|
|
3454
3976
|
*
|
|
3455
3977
|
* Idempotent no-op when the strand manager already tracks `strand.Id` — the
|
|
3456
3978
|
* ordinary founding sequence is `addStrand` (starts it locally) followed by
|
|
@@ -3464,7 +3986,6 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3464
3986
|
private async launchStrand(
|
|
3465
3987
|
strand: StrandRow,
|
|
3466
3988
|
sAppConfig: SAppConfig,
|
|
3467
|
-
explicitMode?: StrandMode,
|
|
3468
3989
|
founder?: boolean
|
|
3469
3990
|
): Promise<StrandInstance> {
|
|
3470
3991
|
const existing = this.strandManager.getInstance(strand.Id);
|
|
@@ -3497,8 +4018,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3497
4018
|
// `libp2p.start()` (the responder records the grant before replying; the
|
|
3498
4019
|
// client awaits the replies).
|
|
3499
4020
|
const delegatePeerId = transportKey ? peerIdFromPrivateKey(transportKey).toString() : undefined;
|
|
3500
|
-
const
|
|
3501
|
-
const mode = selectStrandMode(explicitMode, seed.hasOtherPeers);
|
|
4021
|
+
const bootstrapNodes = await this.resolveCohortSeed(strand.Id, delegatePeerId);
|
|
3502
4022
|
|
|
3503
4023
|
const instance = await this.strandManager.startStrand({
|
|
3504
4024
|
strandRow: strand,
|
|
@@ -3508,32 +4028,40 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3508
4028
|
profile: this.config.profile,
|
|
3509
4029
|
defaultLatencyHint: this.config.hibernation?.defaultLatencyHint ?? 'interactive',
|
|
3510
4030
|
privateKey: transportKey,
|
|
3511
|
-
bootstrapNodes
|
|
3512
|
-
mode,
|
|
4031
|
+
bootstrapNodes,
|
|
3513
4032
|
requireSignedSchemas: this.config.requireSignedSchemas,
|
|
3514
4033
|
clusterSize: this.config.strandClusterSize,
|
|
3515
4034
|
backfill: this.config.strandBackfill,
|
|
3516
4035
|
founder
|
|
3517
4036
|
});
|
|
3518
4037
|
|
|
4038
|
+
// The seed reached the node as `bootstrapNodes`, which only enters the
|
|
4039
|
+
// address book via @libp2p/bootstrap discovery. Merge it directly too, so a
|
|
4040
|
+
// sibling is dialable by bare peer id from the first moment (see
|
|
4041
|
+
// mergeStrandPeerAddrs; refreshStrandPeerAddrs keeps it warm from here on).
|
|
4042
|
+
if (instance.libp2pNode) {
|
|
4043
|
+
await this.mergeStrandPeerAddrs(instance.libp2pNode, bootstrapNodes, strand.Id);
|
|
4044
|
+
}
|
|
4045
|
+
|
|
3519
4046
|
this.hibernationManager.trackStrand(instance);
|
|
3520
4047
|
this.emit('strand:started', { strandId: strand.Id });
|
|
3521
4048
|
return instance;
|
|
3522
4049
|
}
|
|
3523
4050
|
|
|
3524
4051
|
/**
|
|
3525
|
-
* Resolve a strand's discovery seed
|
|
3526
|
-
*
|
|
3527
|
-
*
|
|
3528
|
-
*
|
|
3529
|
-
* addresses that must not
|
|
4052
|
+
* Resolve a strand's discovery seed — the dialable strand-network multiaddr
|
|
4053
|
+
* strings for cohort siblings. Membership comes from the control network's
|
|
4054
|
+
* CadrePeer rows; the **strand-network** bootstrap addresses are resolved on
|
|
4055
|
+
* demand over the control mesh via the strand-addr RPC — deliberately NOT
|
|
4056
|
+
* from `CadrePeer.Multiaddr`, which carries *control* addresses that must not
|
|
4057
|
+
* seed the strand mesh.
|
|
3530
4058
|
*
|
|
3531
4059
|
* Only siblings we already hold an open control connection to are RPC'd: they
|
|
3532
4060
|
* are the ones that can answer right now, and dialing them by peerId reuses the
|
|
3533
4061
|
* live connection. When no connected sibling yet runs the strand the seed is
|
|
3534
|
-
* empty (`[]`) —
|
|
3535
|
-
*
|
|
3536
|
-
*
|
|
4062
|
+
* empty (`[]`) — the empty seed self-heals on the next resume / check-in pass.
|
|
4063
|
+
* Returns an empty seed when the control DB or node is absent (not yet
|
|
4064
|
+
* started / torn down).
|
|
3537
4065
|
*
|
|
3538
4066
|
* When `delegatePeerId` is given (a strand launch/resume is imminent), the
|
|
3539
4067
|
* pass doubles as the delegate ANNOUNCEMENT: our circuit relays are merged
|
|
@@ -3546,20 +4074,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3546
4074
|
* needs no grant. The relay's direct addr rides along as the dial fallback
|
|
3547
4075
|
* for a relay we are not yet connected to.
|
|
3548
4076
|
*/
|
|
3549
|
-
private async resolveCohortSeed(strandId: string, delegatePeerId?: string): Promise<
|
|
4077
|
+
private async resolveCohortSeed(strandId: string, delegatePeerId?: string): Promise<string[]> {
|
|
3550
4078
|
if (!this.controlDatabase || !this.controlNode) {
|
|
3551
|
-
return
|
|
4079
|
+
return [];
|
|
3552
4080
|
}
|
|
3553
|
-
const
|
|
3554
|
-
const { otherPeerIds, hasOtherPeers } =
|
|
3555
|
-
deriveCohortMembers(peers, this.controlNode.peerId.toString());
|
|
3556
|
-
|
|
3557
|
-
// RPC only siblings we already have a control connection to — they can answer
|
|
3558
|
-
// now, and `dialProtocol` by peerId reuses the open connection.
|
|
3559
|
-
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3560
|
-
const targets: StrandAddrPeer[] = otherPeerIds
|
|
3561
|
-
.filter((id) => connected.has(id))
|
|
3562
|
-
.map((peerId) => ({ peerId }));
|
|
4081
|
+
const targets = await this.connectedSiblingTargets();
|
|
3563
4082
|
const relays = delegatePeerId === undefined ? [] : this.circuitRelayTargets();
|
|
3564
4083
|
for (const relay of relays) {
|
|
3565
4084
|
if (!targets.some((t) => t.peerId === relay.relayPeerId)) {
|
|
@@ -3572,16 +4091,72 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3572
4091
|
// Throttle state for the RELAY targets only — refreshDelegateGrants never
|
|
3573
4092
|
// looks up a sibling key, so recording one would only be dead weight.
|
|
3574
4093
|
this.recordDelegateAnnounces(relays.map((r) => r.relayPeerId), strandId);
|
|
3575
|
-
return
|
|
4094
|
+
return bootstrapNodes;
|
|
4095
|
+
}
|
|
4096
|
+
|
|
4097
|
+
/**
|
|
4098
|
+
* The co-cadre siblings worth sending a strand-addr RPC to right now: cohort
|
|
4099
|
+
* members (self excluded) we already hold an open control connection to. They
|
|
4100
|
+
* are the ones that can answer immediately, and dialing them by peerId reuses
|
|
4101
|
+
* the live connection instead of opening a second one.
|
|
4102
|
+
*
|
|
4103
|
+
* Shared by {@link resolveCohortSeed} (launch/resume) and
|
|
4104
|
+
* {@link refreshStrandPeerAddrs} (the periodic pass) so both ask the same set.
|
|
4105
|
+
* Empty when the control DB or node is absent (not yet started / torn down).
|
|
4106
|
+
*
|
|
4107
|
+
* The read below is issued even when the node holds no control connection and
|
|
4108
|
+
* the answer is therefore empty by construction. That is deliberate on the
|
|
4109
|
+
* launch path: `control-database-solo-warm-start.spec.ts` exists to prove this
|
|
4110
|
+
* exact read does not stall as an embedder's FIRST awaited control operation,
|
|
4111
|
+
* on a warm cohort no one can reach. A caller that re-enters often enough for
|
|
4112
|
+
* the read to matter should skip the call itself, as
|
|
4113
|
+
* {@link refreshStrandPeerAddrs} does.
|
|
4114
|
+
*
|
|
4115
|
+
* NOTE: this read is UNBOUNDED and sits on the critical path an embedding app
|
|
4116
|
+
* awaits during startup — `addStrand` cannot resolve until it does, and it is
|
|
4117
|
+
* the first control operation an embedder's boot order issues (before genesis,
|
|
4118
|
+
* before seed bootstrap). Measured fine today: the warm-start-alone shape a
|
|
4119
|
+
* report pointed at — a `CadrePeer` list naming peers that are all gone, read
|
|
4120
|
+
* off real files after a restart — completes in milliseconds, and
|
|
4121
|
+
* `control-database-solo-warm-start.spec.ts` covers it under a deadline. If a
|
|
4122
|
+
* control read ever CAN stall (a transactor change that consults the network
|
|
4123
|
+
* for a local row, a storage backend with blocking I/O), give this one a
|
|
4124
|
+
* timeout — and decide then whether the breach fails `addStrand` or degrades
|
|
4125
|
+
* to an empty seed, because those promise callers different things.
|
|
4126
|
+
*/
|
|
4127
|
+
private async connectedSiblingTargets(): Promise<StrandAddrPeer[]> {
|
|
4128
|
+
const controlNode = this.controlNode;
|
|
4129
|
+
if (!this.controlDatabase || !controlNode) {
|
|
4130
|
+
return [];
|
|
4131
|
+
}
|
|
4132
|
+
const peers = await this.controlDatabase.queryCadrePeers();
|
|
4133
|
+
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
4134
|
+
return deriveCohortMembers(peers, controlNode.peerId.toString())
|
|
4135
|
+
.filter((id) => connected.has(id))
|
|
4136
|
+
.map((peerId) => ({ peerId }));
|
|
3576
4137
|
}
|
|
3577
4138
|
|
|
3578
4139
|
/**
|
|
3579
4140
|
* The circuit relays this node's strand nodes would reserve through: the
|
|
3580
|
-
* union of
|
|
3581
|
-
* `network.listenAddrs` circuit entry
|
|
3582
|
-
*
|
|
3583
|
-
*
|
|
3584
|
-
*
|
|
4141
|
+
* union of CONFIGURED relays — `network.relayAddrs`, plus any hand-written
|
|
4142
|
+
* `network.listenAddrs` circuit entry, both of which a strand node inherits
|
|
4143
|
+
* verbatim — and the control node's own live `/p2p-circuit` multiaddrs (a
|
|
4144
|
+
* reservation this node discovered rather than configured).
|
|
4145
|
+
*
|
|
4146
|
+
* The CONFIGURED half reads `network.relayAddrs` and `network.listenAddrs`
|
|
4147
|
+
* directly rather than going through {@link resolveListenAddrs}: the control
|
|
4148
|
+
* node's resolution takes the `'search'` route, whose bare `/p2p-circuit` entry
|
|
4149
|
+
* names no relay, so a configured relay would be unannounceable here until this
|
|
4150
|
+
* node's own reservation landed — and the announce must be in place BEFORE a
|
|
4151
|
+
* strand node dials its reservation, since the relay's membership gate does not
|
|
4152
|
+
* know the strand's derived peerId (see delegate-admission.ts). Configuration is
|
|
4153
|
+
* authoritative; the live multiaddrs only ADD relays nobody configured.
|
|
4154
|
+
*
|
|
4155
|
+
* The `listenAddrs` half is now unreachable on a control node — `relay-addrs.ts`
|
|
4156
|
+
* rejects a hand-written `<relay>/p2p-circuit` listen entry outright, because the
|
|
4157
|
+
* bring-up quiet period denies the dial that listener makes from inside
|
|
4158
|
+
* `libp2p.start()`. Kept because it costs one spread and this is the one place
|
|
4159
|
+
* that would silently stop announcing if that rejection is ever relaxed.
|
|
3585
4160
|
*
|
|
3586
4161
|
* NOTE: a relay the STRAND node discovers on its own (autorelay — in neither
|
|
3587
4162
|
* source) gets no announcement, and a membership-gated one will deny it; fine
|
|
@@ -3589,7 +4164,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3589
4164
|
*/
|
|
3590
4165
|
private circuitRelayTargets(): CircuitRelayTarget[] {
|
|
3591
4166
|
return extractCircuitRelayTargets([
|
|
3592
|
-
...(
|
|
4167
|
+
...relayCircuitAddrs(this.config.network?.relayAddrs ?? []),
|
|
4168
|
+
...(this.config.network?.listenAddrs ?? []),
|
|
3593
4169
|
...(this.controlNode?.getMultiaddrs().map(String) ?? [])
|
|
3594
4170
|
]);
|
|
3595
4171
|
}
|
|
@@ -3669,6 +4245,166 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3669
4245
|
this.recordDelegateAnnounces(due.map((relay) => relay.relayPeerId), strandId, now);
|
|
3670
4246
|
}
|
|
3671
4247
|
|
|
4248
|
+
/**
|
|
4249
|
+
* Re-resolve each running strand's SIBLING addresses over the control mesh and
|
|
4250
|
+
* merge them into that strand's own libp2p address book, throttled to once per
|
|
4251
|
+
* {@link STRAND_PEER_ADDR_REFRESH_MS} per strand.
|
|
4252
|
+
*
|
|
4253
|
+
* Without this, a strand's address book is written exactly once — the
|
|
4254
|
+
* launch/resume seed — and everything below cadre-core that dials a strand peer
|
|
4255
|
+
* by bare peer id (Optimystic's cluster and repo clients, FRET ping/announce)
|
|
4256
|
+
* loses its address for any sibling it is not currently connected to: a sibling
|
|
4257
|
+
* that restarted its strand node or rotated its relay reservation is never
|
|
4258
|
+
* re-resolved, and even the original seed addresses fall off at the peerStore's
|
|
4259
|
+
* one-hour expiry (see `peer-addr-book.ts`).
|
|
4260
|
+
*
|
|
4261
|
+
* Distinct from {@link refreshDelegateGrants}, deliberately: that pass covers
|
|
4262
|
+
* RELAYS on a `DELEGATE_GRANT_TTL_MS / 2` throttle to keep circuit-relay
|
|
4263
|
+
* admission grants alive, this one covers SIBLINGS on an address-expiry
|
|
4264
|
+
* throttle to keep the address book warm. They overlap only in that both carry
|
|
4265
|
+
* `delegatePeerId`, so a sibling that also runs a relay gets its grant
|
|
4266
|
+
* refreshed here as a side effect. Merging them would tie an admission-grant
|
|
4267
|
+
* TTL to an address-expiry window that has nothing to do with it.
|
|
4268
|
+
*
|
|
4269
|
+
* Strands refresh CONCURRENTLY, like `refreshDelegateGrants`, so one
|
|
4270
|
+
* unreachable sibling's dial timeout does not stack up per strand.
|
|
4271
|
+
*/
|
|
4272
|
+
private async refreshStrandPeerAddrs(now = Date.now()): Promise<void> {
|
|
4273
|
+
if (!this._running || !this.controlNode) {
|
|
4274
|
+
return;
|
|
4275
|
+
}
|
|
4276
|
+
// A hibernating / quiescing strand has no node to seed, so it is not running
|
|
4277
|
+
// for this pass's purposes even though its instance is still tracked.
|
|
4278
|
+
const running = new Set<string>();
|
|
4279
|
+
for (const [strandId, instance] of this.strandManager.getInstances()) {
|
|
4280
|
+
if (instance.libp2pNode) {
|
|
4281
|
+
running.add(strandId);
|
|
4282
|
+
}
|
|
4283
|
+
}
|
|
4284
|
+
// Drop stamps for strands no longer running: the map must not grow for the
|
|
4285
|
+
// node's lifetime, and a resumed strand should refresh immediately rather
|
|
4286
|
+
// than inherit the stamp its previous incarnation left.
|
|
4287
|
+
for (const strandId of this.strandPeerAddrRefreshAt.keys()) {
|
|
4288
|
+
if (!running.has(strandId)) {
|
|
4289
|
+
this.strandPeerAddrRefreshAt.delete(strandId);
|
|
4290
|
+
}
|
|
4291
|
+
}
|
|
4292
|
+
const refreshMs = this.config.network?.controlCohort?.strandAddrRefreshMs ?? STRAND_PEER_ADDR_REFRESH_MS;
|
|
4293
|
+
const due = [...running].filter(
|
|
4294
|
+
(strandId) => now - (this.strandPeerAddrRefreshAt.get(strandId) ?? 0) >= refreshMs
|
|
4295
|
+
);
|
|
4296
|
+
if (due.length === 0) {
|
|
4297
|
+
return;
|
|
4298
|
+
}
|
|
4299
|
+
// A due strand with nobody to ask leaves its throttle unstamped (see below),
|
|
4300
|
+
// so this pass re-enters on EVERY 15 s reconcile tick for as long as the node
|
|
4301
|
+
// is alone — which is the steady state of a solo node running a strand.
|
|
4302
|
+
// `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
4303
|
+
// connections its answer is empty whatever the table holds, so decide it here
|
|
4304
|
+
// from the connection list instead of paying for the read once a tick.
|
|
4305
|
+
if (this.controlNode.getConnections().length === 0) {
|
|
4306
|
+
return;
|
|
4307
|
+
}
|
|
4308
|
+
// NOTE: one strand-addr RPC per (due strand × connected sibling) per refresh
|
|
4309
|
+
// interval — each a tiny request/response on an already-open control
|
|
4310
|
+
// connection. If a node ever runs MANY strands at once, batch the RPC to
|
|
4311
|
+
// carry several strand ids per request rather than one fan-out per strand.
|
|
4312
|
+
const targets = await this.connectedSiblingTargets().catch((error): StrandAddrPeer[] => {
|
|
4313
|
+
log('refreshStrandPeerAddrs: sibling enumeration failed (skipping pass): %o', error);
|
|
4314
|
+
return [];
|
|
4315
|
+
});
|
|
4316
|
+
// No sibling to ask (or a shutdown landed mid-enumeration): leave every stamp
|
|
4317
|
+
// untouched so the next reconcile tick retries, rather than sitting out the
|
|
4318
|
+
// whole refresh interval having asked nobody.
|
|
4319
|
+
if (targets.length === 0 || !this._running || !this.controlNode) {
|
|
4320
|
+
return;
|
|
4321
|
+
}
|
|
4322
|
+
await Promise.all(due.map((strandId) => this.refreshOneStrandPeerAddrs(strandId, targets, now)));
|
|
4323
|
+
}
|
|
4324
|
+
|
|
4325
|
+
/**
|
|
4326
|
+
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings, then
|
|
4327
|
+
* merge their answers into this strand's address book. Errors are logged and
|
|
4328
|
+
* swallowed so one strand's failure never costs the others their refresh.
|
|
4329
|
+
*/
|
|
4330
|
+
private async refreshOneStrandPeerAddrs(
|
|
4331
|
+
strandId: string,
|
|
4332
|
+
targets: readonly StrandAddrPeer[],
|
|
4333
|
+
now: number
|
|
4334
|
+
): Promise<void> {
|
|
4335
|
+
const controlNode = this.controlNode;
|
|
4336
|
+
const strandNode = this.strandManager.getInstance(strandId)?.libp2pNode;
|
|
4337
|
+
if (!controlNode || !strandNode) {
|
|
4338
|
+
return;
|
|
4339
|
+
}
|
|
4340
|
+
try {
|
|
4341
|
+
// The running strand node's own peerId is the delegate to announce — see
|
|
4342
|
+
// the relationship note on refreshStrandPeerAddrs.
|
|
4343
|
+
const addrs = await collectStrandAddrs(controlNode, [...targets], strandId, {
|
|
4344
|
+
delegatePeerId: strandNode.peerId.toString()
|
|
4345
|
+
});
|
|
4346
|
+
// Stamped on the RPC having happened, not on its answer: the fan-out is
|
|
4347
|
+
// what the throttle exists to bound, and an all-empty round is a normal
|
|
4348
|
+
// steady state for a strand no connected sibling currently runs.
|
|
4349
|
+
this.strandPeerAddrRefreshAt.set(strandId, now);
|
|
4350
|
+
// Re-read the instance after the await — a strand stopped mid-pass (or one
|
|
4351
|
+
// already restarted onto a new node) must never have its store written to.
|
|
4352
|
+
if (!this._running || this.strandManager.getInstance(strandId)?.libp2pNode !== strandNode) {
|
|
4353
|
+
return;
|
|
4354
|
+
}
|
|
4355
|
+
await this.mergeStrandPeerAddrs(strandNode, addrs, strandId);
|
|
4356
|
+
} catch (error) {
|
|
4357
|
+
log('refreshStrandPeerAddrs: strand %s refresh failed (continuing): %o', strandId, error);
|
|
4358
|
+
}
|
|
4359
|
+
}
|
|
4360
|
+
|
|
4361
|
+
/**
|
|
4362
|
+
* Merge strand-network addresses into ONE strand node's libp2p address book,
|
|
4363
|
+
* attributed per peer. Best-effort throughout: an address-book write must never
|
|
4364
|
+
* fail a launch, a resume, or a reconcile pass.
|
|
4365
|
+
*
|
|
4366
|
+
* `addrs` is the peer-agnostic union the strand-addr RPC returns, so
|
|
4367
|
+
* {@link groupAddrsByPeerId} attributes each entry to the **strand transport**
|
|
4368
|
+
* peerId in its final `/p2p/` component — never the sibling's control peerId,
|
|
4369
|
+
* which names a different libp2p node entirely. Entries that name no peer are
|
|
4370
|
+
* dropped there and counted here, once per pass.
|
|
4371
|
+
*
|
|
4372
|
+
* NOTE: a member could answer with arbitrary multiaddrs bound to arbitrary peer
|
|
4373
|
+
* ids and poison this address book. That is the same exposure the launch-time
|
|
4374
|
+
* `bootstrapNodes` seeding already accepts, and the cost is bounded: an address
|
|
4375
|
+
* grants no authority, the dialed peer authenticates by peer id at the
|
|
4376
|
+
* handshake, and a bad entry costs one failed dial that ages out at the
|
|
4377
|
+
* peerStore's one-hour expiry. No new gating here — cross-party strand trust is
|
|
4378
|
+
* `backlog/strand-network-nat-relay-reachability`.
|
|
4379
|
+
*/
|
|
4380
|
+
private async mergeStrandPeerAddrs(strandNode: Libp2p, addrs: string[], strandId: string): Promise<void> {
|
|
4381
|
+
const counts: Record<MergeAddrsResult, number> = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
4382
|
+
let grouped = 0;
|
|
4383
|
+
let peers = 0;
|
|
4384
|
+
try {
|
|
4385
|
+
const groups = groupAddrsByPeerId(addrs);
|
|
4386
|
+
peers = groups.size;
|
|
4387
|
+
const selfId = strandNode.peerId.toString();
|
|
4388
|
+
for (const [peerId, peerAddrs] of groups) {
|
|
4389
|
+
grouped += peerAddrs.length;
|
|
4390
|
+
// We never RPC ourselves, so self should never appear — belt and braces:
|
|
4391
|
+
// a node must not write its own addresses into its own address book.
|
|
4392
|
+
if (peerId === selfId) {
|
|
4393
|
+
continue;
|
|
4394
|
+
}
|
|
4395
|
+
counts[await mergePeerAddrs(strandNode, peerId, peerAddrs)]++;
|
|
4396
|
+
}
|
|
4397
|
+
} catch (error) {
|
|
4398
|
+
// `mergePeerAddrs` folds its own failures, so reaching here means the node
|
|
4399
|
+
// itself is unusable (torn down mid-merge). Never fail a launch/resume or a
|
|
4400
|
+
// reconcile pass over an address book.
|
|
4401
|
+
log('strand %s address book merge failed (continuing): %o', strandId, error);
|
|
4402
|
+
}
|
|
4403
|
+
log('strand %s address book merged (peers=%d, merged=%d, restamped=%d, skipped=%d, failed=%d, dropped=%d)',
|
|
4404
|
+
strandId, peers, counts.merged, counts.restamped, counts.skipped, counts.failed,
|
|
4405
|
+
addrs.length - grouped);
|
|
4406
|
+
}
|
|
4407
|
+
|
|
3672
4408
|
/**
|
|
3673
4409
|
* The local strand instance's dialable strand-network multiaddrs for the
|
|
3674
4410
|
* strand-addr RPC, ordered signaling-first (reusing the control node's
|
|
@@ -3913,16 +4649,61 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3913
4649
|
return this.controlNode;
|
|
3914
4650
|
}
|
|
3915
4651
|
|
|
4652
|
+
/**
|
|
4653
|
+
* The boot-path half of {@link reserveRelays}: reserve through every
|
|
4654
|
+
* `network.relayAddrs` entry, or fail `start()`.
|
|
4655
|
+
*
|
|
4656
|
+
* WHY IT RUNS HERE, AT THE END. `network.relayAddrs` used to resolve to a
|
|
4657
|
+
* CONFIGURED circuit listener, which libp2p dials from inside `libp2p.start()`
|
|
4658
|
+
* — so the relay was already a connected peer, and therefore already in this
|
|
4659
|
+
* node's Optimystic cohort, before `ControlDatabase.initialize()` ran. Building
|
|
4660
|
+
* that database is a long chain of cohort-consulting block probes (catalog
|
|
4661
|
+
* hydration, then one per table, then the indexes), and a sibling that has not
|
|
4662
|
+
* yet replicated this node's `CadrePeer` row correctly refuses every one of
|
|
4663
|
+
* them, so bring-up died on `BlockUnavailableError` — deterministically, on
|
|
4664
|
+
* every boot of a relay-only node. Retrying could not converge either: the
|
|
4665
|
+
* condition that clears the refusal is this node's own row reaching the
|
|
4666
|
+
* sibling, and writing that row needs the database the retry is building.
|
|
4667
|
+
*
|
|
4668
|
+
* So the invariant is ordering, not retrying: the control database is built
|
|
4669
|
+
* while this node holds ZERO control connections (a cohort of one, entirely
|
|
4670
|
+
* local), and only then does it reach out. Everything above in {@link start}
|
|
4671
|
+
* has completed by the time this runs.
|
|
4672
|
+
*
|
|
4673
|
+
* BUDGET: the supervisor's first attempt — `relay-reservation.ts`'s
|
|
4674
|
+
* `DEFAULT_RELAY_RESERVE_TIMEOUT_MS`, 10 s — is what `start()` waits on — deliberately the module default rather
|
|
4675
|
+
* than a boot-specific one. A healthy dial-plus-reserve is sub-second even over
|
|
4676
|
+
* a WAN, so 10 s is slack for a slow link; going much longer would make a dead
|
|
4677
|
+
* relay indistinguishable from a hung start, and much shorter would fail nodes
|
|
4678
|
+
* on links that were merely slow. The retries carry on in the background after
|
|
4679
|
+
* this resolves, exactly as they do for a {@link reserveRelays} caller.
|
|
4680
|
+
*/
|
|
4681
|
+
private async driveControlRelayReservation(): Promise<void> {
|
|
4682
|
+
const relayAddrs = this.config.network?.relayAddrs ?? [];
|
|
4683
|
+
if (relayAddrs.length === 0) {
|
|
4684
|
+
return;
|
|
4685
|
+
}
|
|
4686
|
+
const t0 = performance.now();
|
|
4687
|
+
const state = await this.reserveRelays([...relayAddrs]);
|
|
4688
|
+
timing('[start] reserveRelays: %dms', Math.round(performance.now() - t0));
|
|
4689
|
+
if (state.status !== 'reserved') {
|
|
4690
|
+
throw new RelayReservationFailedError(relayAddrs, state);
|
|
4691
|
+
}
|
|
4692
|
+
log('Reserved a relay slot: %o', state.circuitAddrs);
|
|
4693
|
+
}
|
|
4694
|
+
|
|
3916
4695
|
/**
|
|
3917
4696
|
* Start keeping a relay reservation: dial the given relay(s) from the control
|
|
3918
4697
|
* node, ask the first one that answers for a reservation slot, wait until the
|
|
3919
4698
|
* resulting `/p2p-circuit` address makes this node dialable — and keep a
|
|
3920
4699
|
* supervisor running that re-drives whenever the reservation is later lost.
|
|
3921
4700
|
*
|
|
3922
|
-
*
|
|
3923
|
-
*
|
|
3924
|
-
*
|
|
3925
|
-
*
|
|
4701
|
+
* The FAIL-SOFT entry point, for a caller that discovers its relay at runtime
|
|
4702
|
+
* (a browser tab, a host UI). A node that names `network.relayAddrs` gets the
|
|
4703
|
+
* same drive from {@link driveControlRelayReservation} at the end of
|
|
4704
|
+
* {@link start}, which is fail-fast instead. Both fill the pending reservation
|
|
4705
|
+
* the bare `/p2p-circuit` search listener registered; calling this afterwards
|
|
4706
|
+
* simply replaces the supervisor with one over the new list.
|
|
3926
4707
|
*
|
|
3927
4708
|
* Resolves once the FIRST attempt has settled, exactly as it did when it drove
|
|
3928
4709
|
* once; the retries continue in the background until {@link stop} or the next
|
|
@@ -4076,6 +4857,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4076
4857
|
* Push the multiaddrs that future invites should advertise. Pass `null` to
|
|
4077
4858
|
* revert to the libp2p-reported addresses (the default). The host calls this
|
|
4078
4859
|
* at spawn and on every NAT change.
|
|
4860
|
+
*
|
|
4861
|
+
* Entries need NOT carry a `/p2p/<peerId>` suffix — {@link resolveInviteAddresses}
|
|
4862
|
+
* appends this node's own before anything publishes or dials them (see
|
|
4863
|
+
* {@link normalizeSelfAddrs}). Passing them suffixed is equally fine.
|
|
4079
4864
|
*/
|
|
4080
4865
|
setInviteAddresses(addresses: string[] | null): void {
|
|
4081
4866
|
this.latestInviteAddresses = addresses;
|
|
@@ -4084,18 +4869,60 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4084
4869
|
|
|
4085
4870
|
/**
|
|
4086
4871
|
* Resolve the addresses to embed in invites. Prefers pushed addresses, then
|
|
4087
|
-
* any config-supplied resolver, then the libp2p-observed multiaddrs.
|
|
4872
|
+
* any config-supplied resolver, then the libp2p-observed multiaddrs. The two
|
|
4873
|
+
* app-supplied sources are normalized onto `/p2p/<self>` (see
|
|
4874
|
+
* {@link normalizeSelfAddrs}) so neither hook has to remember the suffix;
|
|
4875
|
+
* libp2p's own addresses already carry it.
|
|
4088
4876
|
*/
|
|
4089
4877
|
private async resolveInviteAddresses(): Promise<string[]> {
|
|
4090
4878
|
if (this.latestInviteAddresses !== null) {
|
|
4091
|
-
return this.latestInviteAddresses;
|
|
4879
|
+
return this.normalizeSelfAddrs(this.latestInviteAddresses);
|
|
4092
4880
|
}
|
|
4093
4881
|
if (this.config.network?.inviteAddressResolver) {
|
|
4094
|
-
return this.config.network.inviteAddressResolver();
|
|
4882
|
+
return this.normalizeSelfAddrs(await this.config.network.inviteAddressResolver());
|
|
4095
4883
|
}
|
|
4884
|
+
// Not normalized: this branch is libp2p's own output, which already
|
|
4885
|
+
// encapsulates the peer id into every address it reports. Only the two app
|
|
4886
|
+
// hooks above take arbitrary strings and therefore need the guarantee.
|
|
4096
4887
|
return this.getMultiaddrs();
|
|
4097
4888
|
}
|
|
4098
4889
|
|
|
4890
|
+
/**
|
|
4891
|
+
* Normalize addresses an app handed us for THIS node onto `/p2p/<self>`, the
|
|
4892
|
+
* same invariant {@link normalizeDialAddrs} enforces on the way out.
|
|
4893
|
+
*
|
|
4894
|
+
* `setInviteAddresses` (the admin API `PUT /admin/invite-addresses`) and
|
|
4895
|
+
* `network.inviteAddressResolver` both take arbitrary strings, and whatever
|
|
4896
|
+
* they return ends up in this node's published `CadrePeer` record — so ONE
|
|
4897
|
+
* unsuffixed entry from either hook is enough to give every sibling in the
|
|
4898
|
+
* party a mixed candidate list for this peer. Establishing the suffix here,
|
|
4899
|
+
* where those strings enter the system, means neither hook has to know the
|
|
4900
|
+
* rule; `cadre-host`'s `buildInviteAddresses` already appends it, and
|
|
4901
|
+
* re-normalizing an already-suffixed address is a no-op.
|
|
4902
|
+
*
|
|
4903
|
+
* Unparsable and other-peer-addressed entries are passed through untouched
|
|
4904
|
+
* rather than dropped: publication is not the place to police an address's
|
|
4905
|
+
* validity, and {@link resolvePeerAddrs} already drops both on the read side.
|
|
4906
|
+
* Before `start()` resolves an identity there is no peer id to append, so the
|
|
4907
|
+
* list is returned as given.
|
|
4908
|
+
*/
|
|
4909
|
+
private normalizeSelfAddrs(addrs: string[]): string[] {
|
|
4910
|
+
const selfPeerId = this.peerId?.toString();
|
|
4911
|
+
if (!selfPeerId) {
|
|
4912
|
+
return addrs;
|
|
4913
|
+
}
|
|
4914
|
+
return addrs.map((addr) => {
|
|
4915
|
+
let parsed: Multiaddr;
|
|
4916
|
+
try {
|
|
4917
|
+
parsed = multiaddr(addr);
|
|
4918
|
+
} catch (error) {
|
|
4919
|
+
log('normalizeSelfAddrs: leaving unparsable multiaddr %s as-is: %o', addr, error);
|
|
4920
|
+
return addr;
|
|
4921
|
+
}
|
|
4922
|
+
return withTrailingPeerId(parsed, selfPeerId)?.toString() ?? addr;
|
|
4923
|
+
});
|
|
4924
|
+
}
|
|
4925
|
+
|
|
4099
4926
|
/**
|
|
4100
4927
|
* Enumerate the cadre's `CadrePeer` membership — the ADDRESSABLE surface (see
|
|
4101
4928
|
* {@link isMember}). Rows whose stamp is retired in `CadreControl.Revocation`
|
|
@@ -4159,13 +4986,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4159
4986
|
* legit member goes un-authorized on readers that only pin the new key. Full
|
|
4160
4987
|
* rotation handling (re-vouch on rotate) is the
|
|
4161
4988
|
* `flip-strand-membership-rotation-known-gap` work, not this predicate's.
|
|
4989
|
+
*
|
|
4990
|
+
* @param retry - Whether the underlying membership read may retry a transient cluster
|
|
4991
|
+
* failure. Only {@link refreshAuthorizedControlPeers} passes `false`, because it runs
|
|
4992
|
+
* as the control database's membership listener with that database's write lock held;
|
|
4993
|
+
* see its comment.
|
|
4162
4994
|
*/
|
|
4163
|
-
async listAuthorizedMembers(): Promise<Array<{ peerId: string; multiaddr: string | null }>> {
|
|
4995
|
+
async listAuthorizedMembers(retry = true): Promise<Array<{ peerId: string; multiaddr: string | null }>> {
|
|
4164
4996
|
if (!this.controlDatabase) {
|
|
4165
4997
|
throw new Error('CadreNode must be started before listing members');
|
|
4166
4998
|
}
|
|
4167
4999
|
const selfPeerId = this.peerId?.toString();
|
|
4168
|
-
const rows = await this.controlDatabase.queryCadrePeers();
|
|
5000
|
+
const rows = await this.controlDatabase.queryCadrePeers(retry);
|
|
4169
5001
|
const authorized = rows
|
|
4170
5002
|
.filter(row => row.peerId !== selfPeerId && this.hasAnchoredVoucher(row))
|
|
4171
5003
|
.map(({ peerId, multiaddr }) => ({ peerId, multiaddr }));
|