@serfab/cadre-core 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +255 -255
- package/dist/cadre-node.d.ts +203 -28
- package/dist/cadre-node.js +445 -84
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-cohort.d.ts +26 -0
- package/dist/control-cohort.js +26 -0
- package/dist/control-cohort.js.map +1 -1
- package/dist/control-database.d.ts +82 -117
- package/dist/control-database.js +217 -295
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +8 -7
- package/dist/control-formation-recorder.js +8 -7
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +789 -774
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +15 -5
- package/dist/control-write-retry.js +19 -18
- package/dist/control-write-retry.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/peer-addr-book.d.ts +92 -0
- package/dist/peer-addr-book.js +220 -0
- package/dist/peer-addr-book.js.map +1 -0
- package/dist/seed-bootstrap.d.ts +7 -0
- package/dist/seed-bootstrap.js +15 -13
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/strand-cohort.d.ts +4 -37
- package/dist/strand-cohort.js +4 -15
- package/dist/strand-cohort.js.map +1 -1
- package/dist/strand-database.d.ts +11 -16
- package/dist/strand-database.js +19 -7
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +8 -8
- package/dist/strand-formation-manager.js +15 -18
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +52 -21
- package/dist/strand-instance-manager.js +100 -34
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +6 -4
- package/dist/strand-membership-writer.js +38 -36
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/types.d.ts +39 -31
- package/dist/types.js.map +1 -1
- package/package.json +8 -7
- package/src/arachnode-stub.ts +159 -159
- package/src/bootstrap-peer-store-file.ts +44 -44
- package/src/bootstrap-peer-store.ts +172 -172
- package/src/cadre-node.ts +5158 -4771
- package/src/canonical-json.ts +30 -30
- package/src/control-authorization.ts +114 -114
- package/src/control-cohort.ts +130 -103
- package/src/control-database.ts +2406 -2506
- package/src/control-formation-recorder.ts +305 -304
- package/src/control-schema.ts +801 -786
- package/src/control-stream.ts +205 -205
- package/src/control-write-retry.ts +483 -473
- package/src/delegate-admission.ts +302 -302
- package/src/device-token.ts +94 -94
- package/src/diagnostics/connection-path.ts +277 -277
- package/src/diagnostics/webrtc-turn-tracker.ts +222 -222
- package/src/ed25519-key.ts +134 -134
- package/src/enrollment.ts +199 -199
- package/src/file-durable-slot.ts +61 -61
- package/src/formation-approval.ts +604 -604
- package/src/fs-atomic.ts +107 -107
- package/src/hibernation-manager.ts +437 -437
- package/src/identity-key.ts +109 -109
- package/src/index.ts +1 -0
- package/src/key-store-file.ts +132 -132
- package/src/key-store.ts +99 -99
- package/src/membership-connection-gater.ts +204 -204
- package/src/node-local-snapshot.ts +260 -260
- package/src/peer-addr-book.ts +250 -0
- package/src/peer-authorization.ts +259 -259
- package/src/peer-record.ts +164 -164
- package/src/push-credentials.ts +84 -84
- package/src/push-node.ts +64 -64
- package/src/push-notifier-apns.ts +247 -247
- package/src/push-notifier-fcm.ts +220 -220
- package/src/push-notifier-shared.ts +28 -28
- package/src/push-notifier.ts +52 -52
- package/src/relay-addrs.ts +111 -111
- package/src/relay-reservation.ts +823 -823
- package/src/seed-bootstrap.ts +15 -15
- package/src/seed-trust-policy.ts +140 -140
- package/src/strand-addr-protocol.ts +441 -441
- package/src/strand-backfill.ts +366 -366
- package/src/strand-cohort.ts +38 -77
- package/src/strand-database.ts +202 -204
- package/src/strand-formation-manager.ts +491 -494
- package/src/strand-formation-protocol.ts +692 -692
- package/src/strand-instance-manager.ts +108 -50
- package/src/strand-member-key.ts +39 -39
- package/src/strand-member-registry.ts +182 -182
- package/src/strand-membership-writer.ts +1432 -1430
- package/src/strand-solicitation.ts +472 -472
- package/src/strand-transport-key.ts +58 -58
- package/src/strand-wake-payload.ts +32 -32
- package/src/strand-wake-protocol.ts +314 -314
- package/src/strand-watcher.ts +314 -314
- package/src/trusted-owner-store-file.ts +41 -41
- package/src/trusted-owner-store.ts +171 -171
- package/src/types.ts +1309 -1303
package/dist/cadre-node.js
CHANGED
|
@@ -2,6 +2,7 @@ import debug from 'debug';
|
|
|
2
2
|
import { toString as uint8ArrayToString, fromString as uint8ArrayFromString } from 'uint8arrays';
|
|
3
3
|
import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
4
4
|
import { createLibp2pNode } from '@optimystic/db-p2p';
|
|
5
|
+
import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
|
|
5
6
|
import { multiaddr } from '@multiformats/multiaddr';
|
|
6
7
|
import { CONTROL_CLUSTER_POLICY, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
7
8
|
import { sign } from '@optimystic/quereus-plugin-crypto';
|
|
@@ -11,14 +12,16 @@ import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
|
11
12
|
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
12
13
|
import { MemoryTrustedOwnerStore } from './trusted-owner-store.js';
|
|
13
14
|
import { MemoryBootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
15
|
+
import { mergePeerAddrs, groupAddrsByPeerId } from './peer-addr-book.js';
|
|
14
16
|
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
15
17
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
16
18
|
import { signPeerRecord, verifyPeerRecordSignature, isPeerRecordFresh, orderSignalingFirst, isSignalingAddr, trailingPeerId, currentMemberTrustPolicy, DEFAULT_PEER_RECORD_MAX_AGE_MS, DEFAULT_PEER_RECORD_HEARTBEAT_MS } from './peer-record.js';
|
|
17
19
|
import { signDeviceTokenRecord, verifyDeviceTokenSignature, isPushPlatform } from './device-token.js';
|
|
18
20
|
import { StrandWatcher } from './strand-watcher.js';
|
|
19
21
|
import { StrandInstanceManager } from './strand-instance-manager.js';
|
|
20
|
-
import { deriveCohortMembers
|
|
21
|
-
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
22
|
+
import { deriveCohortMembers } from './strand-cohort.js';
|
|
23
|
+
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
24
|
+
import { withDeadline } from './control-stream.js';
|
|
22
25
|
import { EnrollmentService } from './enrollment.js';
|
|
23
26
|
import { HibernationManager } from './hibernation-manager.js';
|
|
24
27
|
import { ControlDatabase } from './control-database.js';
|
|
@@ -41,6 +44,18 @@ const timing = debug('sereus:cadre:timing');
|
|
|
41
44
|
* `RTCPeerConnection` reaching `connected` and libp2p surfacing the connection.
|
|
42
45
|
*/
|
|
43
46
|
const TURN_CONSUME_WINDOW_MS = 1000;
|
|
47
|
+
/**
|
|
48
|
+
* How often a running strand re-resolves its siblings' strand-network addresses
|
|
49
|
+
* and re-merges them into its own libp2p address book
|
|
50
|
+
* ({@link CadreNode.refreshStrandPeerAddrs}). Overridable per node via
|
|
51
|
+
* `network.controlCohort.strandAddrRefreshMs`.
|
|
52
|
+
*
|
|
53
|
+
* Ten minutes sits comfortably inside the peerStore's one-hour address expiry
|
|
54
|
+
* (`MAX_ADDRESS_AGE`, see `peer-addr-book.ts`) with headroom for several missed
|
|
55
|
+
* passes, and far above the 15 s reconcile cadence the refresh rides on, so the
|
|
56
|
+
* strand-addr RPC fan-out stays cheap.
|
|
57
|
+
*/
|
|
58
|
+
export const STRAND_PEER_ADDR_REFRESH_MS = 10 * 60 * 1000;
|
|
44
59
|
/**
|
|
45
60
|
* A circuit relay as a strand-addr RPC target: dialed by peerId, with its
|
|
46
61
|
* direct addr riding along as the fallback for a relay we hold no connection to.
|
|
@@ -110,6 +125,14 @@ export class CadreNode {
|
|
|
110
125
|
this.pushFanoutService = null;
|
|
111
126
|
/** Backing field for the {@link running} / {@link isRunning} getters. */
|
|
112
127
|
this._running = false;
|
|
128
|
+
/**
|
|
129
|
+
* The control database's own raw storage (cache-wrapped), resolved once per
|
|
130
|
+
* `start()` and released in {@link cleanup}. Owning it is what keeps
|
|
131
|
+
* `config.storage.provider('control')` a once-per-runtime call: a `stop()` then
|
|
132
|
+
* `start()` cycle on this object re-resolves against a live cache, and never
|
|
133
|
+
* orphans the previous wrapper's registration in the shared cache pool.
|
|
134
|
+
*/
|
|
135
|
+
this.controlStorage = null;
|
|
113
136
|
/**
|
|
114
137
|
* In-flight {@link serviceWake} operations keyed by strandId. Coalesces
|
|
115
138
|
* concurrent on-demand wakes for the same strand into one runtime build + one
|
|
@@ -218,6 +241,16 @@ export class CadreNode {
|
|
|
218
241
|
* strand is no longer running are pruned on each reconcile pass.
|
|
219
242
|
*/
|
|
220
243
|
this.delegateAnnounceAt = new Map();
|
|
244
|
+
/**
|
|
245
|
+
* When each running strand last had its siblings' strand addresses re-resolved
|
|
246
|
+
* into its own address book, keyed by strandId. Throttles
|
|
247
|
+
* {@link refreshStrandPeerAddrs} to once per {@link STRAND_PEER_ADDR_REFRESH_MS}
|
|
248
|
+
* per strand so the 15 s reconcile tick never becomes per-tick RPC chatter.
|
|
249
|
+
* Entries for strands that are no longer running are pruned on each pass, so a
|
|
250
|
+
* hibernated-then-resumed strand refreshes immediately rather than inheriting a
|
|
251
|
+
* stale stamp.
|
|
252
|
+
*/
|
|
253
|
+
this.strandPeerAddrRefreshAt = new Map();
|
|
221
254
|
/**
|
|
222
255
|
* Cold-start bootstrap dial targets: the owner-flagged peers of every seed
|
|
223
256
|
* this node has applied, keyed by peer id, valued by the seed's multiaddr
|
|
@@ -805,6 +838,36 @@ export class CadreNode {
|
|
|
805
838
|
async createControlNode() {
|
|
806
839
|
return await createLibp2pNode(this.buildControlNodeOptions());
|
|
807
840
|
}
|
|
841
|
+
/**
|
|
842
|
+
* The control database's cache-wrapped raw storage, resolved from
|
|
843
|
+
* `config.storage.provider` on first use and then held in {@link controlStorage}
|
|
844
|
+
* for the rest of this runtime.
|
|
845
|
+
*
|
|
846
|
+
* Resolving once is the contract `RawStorageProvider` (types.ts) states: a provider is
|
|
847
|
+
* called once per scope per runtime lifetime. Re-entering it per call would mint a
|
|
848
|
+
* second store over one backend — a fresh, cold cache, with the previous wrapper's
|
|
849
|
+
* registration orphaned in the process-wide pool.
|
|
850
|
+
*
|
|
851
|
+
* Lazy rather than eager in `start()` so the pure-unit call path
|
|
852
|
+
* (`cadre-node-control-node-options.spec.ts` calls `buildControlNodeOptions` on a
|
|
853
|
+
* bare `new CadreNode`) still resolves — and, on that path too, only once.
|
|
854
|
+
*
|
|
855
|
+
* Wrapped in the write-through raw-storage cache (quereus-plugin-sereus's
|
|
856
|
+
* `cached-storage.ts`) because a control start's cost is its raw-storage
|
|
857
|
+
* operation count.
|
|
858
|
+
*/
|
|
859
|
+
resolveControlStorage() {
|
|
860
|
+
if (this.controlStorage) {
|
|
861
|
+
return this.controlStorage;
|
|
862
|
+
}
|
|
863
|
+
const provider = this.config.storage?.provider;
|
|
864
|
+
if (!provider) {
|
|
865
|
+
return undefined;
|
|
866
|
+
}
|
|
867
|
+
const resolved = typeof provider === 'function' ? provider('control') : provider;
|
|
868
|
+
this.controlStorage = wrapStorageWithCache(resolved, 'control');
|
|
869
|
+
return this.controlStorage;
|
|
870
|
+
}
|
|
808
871
|
/**
|
|
809
872
|
* Map this node's config onto the control network's libp2p node options.
|
|
810
873
|
*
|
|
@@ -814,19 +877,15 @@ export class CadreNode {
|
|
|
814
877
|
* only caller in production is `createControlNode`.
|
|
815
878
|
*/
|
|
816
879
|
buildControlNodeOptions() {
|
|
817
|
-
const { controlNetwork, network,
|
|
880
|
+
const { controlNetwork, network, profile } = this.config;
|
|
818
881
|
const identityKey = this.identityKey;
|
|
819
882
|
const listenAddrs = resolveListenAddrs(network);
|
|
820
883
|
// Determine relay mode: if explicitly set in config, use that;
|
|
821
884
|
// otherwise default to true for storage profile nodes (better connectivity/uptime)
|
|
822
885
|
const enableRelay = network?.enableRelay ?? (profile === 'storage');
|
|
823
|
-
//
|
|
824
|
-
//
|
|
825
|
-
const controlStorageProvider =
|
|
826
|
-
? (typeof storage.provider === 'function'
|
|
827
|
-
? storage.provider('control')
|
|
828
|
-
: storage.provider)
|
|
829
|
-
: undefined;
|
|
886
|
+
// The control node's own storage, resolved once per runtime (see
|
|
887
|
+
// {@link resolveControlStorage}) rather than per call.
|
|
888
|
+
const controlStorageProvider = this.resolveControlStorage();
|
|
830
889
|
const nodeOptions = {
|
|
831
890
|
port: 0,
|
|
832
891
|
bootstrapNodes: controlNetwork.bootstrapNodes,
|
|
@@ -1203,7 +1262,11 @@ export class CadreNode {
|
|
|
1203
1262
|
const existing = await this.controlDatabase.queryPeerRecord(peerId);
|
|
1204
1263
|
if (!existing) {
|
|
1205
1264
|
if (!this.seedBootstrapService) {
|
|
1206
|
-
|
|
1265
|
+
// "No row" also covers "revoked": queryPeerRecord reads a row whose StampId is
|
|
1266
|
+
// retired in Revocation as absent, so a removed node lands here every heartbeat
|
|
1267
|
+
// and stops refreshing its own record — correct, but say so, or the message reads
|
|
1268
|
+
// as "never added" for a node that was removed.
|
|
1269
|
+
log('registerSelf: no readable CadrePeer row for self (never added, or removed and its stamp retired) and no owner service to self-insert; skipping');
|
|
1207
1270
|
return 'skipped';
|
|
1208
1271
|
}
|
|
1209
1272
|
// First-time row: requires an owner signature (the node is its own
|
|
@@ -1225,9 +1288,10 @@ export class CadreNode {
|
|
|
1225
1288
|
// first publish", add a fourth SelfRegistrationOutcome rather than re-labelling.
|
|
1226
1289
|
log('registerSelf: own CadrePeer row appeared mid-publish (concurrent authorize); self-updating to carry the signature');
|
|
1227
1290
|
}
|
|
1228
|
-
//
|
|
1291
|
+
// Reached on the fall-through above, where the pre-race read missed the row a
|
|
1229
1292
|
// concurrent authorize then seated — so re-read to get that row's `UpdatedAt`, which
|
|
1230
|
-
// the strictly-greater self-update rule is measured against.
|
|
1293
|
+
// the strictly-greater self-update rule is measured against. The re-read can still
|
|
1294
|
+
// come back null (see the branch below).
|
|
1231
1295
|
//
|
|
1232
1296
|
// NOTE: when `existing` is set it predates this publish, so a removePeer(self) racing
|
|
1233
1297
|
// in would leave the UPDATE below matching no rows while still reporting 'refreshed'.
|
|
@@ -1235,7 +1299,13 @@ export class CadreNode {
|
|
|
1235
1299
|
// if removing self ever becomes something that happens concurrently in practice.
|
|
1236
1300
|
const current = existing ?? await this.controlDatabase.queryPeerRecord(peerId);
|
|
1237
1301
|
if (!current) {
|
|
1238
|
-
|
|
1302
|
+
// Two causes, and the second is a steady state rather than a race: the row really
|
|
1303
|
+
// vanished mid-publish (next refresh re-inserts), or this node was REVOKED — its row
|
|
1304
|
+
// is physically present, so insertCadrePeer's in-lock existence check no-op'd above,
|
|
1305
|
+
// but queryPeerRecord reads it as absent because its stamp is retired. The revoked
|
|
1306
|
+
// case repeats every heartbeat and never re-inserts; that is intended (a revoked node
|
|
1307
|
+
// has nothing to publish), so this stays a skip, not an error.
|
|
1308
|
+
log('registerSelf: own CadrePeer row not readable (vanished mid-publish, or this node is revoked and its stamp retired); skipping');
|
|
1239
1309
|
return 'skipped';
|
|
1240
1310
|
}
|
|
1241
1311
|
const record = this.signSelfRecord(peerId, signingKey, addrs, current);
|
|
@@ -1530,8 +1600,11 @@ export class CadreNode {
|
|
|
1530
1600
|
// than a local write) are picked up here, bounding the snapshot's staleness
|
|
1531
1601
|
// to the reconcile interval.
|
|
1532
1602
|
// NOTE: this refresh and the sibling enumeration below each run their own
|
|
1533
|
-
// CadrePeer query (two reads per pass)
|
|
1534
|
-
//
|
|
1603
|
+
// CadrePeer query (two reads per pass), plus a third from
|
|
1604
|
+
// `refreshStrandPeerAddrs` on the passes where a strand is due AND this node
|
|
1605
|
+
// holds a control connection (one read for the whole pass, not one per
|
|
1606
|
+
// strand); if those reads ever get costly, share one row-set across all
|
|
1607
|
+
// three.
|
|
1535
1608
|
await this.refreshMembershipGate('reconcile');
|
|
1536
1609
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1537
1610
|
return;
|
|
@@ -1544,6 +1617,14 @@ export class CadreNode {
|
|
|
1544
1617
|
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1545
1618
|
return;
|
|
1546
1619
|
}
|
|
1620
|
+
// Then re-warm each running strand's own address book from its siblings, on
|
|
1621
|
+
// its own (much longer) throttle. Same reasoning as warmSiblingAddrBook
|
|
1622
|
+
// below, one layer down: replication runs on the STRAND network, and every
|
|
1623
|
+
// layer under cadre-core dials strand peers by bare peer id.
|
|
1624
|
+
await this.refreshStrandPeerAddrs();
|
|
1625
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1626
|
+
return;
|
|
1627
|
+
}
|
|
1547
1628
|
const selfPeerId = this.controlNode.peerId.toString();
|
|
1548
1629
|
// Reap guarded rows this node still holds for incarnations an already-committed
|
|
1549
1630
|
// Revocation tombstone retires — the half of a removal replication cannot carry
|
|
@@ -1622,9 +1703,16 @@ export class CadreNode {
|
|
|
1622
1703
|
// Don't silently bound coverage — surface what the out-degree cap dropped.
|
|
1623
1704
|
log('reconcileControlCohort: capped %d non-owner sibling(s) at targetDegree=%d', cappedNonOwner, targetDegree);
|
|
1624
1705
|
}
|
|
1625
|
-
// 3.
|
|
1706
|
+
// 3. Warm the libp2p address book with EVERY sibling's verified addresses,
|
|
1707
|
+
// keeping what resolved for the dial loop below.
|
|
1708
|
+
const resolved = await this.warmSiblingAddrBook(siblings);
|
|
1709
|
+
if (!this._running || !this.controlNode || !this.controlDatabase) {
|
|
1710
|
+
return;
|
|
1711
|
+
}
|
|
1712
|
+
// 4. Skip already-connected peers (no re-dial / churn for live connections).
|
|
1626
1713
|
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
1627
|
-
//
|
|
1714
|
+
// 5. Dial each selected, not-yet-connected sibling, best-effort, from the
|
|
1715
|
+
// addresses step 3 already resolved for it.
|
|
1628
1716
|
let dialed = 0;
|
|
1629
1717
|
for (const sibling of dials) {
|
|
1630
1718
|
if (!this._running || !this.controlNode) {
|
|
@@ -1633,34 +1721,109 @@ export class CadreNode {
|
|
|
1633
1721
|
if (connected.has(sibling.peerId)) {
|
|
1634
1722
|
continue;
|
|
1635
1723
|
}
|
|
1636
|
-
if (await this.dialControlSibling(sibling)) {
|
|
1724
|
+
if (await this.dialControlSibling(sibling, resolved.get(sibling.peerId) ?? [])) {
|
|
1637
1725
|
dialed++;
|
|
1638
1726
|
}
|
|
1639
1727
|
}
|
|
1640
1728
|
log('reconcileControlCohort: pass complete (siblings=%d, selected=%d, dialed=%d)', siblings.length, dials.length, dialed);
|
|
1641
1729
|
}
|
|
1642
1730
|
/**
|
|
1643
|
-
* Resolve
|
|
1644
|
-
*
|
|
1731
|
+
* Resolve every sibling's signed address record once and merge what resolves
|
|
1732
|
+
* into the control node's libp2p **address book** (peerStore). Returns the
|
|
1733
|
+
* resolved addresses per sibling so the dial loop reuses them rather than
|
|
1734
|
+
* re-resolving — one `queryPeerRecord` per sibling per pass, not two.
|
|
1735
|
+
*
|
|
1736
|
+
* Deliberately every sibling, not the {@link selectControlCohortDials} subset,
|
|
1737
|
+
* and deliberately including already-connected ones: everything below
|
|
1738
|
+
* cadre-core dials by bare peer id (Optimystic's cluster/repo clients, FRET
|
|
1739
|
+
* ping/announce), so the address book has to be warm BEFORE a live connection
|
|
1740
|
+
* drops, and a sibling the out-degree cap declines to dial is still one those
|
|
1741
|
+
* layers may need to reach.
|
|
1645
1742
|
*
|
|
1646
|
-
*
|
|
1647
|
-
*
|
|
1648
|
-
*
|
|
1649
|
-
*
|
|
1743
|
+
* Only `resolvePeerAddrs` output is merged — never the cold-start
|
|
1744
|
+
* `peerStoreAddrs` fallback, which came out of the address book to begin with
|
|
1745
|
+
* and would restamp unverified seed addresses indefinitely. A sibling that
|
|
1746
|
+
* resolves to nothing (revoked, stale, untrusted) is not written at all, so its
|
|
1747
|
+
* existing entry ages out on its own.
|
|
1748
|
+
*
|
|
1749
|
+
* NOTE: this resolves EVERY sibling serially before the dial loop below runs,
|
|
1750
|
+
* so it costs one record query per sibling per reconcile pass (~15s) and each
|
|
1751
|
+
* one delays the pass's first dial. A cadre is a handful of devices, so today
|
|
1752
|
+
* that is a few extra local reads; if cadres ever grow large, batch the records
|
|
1753
|
+
* into one query, merge only on change, or move the warm pass after the dials.
|
|
1754
|
+
*/
|
|
1755
|
+
async warmSiblingAddrBook(siblings) {
|
|
1756
|
+
const resolved = new Map();
|
|
1757
|
+
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
1758
|
+
for (const sibling of siblings) {
|
|
1759
|
+
if (!this._running || !this.controlNode) {
|
|
1760
|
+
break;
|
|
1761
|
+
}
|
|
1762
|
+
const addrs = await this.resolveSiblingAddrs(sibling.peerId);
|
|
1763
|
+
resolved.set(sibling.peerId, addrs);
|
|
1764
|
+
// Re-guarded after the resolve await as well as before it: the write is
|
|
1765
|
+
// the half that matters, and a torn-down node must never be written to.
|
|
1766
|
+
const controlNode = this.controlNode;
|
|
1767
|
+
if (!this._running || !controlNode) {
|
|
1768
|
+
break;
|
|
1769
|
+
}
|
|
1770
|
+
// `mergePeerAddrs` owns the whole best-effort contract — including parsing
|
|
1771
|
+
// a malformed `CadrePeer.PeerId` — so one bad row cannot abort the pass.
|
|
1772
|
+
counts[await mergePeerAddrs(controlNode, sibling.peerId, addrs)]++;
|
|
1773
|
+
}
|
|
1774
|
+
log('reconcileControlCohort: address book warmed (siblings=%d, merged=%d, restamped=%d, skipped=%d, failed=%d)', resolved.size, counts.merged, counts.restamped, counts.skipped, counts.failed);
|
|
1775
|
+
return resolved;
|
|
1776
|
+
}
|
|
1777
|
+
/**
|
|
1778
|
+
* {@link resolvePeerAddrs} for one sibling, best-effort: a control-DB read
|
|
1779
|
+
* failure yields `[]` (and the cold-start fallback then gets its turn) rather
|
|
1780
|
+
* than aborting the whole pass, like every other step here.
|
|
1650
1781
|
*/
|
|
1651
|
-
async
|
|
1782
|
+
async resolveSiblingAddrs(peerId) {
|
|
1783
|
+
try {
|
|
1784
|
+
return await this.resolvePeerAddrs(peerId);
|
|
1785
|
+
}
|
|
1786
|
+
catch (error) {
|
|
1787
|
+
log('reconcileControlCohort: resolving addrs for %s failed (continuing): %o', peerId, error);
|
|
1788
|
+
return [];
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
/**
|
|
1792
|
+
* Budget for ONE proactive dial of the reconcile pass — steady-state sibling
|
|
1793
|
+
* ({@link dialControlSibling}) and cold-start bootstrap peer
|
|
1794
|
+
* ({@link dialBootstrapPeer}) alike, since both feed a whole address list to
|
|
1795
|
+
* one `dial()` and both run in a sequential loop that the next entry waits on.
|
|
1796
|
+
*
|
|
1797
|
+
* See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why the pass bounds
|
|
1798
|
+
* this itself rather than inheriting each address's libp2p attempt timeout.
|
|
1799
|
+
*/
|
|
1800
|
+
controlDialTimeoutMs() {
|
|
1801
|
+
return this.config.network?.controlCohort?.dialTimeoutMs
|
|
1802
|
+
?? DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS;
|
|
1803
|
+
}
|
|
1804
|
+
/**
|
|
1805
|
+
* Dial one sibling from its already-resolved addresses, best-effort. Returns
|
|
1806
|
+
* whether a dial was attempted (false when no address resolves).
|
|
1807
|
+
*
|
|
1808
|
+
* A per-peer failure (NAT, offline, relay down, connection-gater denial, or
|
|
1809
|
+
* the {@link controlDialTimeoutMs} budget expiring) is logged and swallowed so
|
|
1810
|
+
* one unreachable sibling never aborts the pass — exactly like
|
|
1811
|
+
* {@link SeedBootstrapService.applySeed}'s owner-dial loop. A failed dial is
|
|
1812
|
+
* simply retried on the next pass.
|
|
1813
|
+
*/
|
|
1814
|
+
async dialControlSibling(sibling, resolved) {
|
|
1652
1815
|
const controlNode = this.controlNode;
|
|
1653
1816
|
if (!controlNode) {
|
|
1654
1817
|
return false;
|
|
1655
1818
|
}
|
|
1656
|
-
const addrs = await this.resolveControlDialAddrs(sibling.peerId);
|
|
1819
|
+
const addrs = await this.resolveControlDialAddrs(sibling.peerId, resolved);
|
|
1657
1820
|
if (addrs.length === 0) {
|
|
1658
1821
|
log('reconcileControlCohort: no dialable control address for sibling %s; skipping', sibling.peerId);
|
|
1659
1822
|
return false;
|
|
1660
1823
|
}
|
|
1661
1824
|
try {
|
|
1662
1825
|
log('reconcileControlCohort: dialing sibling %s (%d addr(s))', sibling.peerId, addrs.length);
|
|
1663
|
-
await controlNode.dial(addrs);
|
|
1826
|
+
await withDeadline(this.controlDialTimeoutMs(), `reconcileControlCohort dial of sibling ${sibling.peerId}`, (signal) => controlNode.dial(addrs, { signal }));
|
|
1664
1827
|
return true;
|
|
1665
1828
|
}
|
|
1666
1829
|
catch (error) {
|
|
@@ -1669,16 +1832,16 @@ export class CadreNode {
|
|
|
1669
1832
|
}
|
|
1670
1833
|
}
|
|
1671
1834
|
/**
|
|
1672
|
-
*
|
|
1835
|
+
* A sibling's control-network dial addresses for the reconcile pass.
|
|
1673
1836
|
*
|
|
1674
|
-
* Primary (steady state): the signed, fresh, trust-gated control addresses
|
|
1675
|
-
*
|
|
1676
|
-
*
|
|
1677
|
-
*
|
|
1678
|
-
*
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1837
|
+
* Primary (steady state): the signed, fresh, trust-gated control addresses
|
|
1838
|
+
* `resolved` for it by {@link warmSiblingAddrBook} — passed in rather than
|
|
1839
|
+
* re-resolved, so the pass makes one record query per sibling. Fallback (cold
|
|
1840
|
+
* start): the libp2p peerStore entries `applySeed` populated, used only while
|
|
1841
|
+
* the record is not yet resolvable. Returns `[]` (never throws) when neither
|
|
1842
|
+
* yields an address — that sibling is skipped this pass.
|
|
1843
|
+
*/
|
|
1844
|
+
async resolveControlDialAddrs(peerId, resolved) {
|
|
1682
1845
|
if (resolved.length > 0) {
|
|
1683
1846
|
return resolved;
|
|
1684
1847
|
}
|
|
@@ -1839,7 +2002,7 @@ export class CadreNode {
|
|
|
1839
2002
|
}
|
|
1840
2003
|
try {
|
|
1841
2004
|
log('reconcileControlCohort(cold-start): dialing bootstrap peer %s (%d addr(s))', peerId, parsed.length);
|
|
1842
|
-
await controlNode.dial(parsed);
|
|
2005
|
+
await withDeadline(this.controlDialTimeoutMs(), `reconcileControlCohort cold-start dial of bootstrap peer ${peerId}`, (signal) => controlNode.dial(parsed, { signal }));
|
|
1843
2006
|
return true;
|
|
1844
2007
|
}
|
|
1845
2008
|
catch (error) {
|
|
@@ -2510,7 +2673,6 @@ export class CadreNode {
|
|
|
2510
2673
|
return;
|
|
2511
2674
|
}
|
|
2512
2675
|
try {
|
|
2513
|
-
// Discovery path: no explicit mode — infer from cohort membership.
|
|
2514
2676
|
await this.launchStrand(strand, sAppConfig);
|
|
2515
2677
|
}
|
|
2516
2678
|
catch (error) {
|
|
@@ -2611,6 +2773,16 @@ export class CadreNode {
|
|
|
2611
2773
|
await this.controlNode.stop();
|
|
2612
2774
|
this.controlNode = null;
|
|
2613
2775
|
}
|
|
2776
|
+
// Release the control store's cache registration LAST — after the database and
|
|
2777
|
+
// node that write through it are down. Dropping the field is what lets a
|
|
2778
|
+
// stop()/start() cycle re-resolve the provider against a live cache instead of
|
|
2779
|
+
// handing the restarted node a retired wrapper. Logged, never thrown: a
|
|
2780
|
+
// cache-bookkeeping failure must not abort a teardown.
|
|
2781
|
+
if (this.controlStorage) {
|
|
2782
|
+
const controlStorage = this.controlStorage;
|
|
2783
|
+
this.controlStorage = null;
|
|
2784
|
+
await disposeStorageCache(controlStorage).catch((err) => log('Failed to dispose control storage cache: %o', err));
|
|
2785
|
+
}
|
|
2614
2786
|
}
|
|
2615
2787
|
// Hibernation callbacks
|
|
2616
2788
|
async handleStrandIdle(strandId) {
|
|
@@ -2649,7 +2821,7 @@ export class CadreNode {
|
|
|
2649
2821
|
}
|
|
2650
2822
|
/**
|
|
2651
2823
|
* Wake a strand. If it was hibernating (quiesced — no libp2p node), re-resolve
|
|
2652
|
-
* the cohort discovery seed
|
|
2824
|
+
* the cohort discovery seed exactly as `launchStrand` does and rebuild
|
|
2653
2825
|
* its runtime via the strand manager. If it is still live (e.g. waking an idle
|
|
2654
2826
|
* strand, which retains its resources), just flip the status. Overlapping wake
|
|
2655
2827
|
* triggers are coalesced upstream by `HibernationManager`, so this runs once
|
|
@@ -2669,8 +2841,8 @@ export class CadreNode {
|
|
|
2669
2841
|
this.emit('strand:waking', { strandId });
|
|
2670
2842
|
return;
|
|
2671
2843
|
}
|
|
2672
|
-
// Quiesced: re-resolve volatile cohort
|
|
2673
|
-
//
|
|
2844
|
+
// Quiesced: re-resolve the volatile cohort input (the seed may have grown)
|
|
2845
|
+
// and rebuild the runtime.
|
|
2674
2846
|
log('Waking strand %s — rebuilding strand-network resources', strandId);
|
|
2675
2847
|
await this.resumeStrandRuntime(strandId);
|
|
2676
2848
|
instance.lastActivity = new Date();
|
|
@@ -2678,11 +2850,10 @@ export class CadreNode {
|
|
|
2678
2850
|
log('Strand %s awake (resources rebuilt)', strandId);
|
|
2679
2851
|
}
|
|
2680
2852
|
/**
|
|
2681
|
-
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort
|
|
2682
|
-
* first: the discovery seed may have grown
|
|
2683
|
-
* shifted the mode `bootstrap → networked` since the strand last ran. Shared by
|
|
2853
|
+
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort input
|
|
2854
|
+
* first: the discovery seed may have grown since the strand last ran. Shared by
|
|
2684
2855
|
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so
|
|
2685
|
-
* both apply the same fresh seed
|
|
2856
|
+
* both apply the same fresh seed resolution. `resumeStrand` is idempotent
|
|
2686
2857
|
* (returns the live instance unchanged) as a backstop against double-resume.
|
|
2687
2858
|
*/
|
|
2688
2859
|
async resumeStrandRuntime(strandId) {
|
|
@@ -2692,12 +2863,13 @@ export class CadreNode {
|
|
|
2692
2863
|
const delegatePeerId = this.identityKey
|
|
2693
2864
|
? peerIdFromPrivateKey(await strandTransportKey(this.identityKey, strandId)).toString()
|
|
2694
2865
|
: undefined;
|
|
2695
|
-
const
|
|
2696
|
-
const
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2866
|
+
const bootstrapNodes = await this.resolveCohortSeed(strandId, delegatePeerId);
|
|
2867
|
+
const instance = await this.strandManager.resumeStrand(strandId, { bootstrapNodes });
|
|
2868
|
+
// Same reason as the launch path: `bootstrapNodes` only reaches the address
|
|
2869
|
+
// book through @libp2p/bootstrap discovery, so merge it directly as well.
|
|
2870
|
+
if (instance.libp2pNode) {
|
|
2871
|
+
await this.mergeStrandPeerAddrs(instance.libp2pNode, bootstrapNodes, strandId);
|
|
2872
|
+
}
|
|
2701
2873
|
}
|
|
2702
2874
|
/**
|
|
2703
2875
|
* Real cohort check-in for a hibernating strand (the `onCheckIn` callback).
|
|
@@ -2708,7 +2880,7 @@ export class CadreNode {
|
|
|
2708
2880
|
* realized as a resume → bounded window → re-hibernate-if-idle cycle that
|
|
2709
2881
|
* reuses the existing quiesce/resume primitives rather than a bespoke probe:
|
|
2710
2882
|
*
|
|
2711
|
-
* 1. Resume the strand (rebuild node + db, re-resolve cohort seed
|
|
2883
|
+
* 1. Resume the strand (rebuild node + db, re-resolve the cohort seed) so
|
|
2712
2884
|
* its strand network can reach cohort peers — exactly as a wake does.
|
|
2713
2885
|
* 2. Hold it resumed for a bounded window, during which the app may drive
|
|
2714
2886
|
* reads (pull-on-read) and record activity.
|
|
@@ -2732,7 +2904,7 @@ export class CadreNode {
|
|
|
2732
2904
|
}
|
|
2733
2905
|
try {
|
|
2734
2906
|
// 1. Resume exactly as a wake does: re-resolve the (possibly grown) cohort
|
|
2735
|
-
// seed
|
|
2907
|
+
// seed, then rebuild the runtime.
|
|
2736
2908
|
log('Check-in: resuming strand %s to probe the cohort for pending activity', strandId);
|
|
2737
2909
|
await this.resumeStrandRuntime(strandId);
|
|
2738
2910
|
// 2-3. Bounded window for the strand network to connect + the app to act,
|
|
@@ -2837,15 +3009,13 @@ export class CadreNode {
|
|
|
2837
3009
|
if (!this._running) {
|
|
2838
3010
|
throw new Error('CadreNode not running');
|
|
2839
3011
|
}
|
|
2840
|
-
const { strandRow, sAppConfig,
|
|
3012
|
+
const { strandRow, sAppConfig, founder } = config;
|
|
2841
3013
|
// Store sApp config for this strand
|
|
2842
3014
|
this.sAppConfigs.set(strandRow.Id, sAppConfig);
|
|
2843
|
-
log('Registered sAppConfig for strand %s (sApp: %s,
|
|
2844
|
-
// Pass `mode` (possibly undefined) through: an explicit mode wins, while a
|
|
2845
|
-
// caller that omits it gets the same cohort-inferred mode as the discovery path.
|
|
3015
|
+
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)', strandRow.Id, sAppConfig.id, founder ?? false);
|
|
2846
3016
|
// `founder` only flows from the explicit addStrand path — the control-discovered
|
|
2847
3017
|
// join path never founds, so its rows arrive via sync (see handleStrandAdded).
|
|
2848
|
-
return await this.launchStrand(strandRow, sAppConfig,
|
|
3018
|
+
return await this.launchStrand(strandRow, sAppConfig, founder);
|
|
2849
3019
|
}
|
|
2850
3020
|
/**
|
|
2851
3021
|
* Publish a strand row to the shared control database under this node's own
|
|
@@ -3081,8 +3251,8 @@ export class CadreNode {
|
|
|
3081
3251
|
/**
|
|
3082
3252
|
* Shared strand launch path for both the explicit (`addStrand`) and the
|
|
3083
3253
|
* control-discovered (`handleStrandAdded`) entry points. Resolves the cohort
|
|
3084
|
-
* seed,
|
|
3085
|
-
*
|
|
3254
|
+
* seed, starts the strand, and registers it with the hibernation manager
|
|
3255
|
+
* before emitting `strand:started`.
|
|
3086
3256
|
*
|
|
3087
3257
|
* Idempotent no-op when the strand manager already tracks `strand.Id` — the
|
|
3088
3258
|
* ordinary founding sequence is `addStrand` (starts it locally) followed by
|
|
@@ -3093,7 +3263,7 @@ export class CadreNode {
|
|
|
3093
3263
|
* instance that never stopped. The guard runs before the cohort-seed RPC
|
|
3094
3264
|
* fan-out, so a rediscovery costs nothing beyond the map lookup.
|
|
3095
3265
|
*/
|
|
3096
|
-
async launchStrand(strand, sAppConfig,
|
|
3266
|
+
async launchStrand(strand, sAppConfig, founder) {
|
|
3097
3267
|
const existing = this.strandManager.getInstance(strand.Id);
|
|
3098
3268
|
if (existing) {
|
|
3099
3269
|
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
@@ -3122,8 +3292,7 @@ export class CadreNode {
|
|
|
3122
3292
|
// `libp2p.start()` (the responder records the grant before replying; the
|
|
3123
3293
|
// client awaits the replies).
|
|
3124
3294
|
const delegatePeerId = transportKey ? peerIdFromPrivateKey(transportKey).toString() : undefined;
|
|
3125
|
-
const
|
|
3126
|
-
const mode = selectStrandMode(explicitMode, seed.hasOtherPeers);
|
|
3295
|
+
const bootstrapNodes = await this.resolveCohortSeed(strand.Id, delegatePeerId);
|
|
3127
3296
|
const instance = await this.strandManager.startStrand({
|
|
3128
3297
|
strandRow: strand,
|
|
3129
3298
|
sAppConfig,
|
|
@@ -3132,30 +3301,37 @@ export class CadreNode {
|
|
|
3132
3301
|
profile: this.config.profile,
|
|
3133
3302
|
defaultLatencyHint: this.config.hibernation?.defaultLatencyHint ?? 'interactive',
|
|
3134
3303
|
privateKey: transportKey,
|
|
3135
|
-
bootstrapNodes
|
|
3136
|
-
mode,
|
|
3304
|
+
bootstrapNodes,
|
|
3137
3305
|
requireSignedSchemas: this.config.requireSignedSchemas,
|
|
3138
3306
|
clusterSize: this.config.strandClusterSize,
|
|
3139
3307
|
backfill: this.config.strandBackfill,
|
|
3140
3308
|
founder
|
|
3141
3309
|
});
|
|
3310
|
+
// The seed reached the node as `bootstrapNodes`, which only enters the
|
|
3311
|
+
// address book via @libp2p/bootstrap discovery. Merge it directly too, so a
|
|
3312
|
+
// sibling is dialable by bare peer id from the first moment (see
|
|
3313
|
+
// mergeStrandPeerAddrs; refreshStrandPeerAddrs keeps it warm from here on).
|
|
3314
|
+
if (instance.libp2pNode) {
|
|
3315
|
+
await this.mergeStrandPeerAddrs(instance.libp2pNode, bootstrapNodes, strand.Id);
|
|
3316
|
+
}
|
|
3142
3317
|
this.hibernationManager.trackStrand(instance);
|
|
3143
3318
|
this.emit('strand:started', { strandId: strand.Id });
|
|
3144
3319
|
return instance;
|
|
3145
3320
|
}
|
|
3146
3321
|
/**
|
|
3147
|
-
* Resolve a strand's discovery seed
|
|
3148
|
-
*
|
|
3149
|
-
*
|
|
3150
|
-
*
|
|
3151
|
-
* addresses that must not
|
|
3322
|
+
* Resolve a strand's discovery seed — the dialable strand-network multiaddr
|
|
3323
|
+
* strings for cohort siblings. Membership comes from the control network's
|
|
3324
|
+
* CadrePeer rows; the **strand-network** bootstrap addresses are resolved on
|
|
3325
|
+
* demand over the control mesh via the strand-addr RPC — deliberately NOT
|
|
3326
|
+
* from `CadrePeer.Multiaddr`, which carries *control* addresses that must not
|
|
3327
|
+
* seed the strand mesh.
|
|
3152
3328
|
*
|
|
3153
3329
|
* Only siblings we already hold an open control connection to are RPC'd: they
|
|
3154
3330
|
* are the ones that can answer right now, and dialing them by peerId reuses the
|
|
3155
3331
|
* live connection. When no connected sibling yet runs the strand the seed is
|
|
3156
|
-
* empty (`[]`) —
|
|
3157
|
-
*
|
|
3158
|
-
*
|
|
3332
|
+
* empty (`[]`) — the empty seed self-heals on the next resume / check-in pass.
|
|
3333
|
+
* Returns an empty seed when the control DB or node is absent (not yet
|
|
3334
|
+
* started / torn down).
|
|
3159
3335
|
*
|
|
3160
3336
|
* When `delegatePeerId` is given (a strand launch/resume is imminent), the
|
|
3161
3337
|
* pass doubles as the delegate ANNOUNCEMENT: our circuit relays are merged
|
|
@@ -3170,16 +3346,9 @@ export class CadreNode {
|
|
|
3170
3346
|
*/
|
|
3171
3347
|
async resolveCohortSeed(strandId, delegatePeerId) {
|
|
3172
3348
|
if (!this.controlDatabase || !this.controlNode) {
|
|
3173
|
-
return
|
|
3349
|
+
return [];
|
|
3174
3350
|
}
|
|
3175
|
-
const
|
|
3176
|
-
const { otherPeerIds, hasOtherPeers } = deriveCohortMembers(peers, this.controlNode.peerId.toString());
|
|
3177
|
-
// RPC only siblings we already have a control connection to — they can answer
|
|
3178
|
-
// now, and `dialProtocol` by peerId reuses the open connection.
|
|
3179
|
-
const connected = new Set(this.controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3180
|
-
const targets = otherPeerIds
|
|
3181
|
-
.filter((id) => connected.has(id))
|
|
3182
|
-
.map((peerId) => ({ peerId }));
|
|
3351
|
+
const targets = await this.connectedSiblingTargets();
|
|
3183
3352
|
const relays = delegatePeerId === undefined ? [] : this.circuitRelayTargets();
|
|
3184
3353
|
for (const relay of relays) {
|
|
3185
3354
|
if (!targets.some((t) => t.peerId === relay.relayPeerId)) {
|
|
@@ -3192,7 +3361,48 @@ export class CadreNode {
|
|
|
3192
3361
|
// Throttle state for the RELAY targets only — refreshDelegateGrants never
|
|
3193
3362
|
// looks up a sibling key, so recording one would only be dead weight.
|
|
3194
3363
|
this.recordDelegateAnnounces(relays.map((r) => r.relayPeerId), strandId);
|
|
3195
|
-
return
|
|
3364
|
+
return bootstrapNodes;
|
|
3365
|
+
}
|
|
3366
|
+
/**
|
|
3367
|
+
* The co-cadre siblings worth sending a strand-addr RPC to right now: cohort
|
|
3368
|
+
* members (self excluded) we already hold an open control connection to. They
|
|
3369
|
+
* are the ones that can answer immediately, and dialing them by peerId reuses
|
|
3370
|
+
* the live connection instead of opening a second one.
|
|
3371
|
+
*
|
|
3372
|
+
* Shared by {@link resolveCohortSeed} (launch/resume) and
|
|
3373
|
+
* {@link refreshStrandPeerAddrs} (the periodic pass) so both ask the same set.
|
|
3374
|
+
* Empty when the control DB or node is absent (not yet started / torn down).
|
|
3375
|
+
*
|
|
3376
|
+
* The read below is issued even when the node holds no control connection and
|
|
3377
|
+
* the answer is therefore empty by construction. That is deliberate on the
|
|
3378
|
+
* launch path: `control-database-solo-warm-start.spec.ts` exists to prove this
|
|
3379
|
+
* exact read does not stall as an embedder's FIRST awaited control operation,
|
|
3380
|
+
* on a warm cohort no one can reach. A caller that re-enters often enough for
|
|
3381
|
+
* the read to matter should skip the call itself, as
|
|
3382
|
+
* {@link refreshStrandPeerAddrs} does.
|
|
3383
|
+
*
|
|
3384
|
+
* NOTE: this read is UNBOUNDED and sits on the critical path an embedding app
|
|
3385
|
+
* awaits during startup — `addStrand` cannot resolve until it does, and it is
|
|
3386
|
+
* the first control operation an embedder's boot order issues (before genesis,
|
|
3387
|
+
* before seed bootstrap). Measured fine today: the warm-start-alone shape a
|
|
3388
|
+
* report pointed at — a `CadrePeer` list naming peers that are all gone, read
|
|
3389
|
+
* off real files after a restart — completes in milliseconds, and
|
|
3390
|
+
* `control-database-solo-warm-start.spec.ts` covers it under a deadline. If a
|
|
3391
|
+
* control read ever CAN stall (a transactor change that consults the network
|
|
3392
|
+
* for a local row, a storage backend with blocking I/O), give this one a
|
|
3393
|
+
* timeout — and decide then whether the breach fails `addStrand` or degrades
|
|
3394
|
+
* to an empty seed, because those promise callers different things.
|
|
3395
|
+
*/
|
|
3396
|
+
async connectedSiblingTargets() {
|
|
3397
|
+
const controlNode = this.controlNode;
|
|
3398
|
+
if (!this.controlDatabase || !controlNode) {
|
|
3399
|
+
return [];
|
|
3400
|
+
}
|
|
3401
|
+
const peers = await this.controlDatabase.queryCadrePeers();
|
|
3402
|
+
const connected = new Set(controlNode.getConnections().map((c) => c.remotePeer.toString()));
|
|
3403
|
+
return deriveCohortMembers(peers, controlNode.peerId.toString())
|
|
3404
|
+
.filter((id) => connected.has(id))
|
|
3405
|
+
.map((peerId) => ({ peerId }));
|
|
3196
3406
|
}
|
|
3197
3407
|
/**
|
|
3198
3408
|
* The circuit relays this node's strand nodes would reserve through: the
|
|
@@ -3277,6 +3487,157 @@ export class CadreNode {
|
|
|
3277
3487
|
await collectStrandAddrs(controlNode, due.map(relayStrandAddrPeer), strandId, { delegatePeerId });
|
|
3278
3488
|
this.recordDelegateAnnounces(due.map((relay) => relay.relayPeerId), strandId, now);
|
|
3279
3489
|
}
|
|
3490
|
+
/**
|
|
3491
|
+
* Re-resolve each running strand's SIBLING addresses over the control mesh and
|
|
3492
|
+
* merge them into that strand's own libp2p address book, throttled to once per
|
|
3493
|
+
* {@link STRAND_PEER_ADDR_REFRESH_MS} per strand.
|
|
3494
|
+
*
|
|
3495
|
+
* Without this, a strand's address book is written exactly once — the
|
|
3496
|
+
* launch/resume seed — and everything below cadre-core that dials a strand peer
|
|
3497
|
+
* by bare peer id (Optimystic's cluster and repo clients, FRET ping/announce)
|
|
3498
|
+
* loses its address for any sibling it is not currently connected to: a sibling
|
|
3499
|
+
* that restarted its strand node or rotated its relay reservation is never
|
|
3500
|
+
* re-resolved, and even the original seed addresses fall off at the peerStore's
|
|
3501
|
+
* one-hour expiry (see `peer-addr-book.ts`).
|
|
3502
|
+
*
|
|
3503
|
+
* Distinct from {@link refreshDelegateGrants}, deliberately: that pass covers
|
|
3504
|
+
* RELAYS on a `DELEGATE_GRANT_TTL_MS / 2` throttle to keep circuit-relay
|
|
3505
|
+
* admission grants alive, this one covers SIBLINGS on an address-expiry
|
|
3506
|
+
* throttle to keep the address book warm. They overlap only in that both carry
|
|
3507
|
+
* `delegatePeerId`, so a sibling that also runs a relay gets its grant
|
|
3508
|
+
* refreshed here as a side effect. Merging them would tie an admission-grant
|
|
3509
|
+
* TTL to an address-expiry window that has nothing to do with it.
|
|
3510
|
+
*
|
|
3511
|
+
* Strands refresh CONCURRENTLY, like `refreshDelegateGrants`, so one
|
|
3512
|
+
* unreachable sibling's dial timeout does not stack up per strand.
|
|
3513
|
+
*/
|
|
3514
|
+
async refreshStrandPeerAddrs(now = Date.now()) {
|
|
3515
|
+
if (!this._running || !this.controlNode) {
|
|
3516
|
+
return;
|
|
3517
|
+
}
|
|
3518
|
+
// A hibernating / quiescing strand has no node to seed, so it is not running
|
|
3519
|
+
// for this pass's purposes even though its instance is still tracked.
|
|
3520
|
+
const running = new Set();
|
|
3521
|
+
for (const [strandId, instance] of this.strandManager.getInstances()) {
|
|
3522
|
+
if (instance.libp2pNode) {
|
|
3523
|
+
running.add(strandId);
|
|
3524
|
+
}
|
|
3525
|
+
}
|
|
3526
|
+
// Drop stamps for strands no longer running: the map must not grow for the
|
|
3527
|
+
// node's lifetime, and a resumed strand should refresh immediately rather
|
|
3528
|
+
// than inherit the stamp its previous incarnation left.
|
|
3529
|
+
for (const strandId of this.strandPeerAddrRefreshAt.keys()) {
|
|
3530
|
+
if (!running.has(strandId)) {
|
|
3531
|
+
this.strandPeerAddrRefreshAt.delete(strandId);
|
|
3532
|
+
}
|
|
3533
|
+
}
|
|
3534
|
+
const refreshMs = this.config.network?.controlCohort?.strandAddrRefreshMs ?? STRAND_PEER_ADDR_REFRESH_MS;
|
|
3535
|
+
const due = [...running].filter((strandId) => now - (this.strandPeerAddrRefreshAt.get(strandId) ?? 0) >= refreshMs);
|
|
3536
|
+
if (due.length === 0) {
|
|
3537
|
+
return;
|
|
3538
|
+
}
|
|
3539
|
+
// A due strand with nobody to ask leaves its throttle unstamped (see below),
|
|
3540
|
+
// so this pass re-enters on EVERY 15 s reconcile tick for as long as the node
|
|
3541
|
+
// is alone — which is the steady state of a solo node running a strand.
|
|
3542
|
+
// `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
3543
|
+
// connections its answer is empty whatever the table holds, so decide it here
|
|
3544
|
+
// from the connection list instead of paying for the read once a tick.
|
|
3545
|
+
if (this.controlNode.getConnections().length === 0) {
|
|
3546
|
+
return;
|
|
3547
|
+
}
|
|
3548
|
+
// NOTE: one strand-addr RPC per (due strand × connected sibling) per refresh
|
|
3549
|
+
// interval — each a tiny request/response on an already-open control
|
|
3550
|
+
// connection. If a node ever runs MANY strands at once, batch the RPC to
|
|
3551
|
+
// carry several strand ids per request rather than one fan-out per strand.
|
|
3552
|
+
const targets = await this.connectedSiblingTargets().catch((error) => {
|
|
3553
|
+
log('refreshStrandPeerAddrs: sibling enumeration failed (skipping pass): %o', error);
|
|
3554
|
+
return [];
|
|
3555
|
+
});
|
|
3556
|
+
// No sibling to ask (or a shutdown landed mid-enumeration): leave every stamp
|
|
3557
|
+
// untouched so the next reconcile tick retries, rather than sitting out the
|
|
3558
|
+
// whole refresh interval having asked nobody.
|
|
3559
|
+
if (targets.length === 0 || !this._running || !this.controlNode) {
|
|
3560
|
+
return;
|
|
3561
|
+
}
|
|
3562
|
+
await Promise.all(due.map((strandId) => this.refreshOneStrandPeerAddrs(strandId, targets, now)));
|
|
3563
|
+
}
|
|
3564
|
+
/**
|
|
3565
|
+
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings, then
|
|
3566
|
+
* merge their answers into this strand's address book. Errors are logged and
|
|
3567
|
+
* swallowed so one strand's failure never costs the others their refresh.
|
|
3568
|
+
*/
|
|
3569
|
+
async refreshOneStrandPeerAddrs(strandId, targets, now) {
|
|
3570
|
+
const controlNode = this.controlNode;
|
|
3571
|
+
const strandNode = this.strandManager.getInstance(strandId)?.libp2pNode;
|
|
3572
|
+
if (!controlNode || !strandNode) {
|
|
3573
|
+
return;
|
|
3574
|
+
}
|
|
3575
|
+
try {
|
|
3576
|
+
// The running strand node's own peerId is the delegate to announce — see
|
|
3577
|
+
// the relationship note on refreshStrandPeerAddrs.
|
|
3578
|
+
const addrs = await collectStrandAddrs(controlNode, [...targets], strandId, {
|
|
3579
|
+
delegatePeerId: strandNode.peerId.toString()
|
|
3580
|
+
});
|
|
3581
|
+
// Stamped on the RPC having happened, not on its answer: the fan-out is
|
|
3582
|
+
// what the throttle exists to bound, and an all-empty round is a normal
|
|
3583
|
+
// steady state for a strand no connected sibling currently runs.
|
|
3584
|
+
this.strandPeerAddrRefreshAt.set(strandId, now);
|
|
3585
|
+
// Re-read the instance after the await — a strand stopped mid-pass (or one
|
|
3586
|
+
// already restarted onto a new node) must never have its store written to.
|
|
3587
|
+
if (!this._running || this.strandManager.getInstance(strandId)?.libp2pNode !== strandNode) {
|
|
3588
|
+
return;
|
|
3589
|
+
}
|
|
3590
|
+
await this.mergeStrandPeerAddrs(strandNode, addrs, strandId);
|
|
3591
|
+
}
|
|
3592
|
+
catch (error) {
|
|
3593
|
+
log('refreshStrandPeerAddrs: strand %s refresh failed (continuing): %o', strandId, error);
|
|
3594
|
+
}
|
|
3595
|
+
}
|
|
3596
|
+
/**
|
|
3597
|
+
* Merge strand-network addresses into ONE strand node's libp2p address book,
|
|
3598
|
+
* attributed per peer. Best-effort throughout: an address-book write must never
|
|
3599
|
+
* fail a launch, a resume, or a reconcile pass.
|
|
3600
|
+
*
|
|
3601
|
+
* `addrs` is the peer-agnostic union the strand-addr RPC returns, so
|
|
3602
|
+
* {@link groupAddrsByPeerId} attributes each entry to the **strand transport**
|
|
3603
|
+
* peerId in its final `/p2p/` component — never the sibling's control peerId,
|
|
3604
|
+
* which names a different libp2p node entirely. Entries that name no peer are
|
|
3605
|
+
* dropped there and counted here, once per pass.
|
|
3606
|
+
*
|
|
3607
|
+
* NOTE: a member could answer with arbitrary multiaddrs bound to arbitrary peer
|
|
3608
|
+
* ids and poison this address book. That is the same exposure the launch-time
|
|
3609
|
+
* `bootstrapNodes` seeding already accepts, and the cost is bounded: an address
|
|
3610
|
+
* grants no authority, the dialed peer authenticates by peer id at the
|
|
3611
|
+
* handshake, and a bad entry costs one failed dial that ages out at the
|
|
3612
|
+
* peerStore's one-hour expiry. No new gating here — cross-party strand trust is
|
|
3613
|
+
* `backlog/strand-network-nat-relay-reachability`.
|
|
3614
|
+
*/
|
|
3615
|
+
async mergeStrandPeerAddrs(strandNode, addrs, strandId) {
|
|
3616
|
+
const counts = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
3617
|
+
let grouped = 0;
|
|
3618
|
+
let peers = 0;
|
|
3619
|
+
try {
|
|
3620
|
+
const groups = groupAddrsByPeerId(addrs);
|
|
3621
|
+
peers = groups.size;
|
|
3622
|
+
const selfId = strandNode.peerId.toString();
|
|
3623
|
+
for (const [peerId, peerAddrs] of groups) {
|
|
3624
|
+
grouped += peerAddrs.length;
|
|
3625
|
+
// We never RPC ourselves, so self should never appear — belt and braces:
|
|
3626
|
+
// a node must not write its own addresses into its own address book.
|
|
3627
|
+
if (peerId === selfId) {
|
|
3628
|
+
continue;
|
|
3629
|
+
}
|
|
3630
|
+
counts[await mergePeerAddrs(strandNode, peerId, peerAddrs)]++;
|
|
3631
|
+
}
|
|
3632
|
+
}
|
|
3633
|
+
catch (error) {
|
|
3634
|
+
// `mergePeerAddrs` folds its own failures, so reaching here means the node
|
|
3635
|
+
// itself is unusable (torn down mid-merge). Never fail a launch/resume or a
|
|
3636
|
+
// reconcile pass over an address book.
|
|
3637
|
+
log('strand %s address book merge failed (continuing): %o', strandId, error);
|
|
3638
|
+
}
|
|
3639
|
+
log('strand %s address book merged (peers=%d, merged=%d, restamped=%d, skipped=%d, failed=%d, dropped=%d)', strandId, peers, counts.merged, counts.restamped, counts.skipped, counts.failed, addrs.length - grouped);
|
|
3640
|
+
}
|
|
3280
3641
|
/**
|
|
3281
3642
|
* The local strand instance's dialable strand-network multiaddrs for the
|
|
3282
3643
|
* strand-addr RPC, ordered signaling-first (reusing the control node's
|