@serfab/cadre-core 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/announce-addrs.d.ts +49 -0
- package/dist/announce-addrs.js +78 -0
- package/dist/announce-addrs.js.map +1 -0
- package/dist/cadre-node.d.ts +607 -53
- package/dist/cadre-node.js +1268 -157
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-database.d.ts +174 -23
- package/dist/control-database.js +317 -107
- package/dist/control-database.js.map +1 -1
- package/dist/control-read-retry.d.ts +103 -0
- package/dist/control-read-retry.js +145 -0
- package/dist/control-read-retry.js.map +1 -0
- package/dist/control-retry.d.ts +101 -0
- package/dist/control-retry.js +108 -0
- package/dist/control-retry.js.map +1 -0
- package/dist/control-schema.d.ts +1 -1
- package/dist/control-schema.js +50 -6
- package/dist/control-schema.js.map +1 -1
- package/dist/control-write-retry.d.ts +59 -35
- package/dist/control-write-retry.js +21 -92
- package/dist/control-write-retry.js.map +1 -1
- package/dist/delegate-admission.d.ts +17 -15
- package/dist/delegate-admission.js +17 -15
- package/dist/delegate-admission.js.map +1 -1
- package/dist/enrolled-machine-store-file.d.ts +13 -0
- package/dist/enrolled-machine-store-file.js +41 -0
- package/dist/enrolled-machine-store-file.js.map +1 -0
- package/dist/enrolled-machine-store.d.ts +81 -0
- package/dist/enrolled-machine-store.js +225 -0
- package/dist/enrolled-machine-store.js.map +1 -0
- package/dist/file-durable-slot.js +5 -3
- package/dist/file-durable-slot.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.js +39 -12
- package/dist/index.js.map +1 -1
- package/dist/membership-connection-gater.d.ts +226 -26
- package/dist/membership-connection-gater.js +316 -30
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/node-local-snapshot.d.ts +6 -0
- package/dist/node-local-snapshot.js +7 -1
- package/dist/node-local-snapshot.js.map +1 -1
- package/dist/{strand-backfill.d.ts → peer-join-backfill.d.ts} +49 -20
- package/dist/{strand-backfill.js → peer-join-backfill.js} +146 -56
- package/dist/peer-join-backfill.js.map +1 -0
- package/dist/peer-record.d.ts +26 -0
- package/dist/peer-record.js +32 -0
- package/dist/peer-record.js.map +1 -1
- package/dist/relay-addrs.d.ts +145 -22
- package/dist/relay-addrs.js +314 -31
- package/dist/relay-addrs.js.map +1 -1
- package/dist/relay-reservation.d.ts +20 -15
- package/dist/relay-reservation.js +18 -14
- package/dist/relay-reservation.js.map +1 -1
- package/dist/strand-addr-protocol.js +16 -0
- package/dist/strand-addr-protocol.js.map +1 -1
- package/dist/strand-database.d.ts +12 -0
- package/dist/strand-database.js +16 -0
- package/dist/strand-database.js.map +1 -1
- package/dist/strand-formation-manager.d.ts +22 -9
- package/dist/strand-formation-manager.js +20 -12
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-formation-protocol.d.ts +83 -9
- package/dist/strand-formation-protocol.js +86 -10
- package/dist/strand-formation-protocol.js.map +1 -1
- package/dist/strand-instance-manager.d.ts +151 -17
- package/dist/strand-instance-manager.js +237 -32
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +118 -14
- package/dist/strand-membership-writer.js +203 -15
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +84 -0
- package/dist/strand-network-config.js +136 -0
- package/dist/strand-network-config.js.map +1 -0
- package/dist/strand-revocation-enforcer.d.ts +392 -0
- package/dist/strand-revocation-enforcer.js +533 -0
- package/dist/strand-revocation-enforcer.js.map +1 -0
- package/dist/strand-solicitation.d.ts +10 -3
- package/dist/strand-solicitation.js +12 -7
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-wake-protocol.d.ts +47 -2
- package/dist/strand-wake-protocol.js +76 -6
- package/dist/strand-wake-protocol.js.map +1 -1
- package/dist/strand-watcher.js +6 -1
- package/dist/strand-watcher.js.map +1 -1
- package/dist/types.d.ts +282 -43
- package/dist/types.js +7 -1
- package/dist/types.js.map +1 -1
- package/package.json +16 -8
- package/src/announce-addrs.ts +93 -0
- package/src/cadre-node.ts +1366 -171
- package/src/control-database.ts +2628 -2406
- package/src/control-read-retry.ts +167 -0
- package/src/control-retry.ts +190 -0
- package/src/control-schema.ts +50 -6
- package/src/control-write-retry.ts +30 -130
- package/src/delegate-admission.ts +17 -15
- package/src/enrolled-machine-store-file.ts +43 -0
- package/src/enrolled-machine-store.ts +280 -0
- package/src/file-durable-slot.ts +5 -3
- package/src/index.ts +477 -407
- package/src/membership-connection-gater.ts +539 -204
- package/src/node-local-snapshot.ts +13 -1
- package/src/peer-addr-book.ts +251 -250
- package/src/{strand-backfill.ts → peer-join-backfill.ts} +187 -75
- package/src/peer-record.ts +33 -0
- package/src/relay-addrs.ts +436 -111
- package/src/relay-reservation.ts +828 -823
- package/src/strand-addr-protocol.ts +17 -0
- package/src/strand-database.ts +17 -0
- package/src/strand-formation-manager.ts +511 -491
- package/src/strand-formation-protocol.ts +808 -692
- package/src/strand-instance-manager.ts +899 -611
- package/src/strand-membership-writer.ts +232 -15
- package/src/strand-network-config.ts +172 -0
- package/src/strand-revocation-enforcer.ts +653 -0
- package/src/strand-solicitation.ts +484 -472
- package/src/strand-wake-protocol.ts +94 -8
- package/src/strand-watcher.ts +319 -314
- package/src/types.ts +1554 -1309
- package/dist/strand-backfill.js.map +0 -1
package/src/cadre-node.ts
CHANGED
|
@@ -11,6 +11,8 @@ import type {
|
|
|
11
11
|
StrandInstance,
|
|
12
12
|
StrandRow,
|
|
13
13
|
StrandConfig,
|
|
14
|
+
FoundStrandConfig,
|
|
15
|
+
FoundStrandResult,
|
|
14
16
|
SAppConfig,
|
|
15
17
|
CadreNodeEvents,
|
|
16
18
|
ControlNetworkSeed,
|
|
@@ -34,7 +36,7 @@ import type {
|
|
|
34
36
|
PeerAddressRecord,
|
|
35
37
|
ResolveDeviceTokenOpts
|
|
36
38
|
} from './types.js';
|
|
37
|
-
import {
|
|
39
|
+
import { controlClusterPolicy, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
38
40
|
import { sign } from '@optimystic/quereus-plugin-crypto';
|
|
39
41
|
import { ed25519KeyPairFromLibp2p, ed25519PublicKeyFromPrivate, requireEd25519PublicKeyB64, type Ed25519KeyPair } from './ed25519-key.js';
|
|
40
42
|
import { strandTransportKey } from './strand-transport-key.js';
|
|
@@ -42,6 +44,7 @@ import { DEFAULT_IDENTITY_KEY_ID } from './key-store.js';
|
|
|
42
44
|
import { loadOrCreateIdentityKey } from './identity-key.js';
|
|
43
45
|
import { MemoryTrustedOwnerStore, type TrustedOwnerStore, type TrustSource } from './trusted-owner-store.js';
|
|
44
46
|
import { MemoryBootstrapPeerStore, type BootstrapPeerStore } from './bootstrap-peer-store.js';
|
|
47
|
+
import { MemoryEnrolledMachineStore, type EnrolledMachineStore } from './enrolled-machine-store.js';
|
|
45
48
|
import { mergePeerAddrs, groupAddrsByPeerId, type MergeAddrsResult } from './peer-addr-book.js';
|
|
46
49
|
import { verifyCadrePeerVoucher } from './peer-authorization.js';
|
|
47
50
|
import { ed25519PublicKeyB64FromPeerId } from './seed-bootstrap.js';
|
|
@@ -51,7 +54,7 @@ import {
|
|
|
51
54
|
isPeerRecordFresh,
|
|
52
55
|
orderSignalingFirst,
|
|
53
56
|
isSignalingAddr,
|
|
54
|
-
|
|
57
|
+
withTrailingPeerId,
|
|
55
58
|
currentMemberTrustPolicy,
|
|
56
59
|
DEFAULT_PEER_RECORD_MAX_AGE_MS,
|
|
57
60
|
DEFAULT_PEER_RECORD_HEARTBEAT_MS
|
|
@@ -63,6 +66,7 @@ import {
|
|
|
63
66
|
} from './device-token.js';
|
|
64
67
|
import { StrandWatcher, type StrandQueryable, type SAppIdLookup } from './strand-watcher.js';
|
|
65
68
|
import { StrandInstanceManager } from './strand-instance-manager.js';
|
|
69
|
+
import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
66
70
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
67
71
|
import type { CohortPeerRow } from './strand-cohort.js';
|
|
68
72
|
import {
|
|
@@ -74,14 +78,20 @@ import {
|
|
|
74
78
|
import { withDeadline } from './control-stream.js';
|
|
75
79
|
import { EnrollmentService } from './enrollment.js';
|
|
76
80
|
import { HibernationManager, type HibernationCallbacks } from './hibernation-manager.js';
|
|
77
|
-
import { ControlDatabase, type RevokedRowRef } from './control-database.js';
|
|
81
|
+
import { ControlDatabase, isStrandIdConflict, type RevokedRowRef } from './control-database.js';
|
|
78
82
|
import { SeedBootstrapService, type SeedEventCallbacks } from './seed-bootstrap.js';
|
|
79
83
|
import type { SeedTrustPolicy } from './seed-trust-policy.js';
|
|
80
84
|
import {
|
|
81
85
|
StrandSolicitationService,
|
|
82
86
|
type StrandSolicitationServiceOptions
|
|
83
87
|
} from './strand-solicitation.js';
|
|
84
|
-
import {
|
|
88
|
+
import { MAX_STRAND_ADDRS } from './strand-formation-protocol.js';
|
|
89
|
+
import {
|
|
90
|
+
createMembershipConnectionGater,
|
|
91
|
+
DEFAULT_ENROLLMENT_WINDOW_MS,
|
|
92
|
+
UnauthorizedReservationBudget,
|
|
93
|
+
type InboundConnectionVerdict
|
|
94
|
+
} from './membership-connection-gater.js';
|
|
85
95
|
import { StrandWakeService, dialWake } from './strand-wake-protocol.js';
|
|
86
96
|
import { StrandAddrService, collectStrandAddrs, type StrandAddrPeer } from './strand-addr-protocol.js';
|
|
87
97
|
import {
|
|
@@ -92,7 +102,8 @@ import {
|
|
|
92
102
|
peerStrandKey,
|
|
93
103
|
type CircuitRelayTarget
|
|
94
104
|
} from './delegate-admission.js';
|
|
95
|
-
import { resolveListenAddrs } from './relay-addrs.js';
|
|
105
|
+
import { relayCircuitAddrs, resolveListenAddrs, resolveTransportOptions, RelayReservationFailedError } from './relay-addrs.js';
|
|
106
|
+
import { replacesAdvertisedAddrs, resolveAnnounceAddrs } from './announce-addrs.js';
|
|
96
107
|
import {
|
|
97
108
|
superviseRelayReservation,
|
|
98
109
|
resolveRelayReservationState,
|
|
@@ -143,6 +154,30 @@ function relayStrandAddrPeer(relay: CircuitRelayTarget): StrandAddrPeer {
|
|
|
143
154
|
return { peerId: relay.relayPeerId, addrs: [multiaddr(relay.relayAddr)] };
|
|
144
155
|
}
|
|
145
156
|
|
|
157
|
+
/**
|
|
158
|
+
* `primary` followed by the entries of `extra` it does not already contain, de-duplicated
|
|
159
|
+
* and order-preserving.
|
|
160
|
+
*
|
|
161
|
+
* Used wherever a strand's discovery seed unions two sources of address strings — the
|
|
162
|
+
* freshly-RPC'd sibling answers and the cross-party addresses learned at formation. The
|
|
163
|
+
* caller passes the FRESHER source as `primary`, so a stale entry can only ever be
|
|
164
|
+
* appended, never promoted ahead of a current one, and each source keeps whatever
|
|
165
|
+
* signaling-first ordering it arrived with.
|
|
166
|
+
*/
|
|
167
|
+
function unionAddrs(primary: readonly string[], extra: readonly string[]): string[] {
|
|
168
|
+
if (extra.length === 0) {
|
|
169
|
+
return [...primary];
|
|
170
|
+
}
|
|
171
|
+
const out = [...primary];
|
|
172
|
+
const seen = new Set(primary);
|
|
173
|
+
for (const addr of extra) {
|
|
174
|
+
if (seen.has(addr)) continue;
|
|
175
|
+
seen.add(addr);
|
|
176
|
+
out.push(addr);
|
|
177
|
+
}
|
|
178
|
+
return out;
|
|
179
|
+
}
|
|
180
|
+
|
|
146
181
|
/**
|
|
147
182
|
* Build the signing callback the `ControlDatabase` writers expect: they hand it the
|
|
148
183
|
* canonical row-bound message BYTES (see `buildAuthorizationMessage`), and it ed25519-signs
|
|
@@ -166,6 +201,60 @@ function requireNonBlank(value: string, label: string): string {
|
|
|
166
201
|
return trimmed;
|
|
167
202
|
}
|
|
168
203
|
|
|
204
|
+
/**
|
|
205
|
+
* Is the live `Strand` row the one a publish of `desired` would have produced?
|
|
206
|
+
*
|
|
207
|
+
* "Identical content" means `(Type, MemberPrivateKey)` and deliberately NOT
|
|
208
|
+
* `FounderOwnerKey`: that column is provenance (WHICH machine published the row), not
|
|
209
|
+
* content. Two machines of one party racing to found the same id must keep resolving as
|
|
210
|
+
* "the winner's row stands" — the loser adopts the row and, since the winner's key is on
|
|
211
|
+
* it, correctly comes up as a joiner (see `launchStrand`'s founder derivation). Comparing
|
|
212
|
+
* it here would turn that benign race into a hard throw. `Id` is the lookup key and
|
|
213
|
+
* `StampId` is a single-use nonce, so the two compared columns are all the content there
|
|
214
|
+
* is. A live row matching on both is, by construction, the state a repeat publish would
|
|
215
|
+
* have reached, whichever branch of `Strand.AuthorizedInsert` seated it (owner-signed, or
|
|
216
|
+
* the unsigned consent branch an invite redemption uses).
|
|
217
|
+
*
|
|
218
|
+
* Returns the mismatching column names, empty when the row matches. The KEY's value is
|
|
219
|
+
* deliberately never returned or logged — it is the closed strand's read-gating secret.
|
|
220
|
+
*/
|
|
221
|
+
function strandRowMismatches(live: StrandRow, desired: StrandRow): string[] {
|
|
222
|
+
const mismatches: string[] = [];
|
|
223
|
+
if (live.Type !== desired.Type) {
|
|
224
|
+
mismatches.push(`Type is '${live.Type}', not the requested '${desired.Type}'`);
|
|
225
|
+
}
|
|
226
|
+
if ((live.MemberPrivateKey ?? null) !== (desired.MemberPrivateKey ?? null)) {
|
|
227
|
+
mismatches.push('MemberPrivateKey differs from the one supplied');
|
|
228
|
+
}
|
|
229
|
+
return mismatches;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* The live row when it matches `desired` ({@link strandRowMismatches}); otherwise throw
|
|
234
|
+
* naming the columns that differ.
|
|
235
|
+
*
|
|
236
|
+
* A mismatch must stay a hard error: accepting one would let a retry reopen a strand the
|
|
237
|
+
* party closed, or swap the key that gates its reads — so the two cases the raw
|
|
238
|
+
* `UNIQUE constraint failed: Strand.Id` could not distinguish ("already done, carry on"
|
|
239
|
+
* vs. "genuine conflict, stop") are separated here rather than left to the caller.
|
|
240
|
+
*
|
|
241
|
+
* @param situation - How the row came to be there, as a verb phrase completing
|
|
242
|
+
* `Strand <id> …`, e.g. `'is already published'`.
|
|
243
|
+
*/
|
|
244
|
+
function requireMatchingStrandRow(live: StrandRow, desired: StrandRow, situation: string): StrandRow {
|
|
245
|
+
const mismatches = strandRowMismatches(live, desired);
|
|
246
|
+
if (mismatches.length > 0) {
|
|
247
|
+
throw new Error(
|
|
248
|
+
`Strand ${desired.Id} ${situation} with DIFFERENT content, so this publish is a ` +
|
|
249
|
+
`conflict, not a repeat: ${mismatches.join('; ')}. Reconcile deliberately — attach ` +
|
|
250
|
+
'the existing strand with addStrand/foundStrand to keep it, or unpublishStrand first ' +
|
|
251
|
+
'to re-seat it (destructive for a closed strand: its MemberPrivateKey is stored ' +
|
|
252
|
+
'nowhere else).'
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
return live;
|
|
256
|
+
}
|
|
257
|
+
|
|
169
258
|
/**
|
|
170
259
|
* CadreNode is the main entry point for a cadre member.
|
|
171
260
|
* It manages:
|
|
@@ -197,6 +286,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
197
286
|
private trustedOwnerStore: TrustedOwnerStore | null = null;
|
|
198
287
|
private controlNode: Libp2p | null = null;
|
|
199
288
|
private controlDatabase: ControlDatabase | null = null;
|
|
289
|
+
/**
|
|
290
|
+
* The CONTROL network's peer-join block catch-up — same module as the
|
|
291
|
+
* per-strand ones, but membership-gated (see {@link startControlBackfill}).
|
|
292
|
+
* Created in {@link start} after the control node and database are up, stopped
|
|
293
|
+
* and dropped in {@link cleanup} before either is torn down, so a stop()→
|
|
294
|
+
* start() cycle rebuilds it with a fresh caught-up-peer memo.
|
|
295
|
+
*/
|
|
296
|
+
private controlBackfill: PeerJoinBackfill | null = null;
|
|
200
297
|
private strandWatcher: StrandWatcher | null = null;
|
|
201
298
|
private strandManager: StrandInstanceManager;
|
|
202
299
|
private hibernationManager: HibernationManager;
|
|
@@ -244,10 +341,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
244
341
|
private eventHandlers: Map<keyof CadreNodeEvents, Set<EventHandler<never>>> = new Map();
|
|
245
342
|
|
|
246
343
|
/**
|
|
247
|
-
* Relay multiaddrs the caller asked {@link reserveRelays} to reserve through
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
*
|
|
344
|
+
* Relay multiaddrs the caller asked {@link reserveRelays} to reserve through —
|
|
345
|
+
* or, on a node that names `network.relayAddrs`, the ones
|
|
346
|
+
* {@link driveControlRelayReservation} asked for at the end of {@link start}.
|
|
347
|
+
* Empty (the default) means neither happened, so {@link getRelayReservationState}
|
|
348
|
+
* reports `none`.
|
|
251
349
|
*/
|
|
252
350
|
private relayReserveAddrs: string[] = [];
|
|
253
351
|
/**
|
|
@@ -263,6 +361,24 @@ export class CadreNode implements SAppIdLookup {
|
|
|
263
361
|
*/
|
|
264
362
|
private relayReserveError: string | null = null;
|
|
265
363
|
|
|
364
|
+
/**
|
|
365
|
+
* True from just before the control libp2p node is created until
|
|
366
|
+
* {@link ControlDatabase.initialize} has settled — the connection gate's
|
|
367
|
+
* BRING-UP QUIET PERIOD (`membership-connection-gater.ts`), during which this
|
|
368
|
+
* node refuses every control connection in both directions.
|
|
369
|
+
*
|
|
370
|
+
* The invariant: the control database is built while this node holds zero
|
|
371
|
+
* control connections. Every same-party sibling connected in that window joins
|
|
372
|
+
* the Optimystic cohort the bring-up's block probes consult, and a sibling that
|
|
373
|
+
* has not yet replicated this node's `CadrePeer` row refuses them all — which
|
|
374
|
+
* fails `start()` outright and cannot be retried into convergence (writing the
|
|
375
|
+
* row that would clear the refusal needs the database being built).
|
|
376
|
+
*
|
|
377
|
+
* Cleared on FAILURE as well as success ({@link cleanup} clears it), so
|
|
378
|
+
* teardown is never gated.
|
|
379
|
+
*/
|
|
380
|
+
private controlBringUpInFlight = false;
|
|
381
|
+
|
|
266
382
|
/** Map of strandId -> sAppConfig for sAppId filtering and management */
|
|
267
383
|
private sAppConfigs: Map<string, SAppConfig> = new Map();
|
|
268
384
|
|
|
@@ -329,13 +445,27 @@ export class CadreNode implements SAppIdLookup {
|
|
|
329
445
|
/**
|
|
330
446
|
* In-memory delegate admission grants (see `delegate-admission.ts`): the
|
|
331
447
|
* strand-node transport peerIds this party's members have announced over the
|
|
332
|
-
* strand-addr RPC, admitted at the CONNECTION
|
|
333
|
-
* strand node can hold a circuit-relay reservation on this
|
|
334
|
-
*
|
|
335
|
-
*
|
|
448
|
+
* strand-addr RPC, admitted at the CONNECTION and RESERVATION levels so a
|
|
449
|
+
* member's NAT'd strand node can hold a circuit-relay reservation on this
|
|
450
|
+
* node (without spending the unauthorized-reservation budget). Consulted
|
|
451
|
+
* ONLY by {@link admitInboundControlConnection} and
|
|
452
|
+
* {@link admitControlRelayReservation}; the fail-closed per-stream gate
|
|
453
|
+
* ({@link authorizeInboundControlStream}) never honors it.
|
|
336
454
|
*/
|
|
337
455
|
private readonly delegateAdmission = new DelegateAdmissionStore();
|
|
338
456
|
|
|
457
|
+
/**
|
|
458
|
+
* Bounded budget of concurrent circuit-relay reservations granted to peers
|
|
459
|
+
* this node cannot (yet) place as members — the boot-ordering window where a
|
|
460
|
+
* genuine sibling reserves before its `CadrePeer` row has replicated here.
|
|
461
|
+
* Consulted only by {@link admitControlRelayReservation} (the gater's
|
|
462
|
+
* `denyInboundRelayReservation` policy); authorized members and announced
|
|
463
|
+
* delegates are admitted before it and never counted. Cap from
|
|
464
|
+
* `network.unauthorizedRelayReservationCap` (default
|
|
465
|
+
* `MAX_UNAUTHORIZED_RELAY_RESERVATIONS`).
|
|
466
|
+
*/
|
|
467
|
+
private readonly unauthorizedRelayReservations: UnauthorizedReservationBudget;
|
|
468
|
+
|
|
339
469
|
/**
|
|
340
470
|
* When this node last ANNOUNCED a delegate, keyed `<targetPeerId>\n<strandId>`
|
|
341
471
|
* (the peer announced TO, not the delegate). Throttles
|
|
@@ -357,6 +487,37 @@ export class CadreNode implements SAppIdLookup {
|
|
|
357
487
|
*/
|
|
358
488
|
private readonly strandPeerAddrRefreshAt = new Map<string, number>();
|
|
359
489
|
|
|
490
|
+
/**
|
|
491
|
+
* CROSS-PARTY strand-network addresses learned at formation, keyed by strandId — the
|
|
492
|
+
* responder's live strand addrs, carried back on the formation result
|
|
493
|
+
* (`FormationResultMessage.strandAddrs`) and recorded by {@link formStrand}.
|
|
494
|
+
*
|
|
495
|
+
* This is the ONLY cross-party discovery input there is. The strand-addr RPC that
|
|
496
|
+
* resolves a strand's addresses is membership-gated and answers own-party siblings
|
|
497
|
+
* only, so without this map a joiner's cohort seed for a two-party strand is empty and
|
|
498
|
+
* the mesh never forms.
|
|
499
|
+
*
|
|
500
|
+
* Read by {@link resolveCohortSeed} (launch + hibernation resume) and re-merged by
|
|
501
|
+
* every {@link refreshStrandPeerAddrs} pass — the re-merge is what keeps the entries
|
|
502
|
+
* alive past the peerStore's one-hour address expiry, since nothing can re-resolve
|
|
503
|
+
* them.
|
|
504
|
+
*
|
|
505
|
+
* IN-MEMORY and ONE-SHOT, deliberately (see `docs/strands.md`): the addresses die with
|
|
506
|
+
* the process, and a restarted initiator with no sibling running the strand is back to
|
|
507
|
+
* an empty seed until the durability work lands
|
|
508
|
+
* (`backlog/feat-cross-party-strand-addr-durability`). They are also never refreshed,
|
|
509
|
+
* so a responder whose relay reservation rotates before the initiator dials leaves a
|
|
510
|
+
* dead entry behind; recovery today is a fresh invitation.
|
|
511
|
+
*
|
|
512
|
+
* NOTE: entries are kept for the node's lifetime, including across a
|
|
513
|
+
* {@link stopStrand} — a stopped strand may be rediscovered by the watcher and must
|
|
514
|
+
* still get its seed. Each strand's list is capped at `MAX_STRAND_ADDRS`, but the
|
|
515
|
+
* number of KEYS is bounded only by the strands this node has ever formed as an
|
|
516
|
+
* initiator (not by time); if a node ever forms strands at scale, evict on
|
|
517
|
+
* `unpublishStrand` or cap the map.
|
|
518
|
+
*/
|
|
519
|
+
private readonly crossPartyStrandAddrs = new Map<string, string[]>();
|
|
520
|
+
|
|
360
521
|
/**
|
|
361
522
|
* Cold-start bootstrap dial targets: the owner-flagged peers of every seed
|
|
362
523
|
* this node has applied, keyed by peer id, valued by the seed's multiaddr
|
|
@@ -392,6 +553,36 @@ export class CadreNode implements SAppIdLookup {
|
|
|
392
553
|
*/
|
|
393
554
|
private bootstrapPeerStore: BootstrapPeerStore | null = null;
|
|
394
555
|
|
|
556
|
+
/**
|
|
557
|
+
* Node-local record of the party's enrolled-machine count (see
|
|
558
|
+
* `enrolled-machine-store.ts`), kept so the CONTROL node can declare a
|
|
559
|
+
* block-repair yardstick at bring-up. Constructed (or adopted from
|
|
560
|
+
* `config.enrolledMachines.store`) by {@link initializeEnrolledMachineStore};
|
|
561
|
+
* written by {@link refreshAuthorizedControlPeers}. Like
|
|
562
|
+
* {@link bootstrapPeerStore} it is deliberately NOT cleared by {@link cleanup},
|
|
563
|
+
* so it survives a stop()→start() cycle of the same node instance.
|
|
564
|
+
*/
|
|
565
|
+
private enrolledMachineStore: EnrolledMachineStore | null = null;
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* The enrolled-machine count this node's CONTROL libp2p node was (or will be)
|
|
569
|
+
* built with — read out of {@link enrolledMachineStore} during {@link start},
|
|
570
|
+
* before {@link createControlNode}, and consumed by
|
|
571
|
+
* {@link buildControlNodeOptions}.
|
|
572
|
+
*
|
|
573
|
+
* A captured FIELD rather than a live `store.count()` read, deliberately: the
|
|
574
|
+
* store moves as membership changes, but Optimystic froze the policy when the
|
|
575
|
+
* node was built, so this field is the honest answer to "what did this node
|
|
576
|
+
* actually declare?" and does not drift away from the running node. It is
|
|
577
|
+
* re-read on each {@link start}, which is what makes a stop()→start() cycle pick
|
|
578
|
+
* up a count recorded during the previous run.
|
|
579
|
+
*
|
|
580
|
+
* `undefined` — a brand-new node, an unreadable slot, or the ephemeral default
|
|
581
|
+
* store — means "this node does not know", which `controlClusterPolicy` answers
|
|
582
|
+
* with the frozen base policy itself.
|
|
583
|
+
*/
|
|
584
|
+
private declaredEnrolledMachines: number | undefined;
|
|
585
|
+
|
|
395
586
|
/** Initial self-registration timer (see {@link scheduleSelfRegistration}). */
|
|
396
587
|
private selfRegistrationTimer: ReturnType<typeof setTimeout> | null = null;
|
|
397
588
|
/** TTL heartbeat that re-publishes the self record before it goes stale. */
|
|
@@ -501,6 +692,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
501
692
|
this.config = config;
|
|
502
693
|
this.strandManager = new StrandInstanceManager();
|
|
503
694
|
this.enrollmentService = new EnrollmentService();
|
|
695
|
+
this.unauthorizedRelayReservations = new UnauthorizedReservationBudget(
|
|
696
|
+
config.network?.unauthorizedRelayReservationCap
|
|
697
|
+
);
|
|
504
698
|
|
|
505
699
|
// Create hibernation manager with callbacks
|
|
506
700
|
const hibernationCallbacks: HibernationCallbacks = {
|
|
@@ -630,20 +824,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
630
824
|
|
|
631
825
|
log('Starting CadreNode for party: %s', this.config.controlNetwork.partyId);
|
|
632
826
|
|
|
633
|
-
// NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
634
|
-
// diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
635
|
-
// second such warning ever appears here, route both through a `CadreNodeEvents` entry the
|
|
636
|
-
// embedder surfaces instead of growing a console surface inside a library.
|
|
637
|
-
if (this.config.network?.announceAddrs && this.config.network.announceAddrs.length > 0) {
|
|
638
|
-
console.warn(
|
|
639
|
-
'network.announceAddrs is set but not yet supported (no upstream db-p2p option to apply it) ' +
|
|
640
|
-
'— this node will keep advertising its listen/relay addresses instead.'
|
|
641
|
-
);
|
|
642
|
-
}
|
|
643
|
-
|
|
644
827
|
try {
|
|
645
828
|
const tTotal = performance.now();
|
|
646
829
|
|
|
830
|
+
// Config pre-flight, inside the guarded region: resolving the listen addrs it reads
|
|
831
|
+
// throws on a malformed `relayAddrs` entry, and that failure belongs on the same
|
|
832
|
+
// logged-and-cleaned-up path as every other one in start().
|
|
833
|
+
this.warnIfAnnounceAddrsDiscardRelay();
|
|
834
|
+
|
|
647
835
|
// Install the WebRTC TURN-relay tracker BEFORE any libp2p bring-up, so it
|
|
648
836
|
// wraps globalThis.RTCPeerConnection before the control node can create one.
|
|
649
837
|
// Inert on Node.js (no RTCPeerConnection); disposed in cleanup().
|
|
@@ -666,6 +854,33 @@ export class CadreNode implements SAppIdLookup {
|
|
|
666
854
|
// loaded before the first reconcile pass could consult them.
|
|
667
855
|
this.initializeBootstrapPeerStore();
|
|
668
856
|
|
|
857
|
+
// Read the party's enrolled-machine count out of its node-local record and
|
|
858
|
+
// capture it for buildControlNodeOptions below. This MUST precede
|
|
859
|
+
// createControlNode: Optimystic freezes the cluster policy when the node is
|
|
860
|
+
// built, and the ControlDatabase that could answer the question live does not
|
|
861
|
+
// exist until after that. Remembering the number across the restart is the
|
|
862
|
+
// only way to declare it at all — see `enrolled-machine-store.ts`.
|
|
863
|
+
this.initializeEnrolledMachineStore();
|
|
864
|
+
|
|
865
|
+
// Arm the connection gate's BRING-UP QUIET PERIOD before the libp2p node
|
|
866
|
+
// exists, so no connection can form ahead of it: `@libp2p/bootstrap` emits
|
|
867
|
+
// its discovery events ~1 s after `libp2p.start()` and the connection
|
|
868
|
+
// manager auto-dials from there, which is INSIDE createControlNode below.
|
|
869
|
+
// The invariant being protected is "zero control connections during
|
|
870
|
+
// control-database bring-up" — see the field's doc and
|
|
871
|
+
// `membership-connection-gater.ts` → "The bring-up quiet period".
|
|
872
|
+
//
|
|
873
|
+
// NOTE: without this gate the bootstrap arm is a RACE, not a guarantee — its
|
|
874
|
+
// safety margin is bring-up duration versus @libp2p/bootstrap's 1 s discovery
|
|
875
|
+
// fuse, and bring-up duration is (raw-storage operations) × per-operation
|
|
876
|
+
// latency. The cold-start operation count and the 50-90 ms/op figure a loaded
|
|
877
|
+
// phone actually sees live in `control-database.ts`'s `loadSchema` note (and
|
|
878
|
+
// are pinned by `control-start-storage-op-budget.spec.ts`): at the top of that
|
|
879
|
+
// range bring-up is 9-15 s, far past the fuse. So if this gate is ever removed
|
|
880
|
+
// or bypassed, that arm reopens for exactly the nodes most likely to hit it —
|
|
881
|
+
// phones joining a party, whose membership row has not replicated yet.
|
|
882
|
+
this.controlBringUpInFlight = true;
|
|
883
|
+
|
|
669
884
|
// Create the control network libp2p node
|
|
670
885
|
let t0 = performance.now();
|
|
671
886
|
this.controlNode = await this.createControlNode();
|
|
@@ -688,6 +903,21 @@ export class CadreNode implements SAppIdLookup {
|
|
|
688
903
|
});
|
|
689
904
|
await this.controlDatabase.initialize();
|
|
690
905
|
timing('[start] controlDatabase.initialize: %dms', Math.round(performance.now() - t0));
|
|
906
|
+
// The database exists locally now, so the quiet period has done its job and
|
|
907
|
+
// this node can be in the conversation. A failure above never reaches here —
|
|
908
|
+
// `cleanup()` on the catch path opens the gate instead, so teardown (and a
|
|
909
|
+
// retry by the embedder) is not gated.
|
|
910
|
+
//
|
|
911
|
+
// NOTE: the window ends HERE, not at the end of start(), so the awaited control-DB
|
|
912
|
+
// work that follows — `strandWatcher.start()`'s first poll — can run against a
|
|
913
|
+
// cohort a connection just joined. Fine now: every block that poll reads was
|
|
914
|
+
// created locally by the initialize() above, so a refusing sibling has nothing to
|
|
915
|
+
// refuse. If a boot ever fails with `BlockUnavailableError` from BELOW this line,
|
|
916
|
+
// that assumption broke. Do not simply move this clear later without measuring:
|
|
917
|
+
// `handleStrandAdded` runs inside `strandWatcher.start()` and seeds each strand
|
|
918
|
+
// node from CONNECTED siblings (`resolveCohortSeed`), so a longer window trades a
|
|
919
|
+
// bring-up hazard for boot-time strand seeds that are always empty.
|
|
920
|
+
this.controlBringUpInFlight = false;
|
|
691
921
|
log('Control database initialized');
|
|
692
922
|
|
|
693
923
|
// Attach the per-stream gate to the control DB's membership hub, so every
|
|
@@ -765,18 +995,41 @@ export class CadreNode implements SAppIdLookup {
|
|
|
765
995
|
// write-while-alone re-replication drain must fire on the first 0→≥1 edge.
|
|
766
996
|
this.wireControlConnectionListeners();
|
|
767
997
|
|
|
998
|
+
// Arm the control network's peer-join block catch-up now that the control
|
|
999
|
+
// DB (which the push-time membership gate reads) is up. Armed even when
|
|
1000
|
+
// this node is alone — like the strand one, it only does work when the
|
|
1001
|
+
// control libp2p node reports a peer connection, and arming it at start is
|
|
1002
|
+
// what closes the "genesised alone, headers never replicate" hole.
|
|
1003
|
+
this.startControlBackfill();
|
|
1004
|
+
|
|
768
1005
|
// Seed the per-stream gate's materialized authorized set now that the
|
|
769
1006
|
// control DB is up (non-blocking; the helper never rejects). Until it
|
|
770
1007
|
// lands the snapshot is empty, which the stream gate treats as cold
|
|
771
1008
|
// start and admits.
|
|
772
1009
|
void this.refreshMembershipGate('start');
|
|
773
1010
|
|
|
1011
|
+
// Reserve the configured relays LAST, so every step above ran against a
|
|
1012
|
+
// cohort of one (see driveControlRelayReservation). Ahead of
|
|
1013
|
+
// scheduleSelfRegistration, so this node's first published `CadrePeer` row
|
|
1014
|
+
// already carries the `/p2p-circuit` address the reservation earns it.
|
|
1015
|
+
// Throws on a relay that will not have us — start() is fail-fast for a node
|
|
1016
|
+
// whose operator named a relay.
|
|
1017
|
+
await this.driveControlRelayReservation();
|
|
1018
|
+
|
|
774
1019
|
// Schedule self-registration in background
|
|
775
1020
|
this.scheduleSelfRegistration();
|
|
776
1021
|
|
|
777
1022
|
} catch (error) {
|
|
778
1023
|
log('Failed to start CadreNode: %o', error);
|
|
1024
|
+
// `driveControlRelayReservation` throws AFTER `control:connected` was emitted,
|
|
1025
|
+
// so a subscriber can have seen this node come up. Balance the edge rather than
|
|
1026
|
+
// leaving it dangling — `cleanup()` has torn the node down either way, and the
|
|
1027
|
+
// events are what an embedder drives its "am I connected" UI from.
|
|
1028
|
+
const wasConnected = this._running;
|
|
779
1029
|
await this.cleanup();
|
|
1030
|
+
if (wasConnected) {
|
|
1031
|
+
this.emit('control:disconnected', undefined);
|
|
1032
|
+
}
|
|
780
1033
|
throw error;
|
|
781
1034
|
}
|
|
782
1035
|
}
|
|
@@ -817,19 +1070,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
817
1070
|
}
|
|
818
1071
|
|
|
819
1072
|
log('Stopping CadreNode');
|
|
820
|
-
// Stop the retry loop BEFORE tearing the node down, not after: `cleanup()`
|
|
821
|
-
// stops the control node partway through, and a tick that fires during it
|
|
822
|
-
// would dial and then poll a half-torn-down node for the whole reserve
|
|
823
|
-
// timeout — logging spurious dial failures and holding the process open.
|
|
824
|
-
this.relayReserveSupervisor?.stop();
|
|
825
|
-
this.relayReserveSupervisor = null;
|
|
826
1073
|
await this.cleanup();
|
|
827
|
-
// Drop the rest of the relay-reservation posture: a stopped node holds no
|
|
828
|
-
// reservation, so a restarted instance must not report the previous run's
|
|
829
|
-
// `reserved`.
|
|
830
|
-
this.relayReserveAddrs = [];
|
|
831
|
-
this.relayReserveError = null;
|
|
832
|
-
this._running = false;
|
|
833
1074
|
this.emit('control:disconnected', undefined);
|
|
834
1075
|
log('CadreNode stopped');
|
|
835
1076
|
}
|
|
@@ -974,6 +1215,50 @@ export class CadreNode implements SAppIdLookup {
|
|
|
974
1215
|
this.bootstrapPeerStore = store;
|
|
975
1216
|
}
|
|
976
1217
|
|
|
1218
|
+
/**
|
|
1219
|
+
* Construct (or adopt) the node-local enrolled-machine store and capture the
|
|
1220
|
+
* count this run will declare into {@link declaredEnrolledMachines}.
|
|
1221
|
+
*
|
|
1222
|
+
* Synchronous for the same reason {@link initializeBootstrapPeerStore} is: an
|
|
1223
|
+
* injected store has already loaded its persisted count by the time it is handed
|
|
1224
|
+
* in (its `open` is the async part), and the in-memory default has nothing to
|
|
1225
|
+
* load. Must run BEFORE {@link createControlNode} — that is the whole point of
|
|
1226
|
+
* the record; see `enrolled-machine-store.ts`.
|
|
1227
|
+
*
|
|
1228
|
+
* The store instance is kept across stop()→start() (so a count recorded during
|
|
1229
|
+
* the previous run is not lost with the ephemeral default), but the DECLARED
|
|
1230
|
+
* count is re-read every time — which is how a restart applies a number the
|
|
1231
|
+
* previous run learned.
|
|
1232
|
+
*
|
|
1233
|
+
* A store scoped to a different party is a configuration error, fail closed. It
|
|
1234
|
+
* is only a repair hint, but a hint sized by a foreign party's membership is a
|
|
1235
|
+
* number nobody chose, and the mismatch always means a miswired embedder.
|
|
1236
|
+
*/
|
|
1237
|
+
private initializeEnrolledMachineStore(): void {
|
|
1238
|
+
const partyId = this.config.controlNetwork.partyId;
|
|
1239
|
+
const store = this.enrolledMachineStore
|
|
1240
|
+
?? this.config.enrolledMachines?.store
|
|
1241
|
+
?? new MemoryEnrolledMachineStore(partyId);
|
|
1242
|
+
if (store.partyId !== partyId) {
|
|
1243
|
+
throw new Error(
|
|
1244
|
+
`CadreNodeConfig: enrolledMachines.store is scoped to party ${store.partyId}, ` +
|
|
1245
|
+
`but this node serves party ${partyId} — refusing to size a repair yardstick from a foreign party`
|
|
1246
|
+
);
|
|
1247
|
+
}
|
|
1248
|
+
this.enrolledMachineStore = store;
|
|
1249
|
+
this.declaredEnrolledMachines = store.count();
|
|
1250
|
+
log('control repair yardstick will be declared from %o enrolled machine(s)', this.declaredEnrolledMachines);
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
/**
|
|
1254
|
+
* The node-local enrolled-machine store (null before {@link start}) — what this
|
|
1255
|
+
* node last knew about its party's size. Exposed for diagnostics and for a host
|
|
1256
|
+
* that wants to show which repair yardstick the next launch will declare.
|
|
1257
|
+
*/
|
|
1258
|
+
getEnrolledMachineStore(): EnrolledMachineStore | null {
|
|
1259
|
+
return this.enrolledMachineStore;
|
|
1260
|
+
}
|
|
1261
|
+
|
|
977
1262
|
/**
|
|
978
1263
|
* The node-local cold-start bootstrap-peer store (null before {@link start}) —
|
|
979
1264
|
* the retained owner addresses {@link reconcileControlCohort}'s cold-start
|
|
@@ -1024,6 +1309,93 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1024
1309
|
return await createLibp2pNode(this.buildControlNodeOptions());
|
|
1025
1310
|
}
|
|
1026
1311
|
|
|
1312
|
+
/**
|
|
1313
|
+
* The control libp2p node's Optimystic network name — ONE binding for every
|
|
1314
|
+
* derivation from it (`db-p2p` namespaces all of the node's protocol ids as
|
|
1315
|
+
* `/optimystic/<networkName>/...`), so the node options and the block-transfer
|
|
1316
|
+
* protocol prefix the control backfill dials can never drift apart.
|
|
1317
|
+
*/
|
|
1318
|
+
private controlNetworkName(): string {
|
|
1319
|
+
return `control-${this.config.controlNetwork.partyId}`;
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
/**
|
|
1323
|
+
* Arm the CONTROL network's peer-join block catch-up: push every block in the
|
|
1324
|
+
* control database's own raw store to each newly connected AUTHORIZED member
|
|
1325
|
+
* this runtime has not yet caught up. This is what physically replicates
|
|
1326
|
+
* control blocks committed while the writer was alone — the named
|
|
1327
|
+
* collection-header blocks written once at genesis above all, whose revision
|
|
1328
|
+
* never moves again, so no later commit ever carries them to a member that
|
|
1329
|
+
* joined after them. Without it, such a member that restarts offline reads
|
|
1330
|
+
* the affected control tables as EMPTY, silently (`isMember()` answers false
|
|
1331
|
+
* for peers it knew about before the restart).
|
|
1332
|
+
*
|
|
1333
|
+
* Unlike the per-strand instances (see `strand-instance-manager.ts`), pushes
|
|
1334
|
+
* are gated on {@link isAuthorizedMember}, judged at push time: the control
|
|
1335
|
+
* network's inbound connection gate deliberately admits non-members in
|
|
1336
|
+
* several states (seed delivery to an un-enrolled node, an open enrollment
|
|
1337
|
+
* window, an outstanding invitation, configured bootstrap/relay peers), and
|
|
1338
|
+
* pushing the whole control store to such a peer would hand a stranger the
|
|
1339
|
+
* party's entire membership, addresses and strand list. The receiving side
|
|
1340
|
+
* needs no work of its own: `createLibp2pNode` registers the block-transfer
|
|
1341
|
+
* handler on every node it builds, control node included, and this node's
|
|
1342
|
+
* per-stream gate (`authorizeInboundControlStream`) covers the inbound
|
|
1343
|
+
* direction. A denied peer is retried on its next `connection:open`, and —
|
|
1344
|
+
* because the production join order is connect-then-authorize — on every
|
|
1345
|
+
* committed membership change via {@link refreshAuthorizedControlPeers}'s
|
|
1346
|
+
* `scheduleConnectedPeers()` call.
|
|
1347
|
+
*
|
|
1348
|
+
* No-ops (logged) when the embedder configured no control storage or the
|
|
1349
|
+
* node exposes no key network — the backfill would have nothing to read or
|
|
1350
|
+
* no way to dial.
|
|
1351
|
+
*/
|
|
1352
|
+
private startControlBackfill(): void {
|
|
1353
|
+
if (this.config.controlBackfill?.enabled === false || !this.controlNode) {
|
|
1354
|
+
return;
|
|
1355
|
+
}
|
|
1356
|
+
// Through the resolver, not the {@link controlStorage} memo field: the resolver is
|
|
1357
|
+
// idempotent (it returns the same wrapped store `buildControlNodeOptions` already
|
|
1358
|
+
// resolved), so the catch-up cannot be silently disarmed by a future reordering that
|
|
1359
|
+
// moves this call ahead of the one that populated the field.
|
|
1360
|
+
const storage = this.resolveControlStorage();
|
|
1361
|
+
const keyNetwork = (this.controlNode as Libp2pNodeWithRepo).keyNetwork;
|
|
1362
|
+
if (!storage || !keyNetwork) {
|
|
1363
|
+
log('Control peer-join block catch-up is inert: %s',
|
|
1364
|
+
!storage ? 'no control storage configured' : 'control node exposes no keyNetwork');
|
|
1365
|
+
return;
|
|
1366
|
+
}
|
|
1367
|
+
const networkName = this.controlNetworkName();
|
|
1368
|
+
this.controlBackfill = new PeerJoinBackfill({
|
|
1369
|
+
label: networkName,
|
|
1370
|
+
libp2p: this.controlNode,
|
|
1371
|
+
peerNetwork: keyNetwork,
|
|
1372
|
+
storage,
|
|
1373
|
+
// The same prefix the receiver registered its block-transfer handler
|
|
1374
|
+
// under — derived from the same networkName binding the node options
|
|
1375
|
+
// used, never re-spelled here.
|
|
1376
|
+
protocolPrefix: `/optimystic/${networkName}`,
|
|
1377
|
+
authorizePeer: (peerId) => this.isAuthorizedMember(peerId)
|
|
1378
|
+
}, {
|
|
1379
|
+
// A shorter settle window than the strand default (1000 ms): the control
|
|
1380
|
+
// store is small (a party's membership — dozens of blocks), a re-push is
|
|
1381
|
+
// idempotent, and the window a joiner is connected before going offline
|
|
1382
|
+
// can be short (converge one row, then stop) — a slow debounce is the
|
|
1383
|
+
// difference between that joiner holding the collection headers and
|
|
1384
|
+
// reading its own membership as empty after an offline restart.
|
|
1385
|
+
//
|
|
1386
|
+
// NOTE: 250 is reasoned, not swept — the delete-while-alone scenario's
|
|
1387
|
+
// phases 1-3 finish in about a second, so the 1000 ms strand default left
|
|
1388
|
+
// the push racing the joiner's stop, and 250 gives margin. Nobody measured
|
|
1389
|
+
// the shape of the curve either side of it. If the control gates start
|
|
1390
|
+
// flaking on a joiner that stopped before being caught up, lower this
|
|
1391
|
+
// before looking anywhere else; if catch-up pushes start showing up as
|
|
1392
|
+
// connection-churn noise, raise it.
|
|
1393
|
+
debounceMs: 250,
|
|
1394
|
+
...this.config.controlBackfill
|
|
1395
|
+
});
|
|
1396
|
+
this.controlBackfill.start();
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1027
1399
|
/**
|
|
1028
1400
|
* The control database's cache-wrapped raw storage, resolved from
|
|
1029
1401
|
* `config.storage.provider` on first use and then held in {@link controlStorage}
|
|
@@ -1055,6 +1427,45 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1055
1427
|
return this.controlStorage;
|
|
1056
1428
|
}
|
|
1057
1429
|
|
|
1430
|
+
/**
|
|
1431
|
+
* Warn the operator when `network.announceAddrs` will silently cost this node the
|
|
1432
|
+
* relay reachability it also configured.
|
|
1433
|
+
*
|
|
1434
|
+
* A non-empty announce set REPLACES everything libp2p advertises, so the
|
|
1435
|
+
* `/p2p-circuit` address earned by a relay reservation is dropped from the node's
|
|
1436
|
+
* advertised addresses even though the reservation itself is still held — peers
|
|
1437
|
+
* behind NAT stop being able to reach it through that relay. Not an error: an
|
|
1438
|
+
* operator whose relay slot is decorative may genuinely want only the announced
|
|
1439
|
+
* address, so this reports and proceeds.
|
|
1440
|
+
*
|
|
1441
|
+
* Keyed off the RESOLVED listen addrs rather than `relayAddrs` alone, so a
|
|
1442
|
+
* hand-written `/p2p-circuit` entry in `listenAddrs` — the same reservation by the
|
|
1443
|
+
* longer route — is caught too.
|
|
1444
|
+
*
|
|
1445
|
+
* NOTE: the only direct `console.*` in this library — a boot-time operator warning, not a
|
|
1446
|
+
* diagnostic trace (those use `debug`, which an operator never sees without `DEBUG=`). If a
|
|
1447
|
+
* second such warning ever appears here, route both through a `CadreNodeEvents` entry the
|
|
1448
|
+
* embedder surfaces instead of growing a console surface inside a library.
|
|
1449
|
+
*/
|
|
1450
|
+
private warnIfAnnounceAddrsDiscardRelay(): void {
|
|
1451
|
+
const network = this.config.network;
|
|
1452
|
+
if (!replacesAdvertisedAddrs(network)) {
|
|
1453
|
+
return;
|
|
1454
|
+
}
|
|
1455
|
+
const listensOnCircuit = (resolveListenAddrs(network) ?? [])
|
|
1456
|
+
.some((addr) => addr.includes('/p2p-circuit'));
|
|
1457
|
+
if (!listensOnCircuit) {
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1460
|
+
console.warn(
|
|
1461
|
+
'network.announceAddrs is set alongside a circuit-relay listener (from network.relayAddrs, ' +
|
|
1462
|
+
'or a /p2p-circuit entry in network.listenAddrs). A non-empty announce set REPLACES every ' +
|
|
1463
|
+
'address this node advertises, so the /p2p-circuit address earned from the relay reservation ' +
|
|
1464
|
+
'will not be advertised and peers will stop reaching this node through that relay. ' +
|
|
1465
|
+
'Use network.appendAnnounceAddrs instead to advertise an extra address without discarding the rest.'
|
|
1466
|
+
);
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1058
1469
|
/**
|
|
1059
1470
|
* Map this node's config onto the control network's libp2p node options.
|
|
1060
1471
|
*
|
|
@@ -1066,11 +1477,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1066
1477
|
private buildControlNodeOptions(): Parameters<typeof createLibp2pNode>[0] {
|
|
1067
1478
|
const { controlNetwork, network, profile } = this.config;
|
|
1068
1479
|
const identityKey = this.identityKey;
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
//
|
|
1072
|
-
//
|
|
1073
|
-
const
|
|
1480
|
+
// `'search'`: `network.relayAddrs` contributes the bare `/p2p-circuit` SEARCH
|
|
1481
|
+
// entry, which opens no connection — the reservation is driven explicitly at
|
|
1482
|
+
// the END of `start()`, so the control database is built while this node holds
|
|
1483
|
+
// zero control connections. See `relay-addrs.ts` and `start()`.
|
|
1484
|
+
const listenAddrs = resolveListenAddrs(network, 'search');
|
|
1485
|
+
// The transports those listen entries imply. A `/ws` entry switches db-p2p's
|
|
1486
|
+
// WebSocket transport on (without it the entry binds nothing and libp2p reports
|
|
1487
|
+
// nothing); anything outside {tcp, ws/wss, p2p-circuit} throws here rather than
|
|
1488
|
+
// being silently dropped at bring-up. See `relay-addrs.ts`.
|
|
1489
|
+
const transportOptions = resolveTransportOptions(network, listenAddrs);
|
|
1490
|
+
|
|
1491
|
+
const enableRelay = this.relayServerEnabled();
|
|
1074
1492
|
|
|
1075
1493
|
// The control node's own storage, resolved once per runtime (see
|
|
1076
1494
|
// {@link resolveControlStorage}) rather than per call.
|
|
@@ -1079,7 +1497,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1079
1497
|
const nodeOptions: Parameters<typeof createLibp2pNode>[0] = {
|
|
1080
1498
|
port: 0,
|
|
1081
1499
|
bootstrapNodes: controlNetwork.bootstrapNodes,
|
|
1082
|
-
networkName:
|
|
1500
|
+
networkName: this.controlNetworkName(),
|
|
1083
1501
|
storage: controlStorageProvider,
|
|
1084
1502
|
fretProfile: profile === 'storage' ? 'core' : 'edge',
|
|
1085
1503
|
relay: enableRelay,
|
|
@@ -1090,20 +1508,42 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1090
1508
|
// CONTROL_CLUSTER_POLICY: only the admission gate defaults to 2, while the read-repair
|
|
1091
1509
|
// corroboration floor would otherwise fall back to this clusterSize of 16.
|
|
1092
1510
|
clusterSize: CONTROL_REPLICATION_BREADTH,
|
|
1093
|
-
|
|
1511
|
+
// The base control policy with the block-repair corroboration yardstick declared
|
|
1512
|
+
// from the machines this party had enrolled at this node's last look
|
|
1513
|
+
// ({@link declaredEnrolledMachines}, read in start() before this runs). Handed
|
|
1514
|
+
// `undefined` — a brand-new node, an unreadable record, or an embedder that
|
|
1515
|
+
// injected no store — this returns the frozen CONTROL_CLUSTER_POLICY object
|
|
1516
|
+
// itself, so the unknown case is provably byte-for-byte the old behaviour.
|
|
1517
|
+
// The yardstick moves ALONE: `assumedClusterSize` stays pinned at 2, because a
|
|
1518
|
+
// party of phones cannot promise three quarters of its machines are awake.
|
|
1519
|
+
clusterPolicy: controlClusterPolicy(this.declaredEnrolledMachines),
|
|
1094
1520
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
1095
1521
|
...(identityKey && { privateKey: identityKey }),
|
|
1096
1522
|
...(network?.transports && { transports: network.transports }),
|
|
1097
|
-
//
|
|
1098
|
-
// `network.
|
|
1523
|
+
// `{ wsPort }` when a listen entry names WebSocket, otherwise `{}` — and always
|
|
1524
|
+
// `{}` when `network.transports` is set, since the embedder owns transport policy
|
|
1525
|
+
// then. Spread NEXT to `transports` because the two answer the same question.
|
|
1526
|
+
...transportOptions,
|
|
1527
|
+
// Configured `listenAddrs`, plus the bare `/p2p-circuit` search listener when
|
|
1528
|
+
// `network.relayAddrs` is set — that listener registers the pending reservation
|
|
1529
|
+
// `driveControlRelayReservation` fills after bring-up.
|
|
1099
1530
|
...(listenAddrs && { listenAddrs }),
|
|
1531
|
+
// What this node ADVERTISES, which is not the same question as what it binds.
|
|
1532
|
+
// Either field is present only when configured non-empty; a non-empty
|
|
1533
|
+
// `announceAddrs` replaces the advertised set (see warnIfAnnounceAddrsDiscardRelay).
|
|
1534
|
+
...resolveAnnounceAddrs(network),
|
|
1100
1535
|
// The CONTROL node composes the membership admission gate onto any
|
|
1101
|
-
// caller-supplied gater (deny from either wins on inbound
|
|
1102
|
-
// hooks pass through). Strand
|
|
1103
|
-
//
|
|
1536
|
+
// caller-supplied gater (deny from either wins on the inbound-encrypted
|
|
1537
|
+
// and relay-reservation hooks; all other hooks pass through). Strand
|
|
1538
|
+
// cohort nodes keep the raw configured gater (see
|
|
1539
|
+
// strand-instance-manager.ts) — their peers are legitimately
|
|
1104
1540
|
// cross-party, so cadre membership must not gate them.
|
|
1105
1541
|
connectionGater: createMembershipConnectionGater(
|
|
1106
|
-
{
|
|
1542
|
+
{
|
|
1543
|
+
admitInbound: (remotePeerId) => this.admitInboundControlConnection(remotePeerId),
|
|
1544
|
+
admitRelayReservation: (remotePeerId) => this.admitControlRelayReservation(remotePeerId),
|
|
1545
|
+
bringUpInFlight: () => this.controlBringUpInFlight
|
|
1546
|
+
},
|
|
1107
1547
|
network?.connectionGater
|
|
1108
1548
|
),
|
|
1109
1549
|
// Fail-closed per-stream authorization for the four Optimystic control-DB
|
|
@@ -1117,6 +1557,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1117
1557
|
return nodeOptions;
|
|
1118
1558
|
}
|
|
1119
1559
|
|
|
1560
|
+
/**
|
|
1561
|
+
* Does this node's CONTROL libp2p run the circuit-relay server? An explicit
|
|
1562
|
+
* `network.enableRelay` wins; otherwise storage-profile nodes default to on
|
|
1563
|
+
* (better connectivity/uptime). Shared by {@link buildControlNodeOptions}
|
|
1564
|
+
* (which configures the server from it) and
|
|
1565
|
+
* {@link admitInboundControlConnection} (whose deny/admit-for-relay branch
|
|
1566
|
+
* must agree with whether a reservation is even servable here).
|
|
1567
|
+
*/
|
|
1568
|
+
private relayServerEnabled(): boolean {
|
|
1569
|
+
return this.config.network?.enableRelay ?? (this.config.profile === 'storage');
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1120
1572
|
/**
|
|
1121
1573
|
* Decide whether an inbound CONTROL-network connection from `remotePeerId`
|
|
1122
1574
|
* should be admitted — the policy behind the connection gater wired in
|
|
@@ -1126,7 +1578,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1126
1578
|
* `membership-connection-gater.ts` for the layer's rationale and the
|
|
1127
1579
|
* stranger-open protocol allowlist).
|
|
1128
1580
|
*
|
|
1129
|
-
*
|
|
1581
|
+
* Returns `'admit'` when ANY of:
|
|
1130
1582
|
* 1. a shared-baseline check admits ({@link admitControlPeerUnconditionally}
|
|
1131
1583
|
* — not running / DB torn down, absent-or-empty trusted-owner anchor, or
|
|
1132
1584
|
* configured bootstrap/relay infrastructure);
|
|
@@ -1163,43 +1615,119 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1163
1615
|
* this node yet is denied even though the formation handler would have
|
|
1164
1616
|
* accepted it, exactly like the unreplicated-membership-row case below.
|
|
1165
1617
|
*
|
|
1618
|
+
* When every check falls through, the verdict depends on whether this node
|
|
1619
|
+
* runs the circuit-relay server ({@link relayServerEnabled}): without one,
|
|
1620
|
+
* `'deny'`; with one, `'admit-for-relay'` — a circuit-relay reservation is
|
|
1621
|
+
* established by the reserving peer DIALING the relay, so a connection deny
|
|
1622
|
+
* here kills the reservation, and that deny is NOT self-healing: an outbound
|
|
1623
|
+
* reconcile re-dial re-establishes a data link, but no outbound dial can
|
|
1624
|
+
* grant the remote peer a reservation, and a relay-only peer has no address
|
|
1625
|
+
* of its own to dial back — the reservation IS its address. The gater admits
|
|
1626
|
+
* such a connection, decides the reservation via
|
|
1627
|
+
* {@link admitControlRelayReservation}, and drops the connection if no
|
|
1628
|
+
* reservation is admitted in time (see `membership-connection-gater.ts` →
|
|
1629
|
+
* "The relay-reservation seam").
|
|
1630
|
+
*
|
|
1166
1631
|
* NOTE: check 4/5 runs a control-DB read per inbound connection
|
|
1167
1632
|
* (`listAuthorizedMembers`); connections are rare and cadres small, and this
|
|
1168
1633
|
* layer is fail-open behind `ADMISSION_DECISION_TIMEOUT_MS`, so the live
|
|
1169
1634
|
* read is safe here — unlike the per-stream gate, which must consult the
|
|
1170
1635
|
* materialized {@link authorizedControlPeers} snapshot instead.
|
|
1171
|
-
* NOTE: a sibling whose membership row has not yet
|
|
1172
|
-
*
|
|
1173
|
-
* either side's next outbound reconcile dial (outbound is never
|
|
1174
|
-
* re-establishes the link — self-healing,
|
|
1175
|
-
|
|
1176
|
-
|
|
1636
|
+
* NOTE: on a relay-DISABLED node, a sibling whose membership row has not yet
|
|
1637
|
+
* replicated here is denied until the row converges (typically via the
|
|
1638
|
+
* owner); either side's next outbound reconcile dial (outbound is never
|
|
1639
|
+
* gated) re-establishes the DATA link — self-healing for data, visible as a
|
|
1640
|
+
* transient deny. That self-healing story never covered a reservation, which
|
|
1641
|
+
* is exactly why the relay-enabled path above exists.
|
|
1642
|
+
*/
|
|
1643
|
+
private async admitInboundControlConnection(remotePeerId: string): Promise<InboundConnectionVerdict> {
|
|
1177
1644
|
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1178
|
-
return
|
|
1645
|
+
return 'admit';
|
|
1179
1646
|
}
|
|
1180
1647
|
if (Date.now() < this.enrollmentWindowUntil) {
|
|
1181
|
-
return
|
|
1648
|
+
return 'admit';
|
|
1182
1649
|
}
|
|
1183
1650
|
if (this.delegateAdmission.has(remotePeerId)) {
|
|
1184
|
-
return
|
|
1651
|
+
return 'admit';
|
|
1185
1652
|
}
|
|
1186
1653
|
const authorized = await this.listAuthorizedMembers();
|
|
1187
1654
|
if (authorized.length === 0) {
|
|
1188
|
-
return
|
|
1655
|
+
return 'admit';
|
|
1189
1656
|
}
|
|
1190
1657
|
if (authorized.some((m) => m.peerId === remotePeerId)) {
|
|
1191
|
-
return
|
|
1658
|
+
return 'admit';
|
|
1192
1659
|
}
|
|
1193
1660
|
try {
|
|
1194
1661
|
if (await this.strandSolicitationService?.hasOutstandingInvitation()) {
|
|
1195
|
-
return
|
|
1662
|
+
return 'admit';
|
|
1196
1663
|
}
|
|
1197
1664
|
} catch (error) {
|
|
1198
1665
|
log('admitInboundControlConnection: outstanding-invitation check threw for %s — admitting (fail-open): %o', remotePeerId, error);
|
|
1199
|
-
return
|
|
1666
|
+
return 'admit';
|
|
1667
|
+
}
|
|
1668
|
+
if (this.relayServerEnabled()) {
|
|
1669
|
+
log('admitInboundControlConnection: admitting %s FOR RELAY ONLY — not an authorized member and no enrollment path open; the gater drops the connection unless a reservation is admitted', remotePeerId);
|
|
1670
|
+
return 'admit-for-relay';
|
|
1200
1671
|
}
|
|
1201
1672
|
log('admitInboundControlConnection: DENYING inbound from %s — not an authorized member and no enrollment path open', remotePeerId);
|
|
1202
|
-
return
|
|
1673
|
+
return 'deny';
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1676
|
+
/**
|
|
1677
|
+
* Should `remotePeerId` be granted a circuit-relay reservation slot on this
|
|
1678
|
+
* node's relay server? The policy behind the gater's
|
|
1679
|
+
* `denyInboundRelayReservation` hook (see `membership-connection-gater.ts` →
|
|
1680
|
+
* "The relay-reservation seam"); the circuit-relay server consults it per
|
|
1681
|
+
* RESERVE request, so it is never called on a node whose relay server is off.
|
|
1682
|
+
*
|
|
1683
|
+
* Admits when ANY of:
|
|
1684
|
+
* 1. a shared-baseline check admits ({@link admitControlPeerUnconditionally}
|
|
1685
|
+
* — not running / DB torn down, absent-or-empty trusted-owner anchor, or
|
|
1686
|
+
* configured bootstrap/relay infrastructure);
|
|
1687
|
+
* 2. the peer holds a live delegate admission grant — a member's strand
|
|
1688
|
+
* node reserving here (see `delegate-admission.ts`); never counted
|
|
1689
|
+
* against the unauthorized budget;
|
|
1690
|
+
* 3. the authorized-member set is empty (cold start) or the peer IS an
|
|
1691
|
+
* authorized member — members are never counted against the budget; or
|
|
1692
|
+
* 4. the peer takes (or already holds) a slot in the bounded
|
|
1693
|
+
* unauthorized-reservation budget — the boot-ordering window where a
|
|
1694
|
+
* genuine member reserves before its `CadrePeer` row replicates here.
|
|
1695
|
+
* Cap via `network.unauthorizedRelayReservationCap` (0 = refuse every
|
|
1696
|
+
* unauthorized reservation).
|
|
1697
|
+
*
|
|
1698
|
+
* Every uncounted admission also RELEASES the budget slot the peer may hold
|
|
1699
|
+
* from an earlier reservation taken while it was still unplaceable — a member
|
|
1700
|
+
* that boots, reserves, and only then has its row replicate here would
|
|
1701
|
+
* otherwise keep that slot spent for the rest of the entry's TTL.
|
|
1702
|
+
*
|
|
1703
|
+
* The connection gate's stranger carve-outs (an open enrollment window, an
|
|
1704
|
+
* outstanding formation invitation) deliberately do NOT extend here: they
|
|
1705
|
+
* exist so a stranger's SEED or FORMATION stream can ride a connection, and
|
|
1706
|
+
* neither needs relay capacity. A genuine invitee that does need a relay slot
|
|
1707
|
+
* takes one from the budget like any other unplaced peer, so an open window
|
|
1708
|
+
* never becomes an unbounded grant of this node's forwarding capacity.
|
|
1709
|
+
*/
|
|
1710
|
+
private async admitControlRelayReservation(remotePeerId: string): Promise<boolean> {
|
|
1711
|
+
if (this.admitControlPeerUnconditionally(remotePeerId)) {
|
|
1712
|
+
return this.admitReservationUncounted(remotePeerId);
|
|
1713
|
+
}
|
|
1714
|
+
if (this.delegateAdmission.has(remotePeerId)) {
|
|
1715
|
+
return this.admitReservationUncounted(remotePeerId);
|
|
1716
|
+
}
|
|
1717
|
+
const authorized = await this.listAuthorizedMembers();
|
|
1718
|
+
if (authorized.length === 0 || authorized.some((m) => m.peerId === remotePeerId)) {
|
|
1719
|
+
return this.admitReservationUncounted(remotePeerId);
|
|
1720
|
+
}
|
|
1721
|
+
const admitted = this.unauthorizedRelayReservations.tryAdmit(remotePeerId);
|
|
1722
|
+
log('admitControlRelayReservation: %s %s under the unauthorized-reservation budget',
|
|
1723
|
+
admitted ? 'admitting' : 'REFUSING', remotePeerId);
|
|
1724
|
+
return admitted;
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
/** Admit a reservation on the peer's own merits, giving back any budget slot it still holds. */
|
|
1728
|
+
private admitReservationUncounted(remotePeerId: string): true {
|
|
1729
|
+
this.unauthorizedRelayReservations.release(remotePeerId);
|
|
1730
|
+
return true;
|
|
1203
1731
|
}
|
|
1204
1732
|
|
|
1205
1733
|
/**
|
|
@@ -1217,7 +1745,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1217
1745
|
/**
|
|
1218
1746
|
* Record (or refresh) a delegate admission grant: `delegatePeerId` is the
|
|
1219
1747
|
* transport peerId of `announcerPeerId`'s strand-`strandId` node, admitted
|
|
1220
|
-
* at the CONNECTION
|
|
1748
|
+
* at the CONNECTION and RESERVATION levels (all a circuit-relay reservation
|
|
1749
|
+
* needs, without spending the unauthorized-reservation budget) for
|
|
1221
1750
|
* `DELEGATE_GRANT_TTL_MS`. Called by the strand-addr responder after its
|
|
1222
1751
|
* authorized-membership gate has passed; public so tests can drive the
|
|
1223
1752
|
* admission policy without a full strand launch. A re-announce for the same
|
|
@@ -1325,15 +1854,83 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1325
1854
|
*
|
|
1326
1855
|
* The SOLE caller is {@link drainMembershipGate}, which serializes refreshes —
|
|
1327
1856
|
* so two reads can no longer settle out of order.
|
|
1857
|
+
*
|
|
1858
|
+
* Reads WITHOUT the transient-failure retry, deliberately: `ControlDatabase` drives
|
|
1859
|
+
* this listener with its write lock HELD (`notifyMembershipChanged`), so a retrying
|
|
1860
|
+
* read here would sleep its backoff holding the lock and stall every other local
|
|
1861
|
+
* writer. Nothing is lost — this refresh already keeps the previous snapshot on
|
|
1862
|
+
* failure and is re-driven by the next membership write and by the timed reconcile.
|
|
1328
1863
|
*/
|
|
1329
1864
|
private async refreshAuthorizedControlPeers(reason: string): Promise<void> {
|
|
1330
1865
|
if (!this._running || !this.controlDatabase) {
|
|
1331
1866
|
return;
|
|
1332
1867
|
}
|
|
1333
1868
|
try {
|
|
1334
|
-
const members = await this.listAuthorizedMembers();
|
|
1869
|
+
const members = await this.listAuthorizedMembers(false);
|
|
1335
1870
|
this.authorizedControlPeers = new Set(members.map((m) => m.peerId));
|
|
1336
1871
|
log('refreshAuthorizedControlPeers(%s): %d authorized peer(s)', reason, this.authorizedControlPeers.size);
|
|
1872
|
+
// Remember the party's size for the NEXT launch's control-node repair yardstick
|
|
1873
|
+
// (see `enrolled-machine-store.ts`): this is the only place the count is known,
|
|
1874
|
+
// and the control node that needs it was built long before this ran. Recorded
|
|
1875
|
+
// from the snapshot just materialized — no second membership query.
|
|
1876
|
+
//
|
|
1877
|
+
// CONTROL-NETWORK ONLY, and that scope is load-bearing. Every enrolled machine
|
|
1878
|
+
// runs the control node by construction, so here the party's machine count IS
|
|
1879
|
+
// the count of machines serving the network — the quantity
|
|
1880
|
+
// `resolveRepairYardstick` asks for. It is NOT that quantity for a strand, which
|
|
1881
|
+
// launches only on machines whose embedder registered its sApp config (see
|
|
1882
|
+
// {@link addStrand}), so this number must never be routed to a strand node:
|
|
1883
|
+
// over-declaring pins Optimystic's repair corroboration floor at two peers and a
|
|
1884
|
+
// strand that can field only one can then never repair. `launchStrand` therefore
|
|
1885
|
+
// declares nothing; see the NOTE there.
|
|
1886
|
+
//
|
|
1887
|
+
// `+ 1` for this node, which is not in its own authorized set. An EMPTY snapshot
|
|
1888
|
+
// is recorded as 1 rather than skipped. That is deliberate, and the reason
|
|
1889
|
+
// is a party that genuinely SHRANK: if empty meant "record nothing", a party
|
|
1890
|
+
// whose other machines were all revoked would keep declaring its old, larger
|
|
1891
|
+
// number forever, and over-declaring is the unsafe direction — at a yardstick of
|
|
1892
|
+
// 3 or more Optimystic pins the repair corroboration floor at two peers, so a
|
|
1893
|
+
// cohort that can only field one peer could never repair at all. Recording 1
|
|
1894
|
+
// lets the number come back down; the floor then turns it into 2, which is
|
|
1895
|
+
// exactly what declaring nothing resolves to.
|
|
1896
|
+
//
|
|
1897
|
+
// NOTE: the cost of that choice is that a TRANSIENT empty read overwrites a good
|
|
1898
|
+
// remembered count with 1 — a membership snapshot is legitimately empty early in
|
|
1899
|
+
// a run on a node whose rows have not replicated yet or whose trusted-owner
|
|
1900
|
+
// anchor is still unseeded (`listAuthorizedMembers` authorizes no one without an
|
|
1901
|
+
// anchor). Bounded and safe: the next refresh once rows land re-records the real
|
|
1902
|
+
// count, and the only node that loses anything is one stopped inside that window,
|
|
1903
|
+
// which then declares 2 — today's behaviour — on its next launch. If a node ever
|
|
1904
|
+
// needs to declare the right number on its FIRST post-restart launch, the fix is
|
|
1905
|
+
// to distinguish "no rows yet" from "no members" at this site, not to skip empty.
|
|
1906
|
+
//
|
|
1907
|
+
// A removed peer stops counting at REVOCATION, not at reap: `queryCadrePeers`
|
|
1908
|
+
// drops rows whose `StampId` is retired, and `listAuthorizedMembers` reads
|
|
1909
|
+
// through it. That is why there is no reap hook writing this record.
|
|
1910
|
+
//
|
|
1911
|
+
// `void`: the store never rejects and logs its own persist failures, and this
|
|
1912
|
+
// method's contract is never-rejects. A failed write leaves the in-memory count
|
|
1913
|
+
// correct for this session and re-lands on the next refresh.
|
|
1914
|
+
void this.enrolledMachineStore?.record(this.authorizedControlPeers.size + 1);
|
|
1915
|
+
// Membership moved: give the control backfill another look at every
|
|
1916
|
+
// connected peer. The production join order is connect-then-authorize, so
|
|
1917
|
+
// a joiner's first catch-up pass was denied at the gate while its
|
|
1918
|
+
// connection stays up — without this re-arm it would wait for a
|
|
1919
|
+
// reconnect. Cheap and idempotent: caught-up peers are skipped before any
|
|
1920
|
+
// timer is set, a peer whose pass is in flight is deferred to the end of
|
|
1921
|
+
// that pass (the pass may already be past its gate check — this very
|
|
1922
|
+
// commit is what would authorize it), and the gate re-judges at push time.
|
|
1923
|
+
//
|
|
1924
|
+
// NOTE: every refresh re-arms every connected peer that is not yet caught
|
|
1925
|
+
// up, and each re-armed pass costs one `CadrePeer` query at the gate. A
|
|
1926
|
+
// connection that is admitted but never authorized — a configured
|
|
1927
|
+
// bootstrap or relay peer, the steady state for a NAT'd node — is
|
|
1928
|
+
// therefore re-judged on every membership write and every timed reconcile
|
|
1929
|
+
// tick, forever. Negligible at a cadre's handful of connections; if a node
|
|
1930
|
+
// ever holds many such connections, or refreshes get frequent, skip
|
|
1931
|
+
// re-arming peers whose last pass was denied under the SAME membership
|
|
1932
|
+
// snapshot instead of re-arming unconditionally.
|
|
1933
|
+
this.controlBackfill?.scheduleConnectedPeers();
|
|
1337
1934
|
} catch (error) {
|
|
1338
1935
|
log('refreshAuthorizedControlPeers(%s) failed — keeping previous snapshot: %o', reason, error);
|
|
1339
1936
|
}
|
|
@@ -1567,6 +2164,30 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1567
2164
|
return null;
|
|
1568
2165
|
}
|
|
1569
2166
|
|
|
2167
|
+
/**
|
|
2168
|
+
* Does the `Strand` row name THIS machine as its founder — i.e. is its
|
|
2169
|
+
* `FounderOwnerKey` this node's own owner key (the key behind its PeerId)?
|
|
2170
|
+
* A null/absent column (a consent-seated strand, or a hand-built row) and a
|
|
2171
|
+
* node with no owner key both derive `false`: without a positive match this
|
|
2172
|
+
* machine must attach, never bootstrap. Pure key derivation, no I/O — cheap
|
|
2173
|
+
* enough for {@link launchStrand}'s tracked-instance early return.
|
|
2174
|
+
*
|
|
2175
|
+
* NOTE: reads "one owner key per machine" — the reference model, where the owner
|
|
2176
|
+
* key IS the key behind the PeerId. Two machines running the SAME identity key
|
|
2177
|
+
* would both derive `true` and each bootstrap on its own replica, which is the
|
|
2178
|
+
* double-`Header` hazard this derivation exists to avoid. Unreachable today: that
|
|
2179
|
+
* configuration also gives both machines one PeerId, which already breaks control
|
|
2180
|
+
* networking well before any strand launches. Revisit if machines ever share an
|
|
2181
|
+
* owner key while holding distinct transport identities — the derivation would
|
|
2182
|
+
* then need a per-machine discriminator (the `CadrePeer` PeerId) on the row.
|
|
2183
|
+
*/
|
|
2184
|
+
private isSelfFoundedRow(strand: StrandRow): boolean {
|
|
2185
|
+
if (strand.FounderOwnerKey == null) {
|
|
2186
|
+
return false;
|
|
2187
|
+
}
|
|
2188
|
+
return strand.FounderOwnerKey === this.getSelfSigningKey()?.publicKeyB64;
|
|
2189
|
+
}
|
|
2190
|
+
|
|
1570
2191
|
/**
|
|
1571
2192
|
* The owner keypair (base64url Ed25519) derived from this node's resolved
|
|
1572
2193
|
* identity key. In the single-key reference model the owner signing key is
|
|
@@ -1635,6 +2256,20 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1635
2256
|
};
|
|
1636
2257
|
this.controlNode.addEventListener('self:peer:update', this.selfPeerUpdateHandler);
|
|
1637
2258
|
|
|
2259
|
+
// NOTE: a node AUTHORIZED AFTER IT BOOTED can wait a whole heartbeat (7.5 min)
|
|
2260
|
+
// to publish its address. `registerSelf` no-ops while a node has no readable
|
|
2261
|
+
// `CadrePeer` row, so its boot-time publish does nothing, and the two triggers
|
|
2262
|
+
// wired here are the only ones after that: `self:peer:update` needs an ADDRESS
|
|
2263
|
+
// CHANGE, and a relay-only node's addresses stop changing the moment its
|
|
2264
|
+
// reservation lands — inside `start()`, before this listener exists. Harmless
|
|
2265
|
+
// where a late-authorized node's addresses keep churning (NAT/relay rotation
|
|
2266
|
+
// fire the event anyway), and invisible while parties authorize members before
|
|
2267
|
+
// they boot. If late enrollment becomes the normal path — an invited phone is
|
|
2268
|
+
// exactly that — republish on the membership-change seam that already exists
|
|
2269
|
+
// (`ControlDatabase.setMembershipChangeListener`) rather than lengthening this
|
|
2270
|
+
// list of triggers. Measured while writing
|
|
2271
|
+
// `relay-only-control-addr.integration.ts` case 4, which drives `registerSelf()`
|
|
2272
|
+
// explicitly because neither trigger is dependable there.
|
|
1638
2273
|
this.recordRefreshTimer = setInterval(() => republish('heartbeat'), DEFAULT_PEER_RECORD_HEARTBEAT_MS);
|
|
1639
2274
|
(this.recordRefreshTimer as { unref?: () => void } | null)?.unref?.();
|
|
1640
2275
|
|
|
@@ -1715,6 +2350,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1715
2350
|
* Survivors are returned signaling (`/p2p-circuit`) first, filtered to
|
|
1716
2351
|
* signaling-only when requested, as parsed `Multiaddr`s (unparsable addrs
|
|
1717
2352
|
* dropped). Any gate failure yields an empty array rather than throwing.
|
|
2353
|
+
*
|
|
2354
|
+
* Every returned address is normalized to terminate in `/p2p/<peerId>` — see
|
|
2355
|
+
* {@link normalizeDialAddrs} for why that invariant, not each dial site's own
|
|
2356
|
+
* handling, is what keeps a mixed list from taking a whole peer offline.
|
|
1718
2357
|
*/
|
|
1719
2358
|
async resolvePeerAddrs(peerId: string, opts: ResolveOpts = {}): Promise<Multiaddr[]> {
|
|
1720
2359
|
if (!this.controlDatabase) {
|
|
@@ -1767,12 +2406,77 @@ export class CadreNode implements SAppIdLookup {
|
|
|
1767
2406
|
|
|
1768
2407
|
// Order signaling-first (the on-record order was what we verified above),
|
|
1769
2408
|
// optionally restrict to signaling addrs, then parse — dropping any addr
|
|
1770
|
-
// that does not parse as a multiaddr
|
|
2409
|
+
// that does not parse as a multiaddr — and finally normalize every survivor
|
|
2410
|
+
// onto `/p2p/<peerId>` so no caller ever sees a mixed list.
|
|
1771
2411
|
let addrs = orderSignalingFirst(record.addrs);
|
|
1772
2412
|
if (opts.signalingOnly) {
|
|
1773
2413
|
addrs = addrs.filter(isSignalingAddr);
|
|
1774
2414
|
}
|
|
1775
|
-
return this.parseMultiaddrs(addrs);
|
|
2415
|
+
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
2416
|
+
}
|
|
2417
|
+
|
|
2418
|
+
/**
|
|
2419
|
+
* Guarantee every address handed to a control-network dial terminates in
|
|
2420
|
+
* `/p2p/<peerId>`, with no duplicates — the invariant every dial path
|
|
2421
|
+
* downstream relies on.
|
|
2422
|
+
*
|
|
2423
|
+
* `libp2p.dial(addrs)` requires that the addresses in ONE dial either all name
|
|
2424
|
+
* a peer id or none do; a list mixing a circuit address (which ends in
|
|
2425
|
+
* `/p2p/<target>`) with a bare direct one (which ends in `/tcp/…`) makes that
|
|
2426
|
+
* call throw, and the whole peer is then silently skipped rather than one bad
|
|
2427
|
+
* address.
|
|
2428
|
+
*
|
|
2429
|
+
* Applied by ALL THREE sources of control-dial candidates, so no call site
|
|
2430
|
+
* invents its own rule and none can produce a mixed list: the signed record
|
|
2431
|
+
* ({@link resolvePeerAddrs}), the libp2p address book
|
|
2432
|
+
* ({@link peerStoreAddrs}), and a seed's retained owner addresses
|
|
2433
|
+
* ({@link bootstrapDialAddrs}).
|
|
2434
|
+
*
|
|
2435
|
+
* An address naming a DIFFERENT trailing peer id is dropped: it does not reach
|
|
2436
|
+
* `peerId`, so it does not belong in this peer's candidate list (libp2p's own
|
|
2437
|
+
* dial queue filters the same shape out one layer lower).
|
|
2438
|
+
*/
|
|
2439
|
+
private normalizeDialAddrs(addrs: Multiaddr[], peerId: string): Multiaddr[] {
|
|
2440
|
+
// Keyed by the NORMALIZED string, so two entries differing only by the suffix
|
|
2441
|
+
// (`/ip4/…/tcp/4001` and `/ip4/…/tcp/4001/p2p/<peerId>`) collapse into one.
|
|
2442
|
+
// Normalization is what makes them equal; a surviving duplicate would cost a
|
|
2443
|
+
// real dial attempt — a whole slice of `dialWake`'s budget — on an address
|
|
2444
|
+
// already being tried.
|
|
2445
|
+
const out = new Map<string, Multiaddr>();
|
|
2446
|
+
for (const addr of addrs) {
|
|
2447
|
+
const bound = this.bindAddrToPeer(addr, peerId);
|
|
2448
|
+
if (bound) {
|
|
2449
|
+
out.set(bound.toString(), bound);
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
return [...out.values()];
|
|
2453
|
+
}
|
|
2454
|
+
|
|
2455
|
+
/**
|
|
2456
|
+
* One address bound to `peerId`, or `null` when it cannot be — logged either
|
|
2457
|
+
* way, never thrown, so every caller's list-shaping stays total.
|
|
2458
|
+
*
|
|
2459
|
+
* Used both to normalize addresses this node DIALS (a `CadrePeer` row, a
|
|
2460
|
+
* seed-supplied bootstrap addr) and to normalize the ones it ANNOUNCES
|
|
2461
|
+
* ({@link getStrandMultiaddrs}); the rule is the same either way — an address
|
|
2462
|
+
* that does not reach `peerId` is not an address for `peerId`.
|
|
2463
|
+
*
|
|
2464
|
+
* `withTrailingPeerId` encapsulates `/p2p/<peerId>`, which throws on a peer id
|
|
2465
|
+
* that does not parse. Unreachable for a `CadrePeer` row (the binding gate
|
|
2466
|
+
* above parsed it already) and for our own node's id, reachable for a
|
|
2467
|
+
* seed-supplied one ({@link bootstrapDialAddrs}).
|
|
2468
|
+
*/
|
|
2469
|
+
private bindAddrToPeer(addr: Multiaddr, peerId: string): Multiaddr | null {
|
|
2470
|
+
try {
|
|
2471
|
+
const bound = withTrailingPeerId(addr, peerId);
|
|
2472
|
+
if (!bound) {
|
|
2473
|
+
log('bindAddrToPeer: dropping addr %s — it names a peer other than %s', addr.toString(), peerId);
|
|
2474
|
+
}
|
|
2475
|
+
return bound;
|
|
2476
|
+
} catch (error) {
|
|
2477
|
+
log('bindAddrToPeer: cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
2478
|
+
return null;
|
|
2479
|
+
}
|
|
1776
2480
|
}
|
|
1777
2481
|
|
|
1778
2482
|
/** Parse multiaddr strings, dropping (and logging) any that fail to parse. */
|
|
@@ -2107,6 +2811,16 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2107
2811
|
* Cold-start fallback: the libp2p peerStore multiaddrs for `peerId` (seeded by
|
|
2108
2812
|
* {@link SeedBootstrapService.applySeed}). Returns `[]` on a missing entry or any
|
|
2109
2813
|
* parse/lookup failure — never throws.
|
|
2814
|
+
*
|
|
2815
|
+
* Normalized through {@link normalizeDialAddrs} for the same reason
|
|
2816
|
+
* {@link resolvePeerAddrs} is, and with more cause: the address book hands back
|
|
2817
|
+
* a list that is inherently MIXED. `@libp2p/peer-store`'s
|
|
2818
|
+
* `dedupeFilterAndSortAddresses` strips a trailing `/p2p/<peerId>` only when
|
|
2819
|
+
* that id is the address's FIRST `/p2p/` component, so a direct address
|
|
2820
|
+
* round-trips bare while a relayed one — whose first `/p2p/` names the relay —
|
|
2821
|
+
* keeps its suffix (the same asymmetry `peer-addr-book.ts`'s `addrKey` exists
|
|
2822
|
+
* to absorb). Feeding both to one `dial()` is exactly the
|
|
2823
|
+
* `InvalidParametersError` that skipped the whole peer.
|
|
2110
2824
|
*/
|
|
2111
2825
|
private async peerStoreAddrs(peerId: string): Promise<Multiaddr[]> {
|
|
2112
2826
|
if (!this.controlNode) {
|
|
@@ -2116,7 +2830,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2116
2830
|
const peer = await this.controlNode.peerStore.get(peerIdFromString(peerId));
|
|
2117
2831
|
// Re-parse through the top-level multiaddr parser so the returned type matches
|
|
2118
2832
|
// resolvePeerAddrs (the peerStore bundles its own @multiformats/multiaddr copy).
|
|
2119
|
-
return this.
|
|
2833
|
+
return this.normalizeDialAddrs(
|
|
2834
|
+
this.parseMultiaddrs(peer.addresses.map((a) => a.multiaddr.toString())),
|
|
2835
|
+
peerId
|
|
2836
|
+
);
|
|
2120
2837
|
} catch (error) {
|
|
2121
2838
|
log('reconcileControlCohort: peerStore lookup for %s failed: %o', peerId, error);
|
|
2122
2839
|
return [];
|
|
@@ -2207,6 +2924,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2207
2924
|
if (!this._running || !this.controlNode) {
|
|
2208
2925
|
return;
|
|
2209
2926
|
}
|
|
2927
|
+
// NOTE: a connection this node still holds in `status: 'open'` after the remote
|
|
2928
|
+
// has already aborted it counts as connected here, so the cold-start retry stays
|
|
2929
|
+
// suppressed for that peer until the connection monitor's next ping notices the
|
|
2930
|
+
// abort — measured at ~9 s in `control-cohort-cold-start-retry.integration.ts`.
|
|
2931
|
+
// Bounded and self-healing today; it would start to matter if the connection
|
|
2932
|
+
// monitor's ping interval were lengthened, or if recovery ever had to meet a
|
|
2933
|
+
// deadline tighter than one ping interval.
|
|
2210
2934
|
if (connected.has(peerId)) {
|
|
2211
2935
|
continue;
|
|
2212
2936
|
}
|
|
@@ -2222,32 +2946,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
2222
2946
|
* Bind a bootstrap peer's seed addresses to its peer id, so the dial
|
|
2223
2947
|
* authenticates the peer it is aiming at rather than trusting whoever answers.
|
|
2224
2948
|
*
|
|
2225
|
-
*
|
|
2226
|
-
*
|
|
2227
|
-
*
|
|
2228
|
-
*
|
|
2229
|
-
* bare
|
|
2949
|
+
* The same {@link normalizeDialAddrs} rule the resolved and address-book paths
|
|
2950
|
+
* use, on the third and last source of control-dial candidates: an address that
|
|
2951
|
+
* already terminates in `/p2p/<id>` must carry THIS peer's id or it is dropped
|
|
2952
|
+
* (a seed that disagrees with itself is not a dial target); one that does not —
|
|
2953
|
+
* a bare listen addr, or a relay hop with the destination missing — gets the id
|
|
2954
|
+
* encapsulated; an unencapsulatable one is dropped rather than dialed bare.
|
|
2230
2955
|
*/
|
|
2231
2956
|
private bootstrapDialAddrs(peerId: string, addrs: string[]): Multiaddr[] {
|
|
2232
|
-
|
|
2233
|
-
for (const addr of this.parseMultiaddrs(addrs)) {
|
|
2234
|
-
const embedded = trailingPeerId(addr);
|
|
2235
|
-
if (embedded === peerId) {
|
|
2236
|
-
out.push(addr);
|
|
2237
|
-
continue;
|
|
2238
|
-
}
|
|
2239
|
-
if (embedded !== null) {
|
|
2240
|
-
log('reconcileControlCohort(cold-start): addr %s names peer %s, not %s; dropping',
|
|
2241
|
-
addr.toString(), embedded, peerId);
|
|
2242
|
-
continue;
|
|
2243
|
-
}
|
|
2244
|
-
try {
|
|
2245
|
-
out.push(addr.encapsulate(`/p2p/${peerId}`));
|
|
2246
|
-
} catch (error) {
|
|
2247
|
-
log('reconcileControlCohort(cold-start): cannot bind %s to %s: %o', addr.toString(), peerId, error);
|
|
2248
|
-
}
|
|
2249
|
-
}
|
|
2250
|
-
return out;
|
|
2957
|
+
return this.normalizeDialAddrs(this.parseMultiaddrs(addrs), peerId);
|
|
2251
2958
|
}
|
|
2252
2959
|
|
|
2253
2960
|
/** Dial one cold-start bootstrap peer, best-effort. Returns whether it connected. */
|
|
@@ -3013,7 +3720,32 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3013
3720
|
}
|
|
3014
3721
|
}
|
|
3015
3722
|
|
|
3723
|
+
/**
|
|
3724
|
+
* Return this node to its pre-{@link start} state. The ONE teardown — `stop()` and
|
|
3725
|
+
* a failed `start()` both come here, so anything a stopped node must not still be
|
|
3726
|
+
* doing belongs in this method and not in `stop()`. `stop()` adds only the
|
|
3727
|
+
* `control:disconnected` emit, which a never-connected start has no edge for.
|
|
3728
|
+
*/
|
|
3016
3729
|
private async cleanup(): Promise<void> {
|
|
3730
|
+
// Open the connection gate's bring-up quiet period unconditionally: teardown
|
|
3731
|
+
// must never be gated, and this is the path a FAILED start() takes.
|
|
3732
|
+
this.controlBringUpInFlight = false;
|
|
3733
|
+
|
|
3734
|
+
// Stop the relay retry loop BEFORE tearing the node down, not after: the control
|
|
3735
|
+
// node is stopped partway through below, and a tick that fires during it would
|
|
3736
|
+
// dial and then poll a half-torn-down node for the whole reserve timeout —
|
|
3737
|
+
// logging spurious dial failures on a `.unref()`'d timer that nothing ever
|
|
3738
|
+
// clears. `driveControlRelayReservation` starts this supervisor from inside
|
|
3739
|
+
// `start()`, so a start that fails ON the reservation leaves one running unless
|
|
3740
|
+
// it is stopped here.
|
|
3741
|
+
this.relayReserveSupervisor?.stop();
|
|
3742
|
+
this.relayReserveSupervisor = null;
|
|
3743
|
+
// Drop the rest of the relay-reservation posture: a torn-down node holds no
|
|
3744
|
+
// reservation, so neither a restarted instance nor a caller inspecting a failed
|
|
3745
|
+
// start reports the previous attempt's `reserved`.
|
|
3746
|
+
this.relayReserveAddrs = [];
|
|
3747
|
+
this.relayReserveError = null;
|
|
3748
|
+
|
|
3017
3749
|
// Resolve + clear any in-flight wake windows first, so an in-flight check-in
|
|
3018
3750
|
// or serviceWake unblocks and tears down cleanly rather than firing a stale
|
|
3019
3751
|
// window timer (or hanging) after the strand manager is stopped below.
|
|
@@ -3080,6 +3812,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3080
3812
|
this.delegateAdmission.clear();
|
|
3081
3813
|
this.delegateAnnounceAt.clear();
|
|
3082
3814
|
|
|
3815
|
+
// Control backfill first — before the database closes and the control node
|
|
3816
|
+
// stops — so no NEW catch-up push is issued against a torn-down transport.
|
|
3817
|
+
// A push already in flight is not awaited; it fails into the module's own
|
|
3818
|
+
// per-chunk catch. (Mirrors releaseRuntime in strand-instance-manager.ts.)
|
|
3819
|
+
if (this.controlBackfill) {
|
|
3820
|
+
this.controlBackfill.stop();
|
|
3821
|
+
this.controlBackfill = null;
|
|
3822
|
+
}
|
|
3823
|
+
|
|
3083
3824
|
// Close control database (this also shuts down the collection factory).
|
|
3084
3825
|
// Detach the membership hub first: nothing this teardown does should drive a
|
|
3085
3826
|
// gate refresh, and a notification arriving after this point is a no-op.
|
|
@@ -3096,11 +3837,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3096
3837
|
this.controlNode = null;
|
|
3097
3838
|
}
|
|
3098
3839
|
|
|
3099
|
-
// Release the control store's cache
|
|
3100
|
-
// node that write through it are down.
|
|
3101
|
-
//
|
|
3102
|
-
//
|
|
3103
|
-
// cache
|
|
3840
|
+
// Release the control store's claim on its cache LAST — after the database and
|
|
3841
|
+
// node that write through it are down. The wrapper counts holders, so this empties
|
|
3842
|
+
// and unregisters the cache only if no other scope still holds it. Dropping the
|
|
3843
|
+
// field is what lets a stop()/start() cycle re-resolve the provider against a live
|
|
3844
|
+
// cache instead of handing the restarted node a retired wrapper. Logged, never
|
|
3845
|
+
// thrown: a cache-bookkeeping failure must not abort a teardown.
|
|
3104
3846
|
if (this.controlStorage) {
|
|
3105
3847
|
const controlStorage = this.controlStorage;
|
|
3106
3848
|
this.controlStorage = null;
|
|
@@ -3108,6 +3850,13 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3108
3850
|
(err) => log('Failed to dispose control storage cache: %o', err)
|
|
3109
3851
|
);
|
|
3110
3852
|
}
|
|
3853
|
+
|
|
3854
|
+
// LAST, so every step above still ran against a node that considered itself up
|
|
3855
|
+
// (several of them early-return while `_running` is false). A failed `start()`
|
|
3856
|
+
// that had already flipped it — the reservation drive throws after that point —
|
|
3857
|
+
// must not leave `isRunning` true over a torn-down node, which would report
|
|
3858
|
+
// healthy and make the embedder's retry a no-op ("already running").
|
|
3859
|
+
this._running = false;
|
|
3111
3860
|
}
|
|
3112
3861
|
|
|
3113
3862
|
// Hibernation callbacks
|
|
@@ -3185,8 +3934,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3185
3934
|
/**
|
|
3186
3935
|
* Rebuild a quiesced strand's runtime, re-resolving the volatile cohort input
|
|
3187
3936
|
* first: the discovery seed may have grown since the strand last ran. Shared by
|
|
3188
|
-
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so
|
|
3189
|
-
*
|
|
3937
|
+
* the wake (`handleStrandWake`) and check-in (`handleStrandCheckIn`) paths so both
|
|
3938
|
+
* apply the same fresh resolution. `resumeStrand` is idempotent
|
|
3190
3939
|
* (returns the live instance unchanged) as a backstop against double-resume.
|
|
3191
3940
|
*/
|
|
3192
3941
|
private async resumeStrandRuntime(strandId: string): Promise<void> {
|
|
@@ -3197,6 +3946,9 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3197
3946
|
? peerIdFromPrivateKey(await strandTransportKey(this.identityKey, strandId)).toString()
|
|
3198
3947
|
: undefined;
|
|
3199
3948
|
const bootstrapNodes = await this.resolveCohortSeed(strandId, delegatePeerId);
|
|
3949
|
+
// NOTE: no `servingMachines` override — see `launchStrand` for why a strand node
|
|
3950
|
+
// declares no repair yardstick, and `StartStrandConfig.servingMachines` for the
|
|
3951
|
+
// count that would legitimately go here once one exists.
|
|
3200
3952
|
const instance = await this.strandManager.resumeStrand(strandId, { bootstrapNodes });
|
|
3201
3953
|
// Same reason as the launch path: `bootstrapNodes` only reaches the address
|
|
3202
3954
|
// book through @libp2p/bootstrap discovery, so merge it directly as well.
|
|
@@ -3338,6 +4090,14 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3338
4090
|
* Add a strand with its sApp configuration.
|
|
3339
4091
|
* The hosting application must provide the sApp schema when creating a strand.
|
|
3340
4092
|
*
|
|
4093
|
+
* This is the ATTACH half only — it starts the local instance and never publishes the
|
|
4094
|
+
* `Strand` row. A joiner (the row arrived over the control network) wants exactly this; a
|
|
4095
|
+
* FOUNDER wants {@link foundStrand}, which publishes and attaches in one resumable call.
|
|
4096
|
+
* With no explicit `founder` flag, founder-ness is derived from the row's
|
|
4097
|
+
* `FounderOwnerKey` (see {@link StrandConfig.founder}) — so attaching a row THIS machine
|
|
4098
|
+
* published (e.g. re-attaching its own orphan after a restart) founds it, and attaching
|
|
4099
|
+
* anyone else's row joins, without the caller needing to know which it is.
|
|
4100
|
+
*
|
|
3341
4101
|
* A rejected call leaves nothing running but DOES leave the sApp config
|
|
3342
4102
|
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
3343
4103
|
* automatic relaunch need it. So once the strand's row is visible on the control
|
|
@@ -3356,10 +4116,12 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3356
4116
|
// Store sApp config for this strand
|
|
3357
4117
|
this.sAppConfigs.set(strandRow.Id, sAppConfig);
|
|
3358
4118
|
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)',
|
|
3359
|
-
strandRow.Id, sAppConfig.id, founder ??
|
|
4119
|
+
strandRow.Id, sAppConfig.id, founder ?? 'derived');
|
|
3360
4120
|
|
|
3361
|
-
// `founder`
|
|
3362
|
-
//
|
|
4121
|
+
// An unset `founder` is DERIVED from the row inside launchStrand (this node founds
|
|
4122
|
+
// iff the row's FounderOwnerKey is its own owner key); an explicit flag wins — the
|
|
4123
|
+
// formation/responder flows pass one deliberately, since their consent-seated rows
|
|
4124
|
+
// carry a null column. See StrandConfig.founder.
|
|
3363
4125
|
return await this.launchStrand(strandRow, sAppConfig, founder);
|
|
3364
4126
|
}
|
|
3365
4127
|
|
|
@@ -3368,16 +4130,41 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3368
4130
|
* owner identity, so other cadre members discover it via control-network
|
|
3369
4131
|
* sync (their {@link StrandWatcher} fires `strand:discovered`).
|
|
3370
4132
|
*
|
|
3371
|
-
*
|
|
3372
|
-
*
|
|
3373
|
-
*
|
|
3374
|
-
*
|
|
3375
|
-
* `addStrand` (the row already exists).
|
|
4133
|
+
* **Founding a strand is TWO steps** — this one (publish the row cadre-wide) and
|
|
4134
|
+
* {@link addStrand} (start the local instance, `founder: true` to write the strand's
|
|
4135
|
+
* `Header`/founding membership). `addStrand` deliberately omits the insert: it only
|
|
4136
|
+
* starts the LOCAL instance, whereas publishing makes the strand visible cadre-wide. A
|
|
4137
|
+
* discovering peer only does `addStrand` (the row already exists). Callers founding a
|
|
4138
|
+
* strand should use {@link foundStrand}, which performs both steps and is safe to
|
|
4139
|
+
* re-run — hand-rolling the pair is what left strands half-founded when an app was
|
|
4140
|
+
* killed between them.
|
|
4141
|
+
*
|
|
4142
|
+
* **Idempotent for identical content.** A live row whose `(Type, MemberPrivateKey)` match
|
|
4143
|
+
* the arguments is the state this call would have produced, so the call logs and returns
|
|
4144
|
+
* that row instead of writing — a publish interrupted after it committed can be repeated.
|
|
4145
|
+
* DIFFERENT content on the same id throws, naming the columns that differ, rather than
|
|
4146
|
+
* surfacing the raw `UNIQUE constraint failed: Strand.Id`: silently accepting it would
|
|
4147
|
+
* let a retry reopen a closed strand or swap the key gating its reads. The read and the
|
|
4148
|
+
* insert are not atomic, so a concurrent founder (two machines of one party, same id) is
|
|
4149
|
+
* caught on the insert's uniqueness rejection and resolved the same way — re-read, no-op
|
|
4150
|
+
* on a match, rethrow otherwise. Never an overwrite either way.
|
|
4151
|
+
*
|
|
4152
|
+
* A tombstoned strand is NOT resurrected here: {@link unpublishStrand} deletes the row, so
|
|
4153
|
+
* the idempotent branch is unreachable and the ordinary publish path re-seats it, exactly
|
|
4154
|
+
* as that method documents.
|
|
4155
|
+
*
|
|
4156
|
+
* **Already stuck on `UNIQUE constraint failed: Strand.Id`?** The row is already published
|
|
4157
|
+
* — attach, do NOT republish: `addStrand` (or {@link foundStrand}) with the id and, for a
|
|
4158
|
+
* closed strand, the `MemberPrivateKey` read back from the row. `unpublishStrand` + a
|
|
4159
|
+
* fresh publish also clears it for an OPEN strand, but is destructive for a closed one
|
|
4160
|
+
* (the key exists nowhere else). No app-data wipe is needed for either.
|
|
3376
4161
|
*
|
|
3377
4162
|
* The insert is signed with the ed25519 key behind this node's PeerId — which
|
|
3378
4163
|
* {@link ed25519KeyPairFromLibp2p} also exposes as the node's owner keypair,
|
|
3379
|
-
* so peer identity and owner key are one and the same. That key
|
|
3380
|
-
*
|
|
4164
|
+
* so peer identity and owner key are one and the same. That key is also persisted on the
|
|
4165
|
+
* row as `FounderOwnerKey` (the schema pins the column to the verified signer), which is
|
|
4166
|
+
* what later lets any launch derive "this machine is the founder" from the row alone.
|
|
4167
|
+
* The key must be enrolled in `OwnerKey` (e.g. via {@link ControlDatabase.ensureOwnerKey}
|
|
3381
4168
|
* at genesis) or the schema's `Strand.AuthorizedInsert` constraint rejects the write.
|
|
3382
4169
|
* Failing loudly here is intentional: a silently-unpublished strand would run
|
|
3383
4170
|
* as a local-only island that no peer could ever discover or join.
|
|
@@ -3386,22 +4173,162 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3386
4173
|
* {@link addStrand}).
|
|
3387
4174
|
* @param type - `'o'` for open (default) or `'c'` for closed.
|
|
3388
4175
|
* @param memberPrivateKey - Optional membership key for a closed strand.
|
|
3389
|
-
* @
|
|
3390
|
-
*
|
|
3391
|
-
|
|
3392
|
-
|
|
4176
|
+
* @returns The live `Strand` row — the one just inserted, or the matching one already
|
|
4177
|
+
* there. A closed strand's caller should carry THIS row's `MemberPrivateKey` forward:
|
|
4178
|
+
* on a repeat it is the stored key, not the argument.
|
|
4179
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank, a
|
|
4180
|
+
* row with the same id holds different content, or the control DB rejects the
|
|
4181
|
+
* (unauthorized) insert.
|
|
4182
|
+
*/
|
|
4183
|
+
async publishStrand(strandId: string, type: 'o' | 'c' = 'o', memberPrivateKey?: string): Promise<StrandRow> {
|
|
3393
4184
|
const signingKey = this.requireOwnerSigningKey(`publish strand ${strandId}`);
|
|
3394
4185
|
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
3395
4186
|
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
3396
4187
|
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
3397
|
-
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
memberPrivateKey
|
|
3403
|
-
|
|
4188
|
+
// FounderOwnerKey records THIS machine as the row's publisher (the schema pins it to
|
|
4189
|
+
// the signing owner), which is what later lets a relaunch derive founder-ness.
|
|
4190
|
+
const desired: StrandRow = {
|
|
4191
|
+
Id: trimmed,
|
|
4192
|
+
Type: type,
|
|
4193
|
+
MemberPrivateKey: memberPrivateKey ?? null,
|
|
4194
|
+
FounderOwnerKey: signingKey.publicKeyB64,
|
|
4195
|
+
};
|
|
4196
|
+
|
|
4197
|
+
// Read first rather than leaning on the insert's collision for BOTH readings. The
|
|
4198
|
+
// ordinary resume (an app killed after its own publish committed) then needs no error
|
|
4199
|
+
// matching at all: the collision path can only recognise itself by the engine's message
|
|
4200
|
+
// TEXT (see isStrandIdConflict), and an upstream reword there must degrade the RARE race,
|
|
4201
|
+
// not the common resume. It also spends no stamp or signature on a known no-op.
|
|
4202
|
+
const existing = await this.controlDatabase!.queryStrand(trimmed);
|
|
4203
|
+
if (existing) {
|
|
4204
|
+
// NOTE: reads local converged state, and `queryStrand` does not filter rows whose
|
|
4205
|
+
// StampId a Revocation has retired. So on a sibling that still physically holds a row
|
|
4206
|
+
// deleted elsewhere while ALONE (the delete-while-alone gap in docs/architecture.md),
|
|
4207
|
+
// an owner-signed republish now no-ops onto that doomed row instead of re-seating the
|
|
4208
|
+
// id under a fresh stamp. Harmless while that gap is itself unresolved — both
|
|
4209
|
+
// behaviours end at the same unconverged state — but if delete-while-alone ever gains
|
|
4210
|
+
// real replay, filter retired stamps here (queryStrandStampId + the Revocation read)
|
|
4211
|
+
// so a re-seat is not mistaken for a repeat.
|
|
4212
|
+
const matched = requireMatchingStrandRow(existing, desired, 'is already published');
|
|
4213
|
+
log('publishStrand(%s): already published (type %s) with identical content — no-op', trimmed, type);
|
|
4214
|
+
return matched;
|
|
4215
|
+
}
|
|
4216
|
+
|
|
4217
|
+
try {
|
|
4218
|
+
await this.controlDatabase!.insertStrand(
|
|
4219
|
+
trimmed,
|
|
4220
|
+
type,
|
|
4221
|
+
signingKey.publicKeyB64,
|
|
4222
|
+
signMessageWith(signingKey.privateKeyB64),
|
|
4223
|
+
memberPrivateKey
|
|
4224
|
+
);
|
|
4225
|
+
} catch (error) {
|
|
4226
|
+
// Only the uniqueness collision is a candidate for the idempotent reading; every
|
|
4227
|
+
// other rejection (unauthorized signer, retired stamp) must keep surfacing.
|
|
4228
|
+
if (!isStrandIdConflict(error)) {
|
|
4229
|
+
throw error;
|
|
4230
|
+
}
|
|
4231
|
+
const landed = await this.controlDatabase!.queryStrand(trimmed);
|
|
4232
|
+
if (!landed) {
|
|
4233
|
+
// Collided, yet no row is readable: not the race this branch handles (a row
|
|
4234
|
+
// deleted between the rejection and the re-read, or a stale local view). Surface
|
|
4235
|
+
// the original rather than guessing.
|
|
4236
|
+
throw error;
|
|
4237
|
+
}
|
|
4238
|
+
log('publishStrand(%s): lost a concurrent founding race; re-read the landed row', trimmed);
|
|
4239
|
+
return requireMatchingStrandRow(landed, desired, 'landed concurrently from another founder');
|
|
4240
|
+
}
|
|
3404
4241
|
log('Published strand %s (type %s) to control DB under owner %s', trimmed, type, signingKey.publicKeyB64);
|
|
4242
|
+
return desired;
|
|
4243
|
+
}
|
|
4244
|
+
|
|
4245
|
+
/**
|
|
4246
|
+
* Found a strand — publish its row cadre-wide AND start the local instance as its
|
|
4247
|
+
* founder — in one resumable call. The single entry point for creating a strand; a
|
|
4248
|
+
* caller joining one someone else founded uses {@link addStrand} alone.
|
|
4249
|
+
*
|
|
4250
|
+
* Safe to re-run from any point of interruption, which hand-rolling
|
|
4251
|
+
* {@link publishStrand} + {@link addStrand} is not. Either half can already have
|
|
4252
|
+
* happened:
|
|
4253
|
+
*
|
|
4254
|
+
* - **row already published** → the stored row is adopted rather than re-published, so a
|
|
4255
|
+
* run killed after the insert committed no longer dies on
|
|
4256
|
+
* `UNIQUE constraint failed: Strand.Id`. For a closed strand the row's STORED
|
|
4257
|
+
* `MemberPrivateKey` wins over `config.memberPrivateKey`: a caller that mints a key per
|
|
4258
|
+
* attempt (as the reference apps do) would otherwise present a key that does not match
|
|
4259
|
+
* the membership already seated in the strand.
|
|
4260
|
+
* - **instance already running** → the tracked instance is returned, and — because the
|
|
4261
|
+
* `Strand` row records its publishing machine (`FounderOwnerKey`) — a founder request
|
|
4262
|
+
* against an instance that was first launched as a joiner now runs the (idempotent,
|
|
4263
|
+
* insert-if-absent) founder bootstrap on it rather than silently skipping it
|
|
4264
|
+
* ({@link StrandInstanceManager.foundExistingStrand}). So a strand that something else
|
|
4265
|
+
* attached first — the reference RN app's `strand:discovered` handler after a restart,
|
|
4266
|
+
* or this node's own {@link StrandWatcher} poll winning `launchStrand`'s
|
|
4267
|
+
* `resolveCohortSeed` window — still ends up with its `Strand.Header` written.
|
|
4268
|
+
*
|
|
4269
|
+
* Founder-ness is resolved FROM THE ROW, not assumed: this machine founds iff the
|
|
4270
|
+
* resolved row's `FounderOwnerKey` is its own owner key. A fresh publish records this
|
|
4271
|
+
* machine, so the common path founds; a machine that lost a concurrent founding race to
|
|
4272
|
+
* a sibling adopts the sibling's row and ATTACHES instead — deliberately, since two
|
|
4273
|
+
* machines bootstrapping the same strand on separate replicas is the double-`Header`
|
|
4274
|
+
* hazard. The returned {@link FoundStrandResult.founded} says which happened.
|
|
4275
|
+
*
|
|
4276
|
+
* `Type` is compared, not adopted: a stored row of the other type means the caller and
|
|
4277
|
+
* the control plane disagree about what this strand IS, so it throws.
|
|
4278
|
+
*
|
|
4279
|
+
* @returns The instance AND the row the strand actually runs under — read the membership
|
|
4280
|
+
* key from the returned row, not from a freshly minted one ({@link FoundStrandResult}).
|
|
4281
|
+
* @throws if the node is not started, the id is blank, a published row of the same id has
|
|
4282
|
+
* a different `Type`, or either half rejects.
|
|
4283
|
+
*/
|
|
4284
|
+
async foundStrand(config: FoundStrandConfig): Promise<FoundStrandResult> {
|
|
4285
|
+
const { strandId, type = 'o', memberPrivateKey, sAppConfig } = config;
|
|
4286
|
+
if (!this._running || !this.controlDatabase) {
|
|
4287
|
+
throw new Error(`CadreNode must be started before attempting to found strand ${strandId}`);
|
|
4288
|
+
}
|
|
4289
|
+
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
4290
|
+
const published = await this.controlDatabase.queryStrand(trimmed);
|
|
4291
|
+
const strandRow = published
|
|
4292
|
+
? this.adoptPublishedStrand(published, type, memberPrivateKey)
|
|
4293
|
+
: await this.publishStrand(trimmed, type, memberPrivateKey);
|
|
4294
|
+
// Derived, not hardcoded `true`: adopting a row another machine published means that
|
|
4295
|
+
// machine runs the bootstrap — this one must attach or it would write a second Header.
|
|
4296
|
+
const founded = this.isSelfFoundedRow(strandRow);
|
|
4297
|
+
if (!founded) {
|
|
4298
|
+
log('foundStrand(%s): the stored row was published by a different machine ' +
|
|
4299
|
+
'(FounderOwnerKey is not this node\'s owner key) — attaching as a joiner; ' +
|
|
4300
|
+
'the founder bootstrap runs on the publishing machine', trimmed);
|
|
4301
|
+
}
|
|
4302
|
+
const instance = await this.addStrand({ strandRow, sAppConfig, founder: founded });
|
|
4303
|
+
return { instance, strandRow, founded };
|
|
4304
|
+
}
|
|
4305
|
+
|
|
4306
|
+
/**
|
|
4307
|
+
* Resume onto an already-published `Strand` row: keep its stored content, reject a type
|
|
4308
|
+
* disagreement. The stored `MemberPrivateKey` is authoritative — see {@link foundStrand}
|
|
4309
|
+
* for why a caller-minted key must lose here (and why that is NOT the same rule as
|
|
4310
|
+
* {@link publishStrand}'s, which throws on a key mismatch because it is being asked to
|
|
4311
|
+
* WRITE that key, not to resume onto what already exists).
|
|
4312
|
+
*/
|
|
4313
|
+
private adoptPublishedStrand(
|
|
4314
|
+
published: StrandRow,
|
|
4315
|
+
type: 'o' | 'c',
|
|
4316
|
+
memberPrivateKey?: string
|
|
4317
|
+
): StrandRow {
|
|
4318
|
+
if (published.Type !== type) {
|
|
4319
|
+
throw new Error(
|
|
4320
|
+
`Cannot found strand ${published.Id} as type '${type}': it is already published as ` +
|
|
4321
|
+
`type '${published.Type}'. Reconcile deliberately rather than re-founding — a ` +
|
|
4322
|
+
'published open strand cannot be closed, nor a closed one opened, by re-publishing.'
|
|
4323
|
+
);
|
|
4324
|
+
}
|
|
4325
|
+
if (memberPrivateKey !== undefined && memberPrivateKey !== published.MemberPrivateKey) {
|
|
4326
|
+
log('foundStrand(%s): already published — resuming on the STORED MemberPrivateKey; ' +
|
|
4327
|
+
'the supplied key is discarded (read it back from the returned instance)', published.Id);
|
|
4328
|
+
} else {
|
|
4329
|
+
log('foundStrand(%s): already published (type %s) — adopting the stored row', published.Id, type);
|
|
4330
|
+
}
|
|
4331
|
+
return published;
|
|
3405
4332
|
}
|
|
3406
4333
|
|
|
3407
4334
|
/**
|
|
@@ -3640,23 +4567,53 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3640
4567
|
* seed, starts the strand, and registers it with the hibernation manager
|
|
3641
4568
|
* before emitting `strand:started`.
|
|
3642
4569
|
*
|
|
3643
|
-
*
|
|
3644
|
-
*
|
|
3645
|
-
*
|
|
3646
|
-
*
|
|
3647
|
-
*
|
|
4570
|
+
* Founder-ness: an explicit `founder` argument wins (the formation/responder flows pass
|
|
4571
|
+
* one deliberately — their consent-seated rows carry a null `FounderOwnerKey`); when
|
|
4572
|
+
* unset it is DERIVED from the row, so the control-discovered path
|
|
4573
|
+
* (`handleStrandAdded`) and a restart's re-attach found this machine's own strands
|
|
4574
|
+
* without the caller having to know. The derivation is pure key comparison — no I/O.
|
|
4575
|
+
*
|
|
4576
|
+
* Idempotent when the strand manager already tracks `strand.Id` — the watcher
|
|
4577
|
+
* rediscovers a row this node already started and calls this again via
|
|
4578
|
+
* `handleStrandAdded`; without the guard that re-entry would resolve a fresh
|
|
3648
4579
|
* (already-connected) cohort seed and re-emit `strand:started` for an
|
|
3649
4580
|
* instance that never stopped. The guard runs before the cohort-seed RPC
|
|
3650
|
-
* fan-out, so a rediscovery costs nothing beyond the map lookup
|
|
4581
|
+
* fan-out, so a rediscovery costs nothing beyond the map lookup plus the (pure)
|
|
4582
|
+
* founder derivation. NOT a silent no-op any more when the launch resolves as a
|
|
4583
|
+
* FOUNDER: the tracked instance may have been launched first as a joiner (an app
|
|
4584
|
+
* attach, or this node's own watcher poll winning the `resolveCohortSeed` window
|
|
4585
|
+
* below), which used to drop the founder request and leave the strand headerless —
|
|
4586
|
+
* now the tracked instance is founded in place
|
|
4587
|
+
* ({@link StrandInstanceManager.foundExistingStrand}), waking it first if quiesced
|
|
4588
|
+
* so the bootstrap actually runs before this resolves.
|
|
3651
4589
|
*/
|
|
3652
4590
|
private async launchStrand(
|
|
3653
4591
|
strand: StrandRow,
|
|
3654
4592
|
sAppConfig: SAppConfig,
|
|
3655
4593
|
founder?: boolean
|
|
3656
4594
|
): Promise<StrandInstance> {
|
|
4595
|
+
const resolvedFounder = founder ?? this.isSelfFoundedRow(strand);
|
|
3657
4596
|
const existing = this.strandManager.getInstance(strand.Id);
|
|
3658
4597
|
if (existing) {
|
|
3659
|
-
|
|
4598
|
+
if (resolvedFounder) {
|
|
4599
|
+
const outcome = await this.strandManager.foundExistingStrand(strand.Id);
|
|
4600
|
+
if (outcome === 'needs-resume') {
|
|
4601
|
+
// Quiesced instance: the retained config now founds, but founding promises
|
|
4602
|
+
// the bootstrap has RUN by the time the caller resolves — wake through the
|
|
4603
|
+
// hibernation manager (coalesced with any in-flight wake, timer-aware) so
|
|
4604
|
+
// the rebuild executes it now rather than at some eventual wake.
|
|
4605
|
+
await this.wakeStrand(strand.Id);
|
|
4606
|
+
// The wake's rebuild founds — UNLESS a wake was already in flight when the
|
|
4607
|
+
// config flipped, in which case it had already read the pre-flip config and
|
|
4608
|
+
// rebuilt as a joiner, and `wakeStrand` merely coalesced onto it. Re-run the
|
|
4609
|
+
// (insert-if-absent) bootstrap so founding never resolves headerless.
|
|
4610
|
+
await this.strandManager.ensureFounderBootstrap(strand.Id);
|
|
4611
|
+
}
|
|
4612
|
+
log('launchStrand: strand %s already tracked — founder request honored (%s)',
|
|
4613
|
+
strand.Id, outcome);
|
|
4614
|
+
} else {
|
|
4615
|
+
log('launchStrand: strand %s already tracked locally — skipping re-launch', strand.Id);
|
|
4616
|
+
}
|
|
3660
4617
|
return existing;
|
|
3661
4618
|
}
|
|
3662
4619
|
|
|
@@ -3697,8 +4654,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3697
4654
|
bootstrapNodes,
|
|
3698
4655
|
requireSignedSchemas: this.config.requireSignedSchemas,
|
|
3699
4656
|
clusterSize: this.config.strandClusterSize,
|
|
4657
|
+
// NOTE: deliberately NO `servingMachines`, so the strand node declares no repair
|
|
4658
|
+
// yardstick and runs the frozen STRAND_CLUSTER_POLICY. The only count this node holds
|
|
4659
|
+
// is the party's enrolled machines (`authorizedControlPeers`), and passing that here
|
|
4660
|
+
// is the regression `bug-strand-yardstick-counts-party-machines` removed — a strand
|
|
4661
|
+
// runs on a subset of the party, and over-declaring makes repair impossible rather
|
|
4662
|
+
// than merely weak. The full argument, and the count that will legitimately go here,
|
|
4663
|
+
// are on `StartStrandConfig.servingMachines`.
|
|
3700
4664
|
backfill: this.config.strandBackfill,
|
|
3701
|
-
|
|
4665
|
+
revocationEnforcement: this.config.strandRevocationEnforcement,
|
|
4666
|
+
onSelfRevoked: (revokedStrandId) => this.emit('strand:revoked', { strandId: revokedStrandId }),
|
|
4667
|
+
// The RESOLVED flag, never the raw argument — see the doc comment above.
|
|
4668
|
+
founder: resolvedFounder
|
|
3702
4669
|
});
|
|
3703
4670
|
|
|
3704
4671
|
// The seed reached the node as `bootstrapNodes`, which only enters the
|
|
@@ -3741,6 +4708,21 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3741
4708
|
* for a relay we are not yet connected to.
|
|
3742
4709
|
*/
|
|
3743
4710
|
private async resolveCohortSeed(strandId: string, delegatePeerId?: string): Promise<string[]> {
|
|
4711
|
+
const siblings = await this.resolveSiblingSeed(strandId, delegatePeerId);
|
|
4712
|
+
// Sibling answers FIRST: they were resolved just now, while a contact-map entry is as
|
|
4713
|
+
// old as the formation that produced it. `unionAddrs` appends only what the siblings
|
|
4714
|
+
// did not already name, so a fresher answer is never displaced and each source keeps
|
|
4715
|
+
// its own signaling-first ordering.
|
|
4716
|
+
return unionAddrs(siblings, this.crossPartyStrandAddrs.get(strandId) ?? []);
|
|
4717
|
+
}
|
|
4718
|
+
|
|
4719
|
+
/**
|
|
4720
|
+
* The own-party half of {@link resolveCohortSeed}: strand-network addresses resolved
|
|
4721
|
+
* from CONNECTED cohort siblings over the control-mesh strand-addr RPC, doubling as
|
|
4722
|
+
* the delegate announcement when `delegatePeerId` is given. Empty when the control DB
|
|
4723
|
+
* or node is absent (not yet started / torn down).
|
|
4724
|
+
*/
|
|
4725
|
+
private async resolveSiblingSeed(strandId: string, delegatePeerId?: string): Promise<string[]> {
|
|
3744
4726
|
if (!this.controlDatabase || !this.controlNode) {
|
|
3745
4727
|
return [];
|
|
3746
4728
|
}
|
|
@@ -3804,11 +4786,25 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3804
4786
|
|
|
3805
4787
|
/**
|
|
3806
4788
|
* The circuit relays this node's strand nodes would reserve through: the
|
|
3807
|
-
* union of
|
|
3808
|
-
* `network.listenAddrs` circuit entry
|
|
3809
|
-
*
|
|
3810
|
-
*
|
|
3811
|
-
*
|
|
4789
|
+
* union of CONFIGURED relays — `network.relayAddrs`, plus any hand-written
|
|
4790
|
+
* `network.listenAddrs` circuit entry, both of which a strand node inherits
|
|
4791
|
+
* verbatim — and the control node's own live `/p2p-circuit` multiaddrs (a
|
|
4792
|
+
* reservation this node discovered rather than configured).
|
|
4793
|
+
*
|
|
4794
|
+
* The CONFIGURED half reads `network.relayAddrs` and `network.listenAddrs`
|
|
4795
|
+
* directly rather than going through {@link resolveListenAddrs}: the control
|
|
4796
|
+
* node's resolution takes the `'search'` route, whose bare `/p2p-circuit` entry
|
|
4797
|
+
* names no relay, so a configured relay would be unannounceable here until this
|
|
4798
|
+
* node's own reservation landed — and the announce must be in place BEFORE a
|
|
4799
|
+
* strand node dials its reservation, since the relay's membership gate does not
|
|
4800
|
+
* know the strand's derived peerId (see delegate-admission.ts). Configuration is
|
|
4801
|
+
* authoritative; the live multiaddrs only ADD relays nobody configured.
|
|
4802
|
+
*
|
|
4803
|
+
* The `listenAddrs` half is now unreachable on a control node — `relay-addrs.ts`
|
|
4804
|
+
* rejects a hand-written `<relay>/p2p-circuit` listen entry outright, because the
|
|
4805
|
+
* bring-up quiet period denies the dial that listener makes from inside
|
|
4806
|
+
* `libp2p.start()`. Kept because it costs one spread and this is the one place
|
|
4807
|
+
* that would silently stop announcing if that rejection is ever relaxed.
|
|
3812
4808
|
*
|
|
3813
4809
|
* NOTE: a relay the STRAND node discovers on its own (autorelay — in neither
|
|
3814
4810
|
* source) gets no announcement, and a membership-gated one will deny it; fine
|
|
@@ -3816,7 +4812,8 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3816
4812
|
*/
|
|
3817
4813
|
private circuitRelayTargets(): CircuitRelayTarget[] {
|
|
3818
4814
|
return extractCircuitRelayTargets([
|
|
3819
|
-
...(
|
|
4815
|
+
...relayCircuitAddrs(this.config.network?.relayAddrs ?? []),
|
|
4816
|
+
...(this.config.network?.listenAddrs ?? []),
|
|
3820
4817
|
...(this.controlNode?.getMultiaddrs().map(String) ?? [])
|
|
3821
4818
|
]);
|
|
3822
4819
|
}
|
|
@@ -3947,36 +4944,42 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3947
4944
|
if (due.length === 0) {
|
|
3948
4945
|
return;
|
|
3949
4946
|
}
|
|
3950
|
-
// A due strand with nobody to ask leaves its throttle unstamped (see below),
|
|
3951
|
-
// so this pass re-enters on EVERY 15 s reconcile tick for as long as the node
|
|
3952
|
-
// is alone — which is the steady state of a solo node running a strand.
|
|
3953
4947
|
// `connectedSiblingTargets`' membership read is unbounded, and with zero
|
|
3954
|
-
// connections its answer is empty whatever the table holds, so decide it
|
|
3955
|
-
//
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
4948
|
+
// connections its answer is empty whatever the table holds, so decide it from the
|
|
4949
|
+
// connection list instead of paying for the read once a tick. Note this only skips
|
|
4950
|
+
// the RPC: a strand carrying cross-party contact addresses still refreshes below,
|
|
4951
|
+
// because those never came from a sibling in the first place.
|
|
4952
|
+
//
|
|
3959
4953
|
// NOTE: one strand-addr RPC per (due strand × connected sibling) per refresh
|
|
3960
4954
|
// interval — each a tiny request/response on an already-open control
|
|
3961
4955
|
// connection. If a node ever runs MANY strands at once, batch the RPC to
|
|
3962
4956
|
// carry several strand ids per request rather than one fan-out per strand.
|
|
3963
|
-
const targets =
|
|
3964
|
-
|
|
3965
|
-
|
|
3966
|
-
|
|
3967
|
-
|
|
3968
|
-
|
|
3969
|
-
//
|
|
3970
|
-
if (
|
|
4957
|
+
const targets = this.controlNode.getConnections().length === 0
|
|
4958
|
+
? []
|
|
4959
|
+
: await this.connectedSiblingTargets().catch((error): StrandAddrPeer[] => {
|
|
4960
|
+
log('refreshStrandPeerAddrs: sibling enumeration failed (skipping pass): %o', error);
|
|
4961
|
+
return [];
|
|
4962
|
+
});
|
|
4963
|
+
// A shutdown landed mid-enumeration.
|
|
4964
|
+
if (!this._running || !this.controlNode) {
|
|
3971
4965
|
return;
|
|
3972
4966
|
}
|
|
3973
4967
|
await Promise.all(due.map((strandId) => this.refreshOneStrandPeerAddrs(strandId, targets, now)));
|
|
3974
4968
|
}
|
|
3975
4969
|
|
|
3976
4970
|
/**
|
|
3977
|
-
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings,
|
|
3978
|
-
*
|
|
3979
|
-
* swallowed so one strand's failure never
|
|
4971
|
+
* One strand's share of {@link refreshStrandPeerAddrs}: RPC the siblings, union their
|
|
4972
|
+
* answers with this strand's cross-party contact addresses, and merge the lot into the
|
|
4973
|
+
* strand's address book. Errors are logged and swallowed so one strand's failure never
|
|
4974
|
+
* costs the others their refresh.
|
|
4975
|
+
*
|
|
4976
|
+
* `targets` may be EMPTY — a solo node, or one whose only control peers are strangers.
|
|
4977
|
+
* The cross-party addresses still have to be re-merged in that case: they were learned
|
|
4978
|
+
* at formation and can never be re-resolved, so this periodic re-merge is the ONLY
|
|
4979
|
+
* thing standing between them and the peerStore's one-hour expiry. A strand with
|
|
4980
|
+
* neither a sibling to ask nor a contact entry does nothing and leaves its throttle
|
|
4981
|
+
* unstamped, so the next reconcile tick retries rather than sitting out the whole
|
|
4982
|
+
* refresh interval having done nothing.
|
|
3980
4983
|
*/
|
|
3981
4984
|
private async refreshOneStrandPeerAddrs(
|
|
3982
4985
|
strandId: string,
|
|
@@ -3988,13 +4991,19 @@ export class CadreNode implements SAppIdLookup {
|
|
|
3988
4991
|
if (!controlNode || !strandNode) {
|
|
3989
4992
|
return;
|
|
3990
4993
|
}
|
|
4994
|
+
const contactAddrs = this.crossPartyStrandAddrs.get(strandId) ?? [];
|
|
4995
|
+
if (targets.length === 0 && contactAddrs.length === 0) {
|
|
4996
|
+
return;
|
|
4997
|
+
}
|
|
3991
4998
|
try {
|
|
3992
4999
|
// The running strand node's own peerId is the delegate to announce — see
|
|
3993
5000
|
// the relationship note on refreshStrandPeerAddrs.
|
|
3994
|
-
const
|
|
3995
|
-
|
|
3996
|
-
|
|
3997
|
-
|
|
5001
|
+
const siblingAddrs = targets.length === 0
|
|
5002
|
+
? []
|
|
5003
|
+
: await collectStrandAddrs(controlNode, [...targets], strandId, {
|
|
5004
|
+
delegatePeerId: strandNode.peerId.toString()
|
|
5005
|
+
});
|
|
5006
|
+
// Stamped on the pass having happened, not on its answer: the fan-out is
|
|
3998
5007
|
// what the throttle exists to bound, and an all-empty round is a normal
|
|
3999
5008
|
// steady state for a strand no connected sibling currently runs.
|
|
4000
5009
|
this.strandPeerAddrRefreshAt.set(strandId, now);
|
|
@@ -4003,7 +5012,7 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4003
5012
|
if (!this._running || this.strandManager.getInstance(strandId)?.libp2pNode !== strandNode) {
|
|
4004
5013
|
return;
|
|
4005
5014
|
}
|
|
4006
|
-
await this.mergeStrandPeerAddrs(strandNode,
|
|
5015
|
+
await this.mergeStrandPeerAddrs(strandNode, unionAddrs(siblingAddrs, contactAddrs), strandId);
|
|
4007
5016
|
} catch (error) {
|
|
4008
5017
|
log('refreshStrandPeerAddrs: strand %s refresh failed (continuing): %o', strandId, error);
|
|
4009
5018
|
}
|
|
@@ -4027,6 +5036,15 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4027
5036
|
* handshake, and a bad entry costs one failed dial that ages out at the
|
|
4028
5037
|
* peerStore's one-hour expiry. No new gating here — cross-party strand trust is
|
|
4029
5038
|
* `backlog/strand-network-nat-relay-reachability`.
|
|
5039
|
+
*
|
|
5040
|
+
* NOTE: one input does NOT age out — the cross-party addresses a formation
|
|
5041
|
+
* carried back ({@link crossPartyStrandAddrs}). Nothing can re-resolve them, so
|
|
5042
|
+
* {@link refreshStrandPeerAddrs} re-merges the same stored list every pass, which
|
|
5043
|
+
* also means a junk entry from a responder survives for the node's lifetime rather
|
|
5044
|
+
* than an hour. Bounded to {@link MAX_STRAND_ADDRS} per strand, from a party this
|
|
5045
|
+
* node deliberately formed a strand with, and still authority-free — so the
|
|
5046
|
+
* exposure is a handful of failed dials, not a trust hole. Evict on
|
|
5047
|
+
* `unpublishStrand` if that ever stops being an acceptable price.
|
|
4030
5048
|
*/
|
|
4031
5049
|
private async mergeStrandPeerAddrs(strandNode: Libp2p, addrs: string[], strandId: string): Promise<void> {
|
|
4032
5050
|
const counts: Record<MergeAddrsResult, number> = { merged: 0, restamped: 0, skipped: 0, failed: 0 };
|
|
@@ -4070,7 +5088,29 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4070
5088
|
if (!node) {
|
|
4071
5089
|
return [];
|
|
4072
5090
|
}
|
|
4073
|
-
|
|
5091
|
+
// Every entry is bound to THIS strand node's id, because both consumers — the
|
|
5092
|
+
// strand-addr RPC answer and the formation result's `strandAddrs` — are attributed
|
|
5093
|
+
// per peer on arrival (`groupAddrsByPeerId`) and an entry naming no destination is
|
|
5094
|
+
// dropped there. libp2p already appends the id to its announced addresses, so this
|
|
5095
|
+
// is normally a no-op; it is the guard for a bare listen addr and for a relay hop
|
|
5096
|
+
// whose trailing `/p2p/` names the RELAY. An addr terminating in a DIFFERENT id is
|
|
5097
|
+
// dropped rather than announced — it does not reach this node, and announcing it
|
|
5098
|
+
// would file our address under someone else's id in the receiver's book.
|
|
5099
|
+
//
|
|
5100
|
+
// Re-parsed through this package's own `multiaddr` rather than passed straight in:
|
|
5101
|
+
// libp2p hands back its nested `@multiformats/multiaddr` copy, a structurally
|
|
5102
|
+
// different type from the one `withTrailingPeerId` takes (see
|
|
5103
|
+
// `backlog/reference-app-web-libp2p-interface-dedup`). Re-parsing a string libp2p
|
|
5104
|
+
// itself produced cannot fail.
|
|
5105
|
+
const selfId = node.peerId.toString();
|
|
5106
|
+
const addrs: string[] = [];
|
|
5107
|
+
for (const ma of node.getMultiaddrs()) {
|
|
5108
|
+
const bound = this.bindAddrToPeer(multiaddr(ma.toString()), selfId);
|
|
5109
|
+
if (bound !== null) {
|
|
5110
|
+
addrs.push(bound.toString());
|
|
5111
|
+
}
|
|
5112
|
+
}
|
|
5113
|
+
return orderSignalingFirst(addrs);
|
|
4074
5114
|
}
|
|
4075
5115
|
|
|
4076
5116
|
/**
|
|
@@ -4300,16 +5340,61 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4300
5340
|
return this.controlNode;
|
|
4301
5341
|
}
|
|
4302
5342
|
|
|
5343
|
+
/**
|
|
5344
|
+
* The boot-path half of {@link reserveRelays}: reserve through every
|
|
5345
|
+
* `network.relayAddrs` entry, or fail `start()`.
|
|
5346
|
+
*
|
|
5347
|
+
* WHY IT RUNS HERE, AT THE END. `network.relayAddrs` used to resolve to a
|
|
5348
|
+
* CONFIGURED circuit listener, which libp2p dials from inside `libp2p.start()`
|
|
5349
|
+
* — so the relay was already a connected peer, and therefore already in this
|
|
5350
|
+
* node's Optimystic cohort, before `ControlDatabase.initialize()` ran. Building
|
|
5351
|
+
* that database is a long chain of cohort-consulting block probes (catalog
|
|
5352
|
+
* hydration, then one per table, then the indexes), and a sibling that has not
|
|
5353
|
+
* yet replicated this node's `CadrePeer` row correctly refuses every one of
|
|
5354
|
+
* them, so bring-up died on `BlockUnavailableError` — deterministically, on
|
|
5355
|
+
* every boot of a relay-only node. Retrying could not converge either: the
|
|
5356
|
+
* condition that clears the refusal is this node's own row reaching the
|
|
5357
|
+
* sibling, and writing that row needs the database the retry is building.
|
|
5358
|
+
*
|
|
5359
|
+
* So the invariant is ordering, not retrying: the control database is built
|
|
5360
|
+
* while this node holds ZERO control connections (a cohort of one, entirely
|
|
5361
|
+
* local), and only then does it reach out. Everything above in {@link start}
|
|
5362
|
+
* has completed by the time this runs.
|
|
5363
|
+
*
|
|
5364
|
+
* BUDGET: the supervisor's first attempt — `relay-reservation.ts`'s
|
|
5365
|
+
* `DEFAULT_RELAY_RESERVE_TIMEOUT_MS`, 10 s — is what `start()` waits on — deliberately the module default rather
|
|
5366
|
+
* than a boot-specific one. A healthy dial-plus-reserve is sub-second even over
|
|
5367
|
+
* a WAN, so 10 s is slack for a slow link; going much longer would make a dead
|
|
5368
|
+
* relay indistinguishable from a hung start, and much shorter would fail nodes
|
|
5369
|
+
* on links that were merely slow. The retries carry on in the background after
|
|
5370
|
+
* this resolves, exactly as they do for a {@link reserveRelays} caller.
|
|
5371
|
+
*/
|
|
5372
|
+
private async driveControlRelayReservation(): Promise<void> {
|
|
5373
|
+
const relayAddrs = this.config.network?.relayAddrs ?? [];
|
|
5374
|
+
if (relayAddrs.length === 0) {
|
|
5375
|
+
return;
|
|
5376
|
+
}
|
|
5377
|
+
const t0 = performance.now();
|
|
5378
|
+
const state = await this.reserveRelays([...relayAddrs]);
|
|
5379
|
+
timing('[start] reserveRelays: %dms', Math.round(performance.now() - t0));
|
|
5380
|
+
if (state.status !== 'reserved') {
|
|
5381
|
+
throw new RelayReservationFailedError(relayAddrs, state);
|
|
5382
|
+
}
|
|
5383
|
+
log('Reserved a relay slot: %o', state.circuitAddrs);
|
|
5384
|
+
}
|
|
5385
|
+
|
|
4303
5386
|
/**
|
|
4304
5387
|
* Start keeping a relay reservation: dial the given relay(s) from the control
|
|
4305
5388
|
* node, ask the first one that answers for a reservation slot, wait until the
|
|
4306
5389
|
* resulting `/p2p-circuit` address makes this node dialable — and keep a
|
|
4307
5390
|
* supervisor running that re-drives whenever the reservation is later lost.
|
|
4308
5391
|
*
|
|
4309
|
-
*
|
|
4310
|
-
*
|
|
4311
|
-
*
|
|
4312
|
-
*
|
|
5392
|
+
* The FAIL-SOFT entry point, for a caller that discovers its relay at runtime
|
|
5393
|
+
* (a browser tab, a host UI). A node that names `network.relayAddrs` gets the
|
|
5394
|
+
* same drive from {@link driveControlRelayReservation} at the end of
|
|
5395
|
+
* {@link start}, which is fail-fast instead. Both fill the pending reservation
|
|
5396
|
+
* the bare `/p2p-circuit` search listener registered; calling this afterwards
|
|
5397
|
+
* simply replaces the supervisor with one over the new list.
|
|
4313
5398
|
*
|
|
4314
5399
|
* Resolves once the FIRST attempt has settled, exactly as it did when it drove
|
|
4315
5400
|
* once; the retries continue in the background until {@link stop} or the next
|
|
@@ -4463,6 +5548,10 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4463
5548
|
* Push the multiaddrs that future invites should advertise. Pass `null` to
|
|
4464
5549
|
* revert to the libp2p-reported addresses (the default). The host calls this
|
|
4465
5550
|
* at spawn and on every NAT change.
|
|
5551
|
+
*
|
|
5552
|
+
* Entries need NOT carry a `/p2p/<peerId>` suffix — {@link resolveInviteAddresses}
|
|
5553
|
+
* appends this node's own before anything publishes or dials them (see
|
|
5554
|
+
* {@link normalizeSelfAddrs}). Passing them suffixed is equally fine.
|
|
4466
5555
|
*/
|
|
4467
5556
|
setInviteAddresses(addresses: string[] | null): void {
|
|
4468
5557
|
this.latestInviteAddresses = addresses;
|
|
@@ -4471,18 +5560,60 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4471
5560
|
|
|
4472
5561
|
/**
|
|
4473
5562
|
* Resolve the addresses to embed in invites. Prefers pushed addresses, then
|
|
4474
|
-
* any config-supplied resolver, then the libp2p-observed multiaddrs.
|
|
5563
|
+
* any config-supplied resolver, then the libp2p-observed multiaddrs. The two
|
|
5564
|
+
* app-supplied sources are normalized onto `/p2p/<self>` (see
|
|
5565
|
+
* {@link normalizeSelfAddrs}) so neither hook has to remember the suffix;
|
|
5566
|
+
* libp2p's own addresses already carry it.
|
|
4475
5567
|
*/
|
|
4476
5568
|
private async resolveInviteAddresses(): Promise<string[]> {
|
|
4477
5569
|
if (this.latestInviteAddresses !== null) {
|
|
4478
|
-
return this.latestInviteAddresses;
|
|
5570
|
+
return this.normalizeSelfAddrs(this.latestInviteAddresses);
|
|
4479
5571
|
}
|
|
4480
5572
|
if (this.config.network?.inviteAddressResolver) {
|
|
4481
|
-
return this.config.network.inviteAddressResolver();
|
|
5573
|
+
return this.normalizeSelfAddrs(await this.config.network.inviteAddressResolver());
|
|
4482
5574
|
}
|
|
5575
|
+
// Not normalized: this branch is libp2p's own output, which already
|
|
5576
|
+
// encapsulates the peer id into every address it reports. Only the two app
|
|
5577
|
+
// hooks above take arbitrary strings and therefore need the guarantee.
|
|
4483
5578
|
return this.getMultiaddrs();
|
|
4484
5579
|
}
|
|
4485
5580
|
|
|
5581
|
+
/**
|
|
5582
|
+
* Normalize addresses an app handed us for THIS node onto `/p2p/<self>`, the
|
|
5583
|
+
* same invariant {@link normalizeDialAddrs} enforces on the way out.
|
|
5584
|
+
*
|
|
5585
|
+
* `setInviteAddresses` (the admin API `PUT /admin/invite-addresses`) and
|
|
5586
|
+
* `network.inviteAddressResolver` both take arbitrary strings, and whatever
|
|
5587
|
+
* they return ends up in this node's published `CadrePeer` record — so ONE
|
|
5588
|
+
* unsuffixed entry from either hook is enough to give every sibling in the
|
|
5589
|
+
* party a mixed candidate list for this peer. Establishing the suffix here,
|
|
5590
|
+
* where those strings enter the system, means neither hook has to know the
|
|
5591
|
+
* rule; `cadre-host`'s `buildInviteAddresses` already appends it, and
|
|
5592
|
+
* re-normalizing an already-suffixed address is a no-op.
|
|
5593
|
+
*
|
|
5594
|
+
* Unparsable and other-peer-addressed entries are passed through untouched
|
|
5595
|
+
* rather than dropped: publication is not the place to police an address's
|
|
5596
|
+
* validity, and {@link resolvePeerAddrs} already drops both on the read side.
|
|
5597
|
+
* Before `start()` resolves an identity there is no peer id to append, so the
|
|
5598
|
+
* list is returned as given.
|
|
5599
|
+
*/
|
|
5600
|
+
private normalizeSelfAddrs(addrs: string[]): string[] {
|
|
5601
|
+
const selfPeerId = this.peerId?.toString();
|
|
5602
|
+
if (!selfPeerId) {
|
|
5603
|
+
return addrs;
|
|
5604
|
+
}
|
|
5605
|
+
return addrs.map((addr) => {
|
|
5606
|
+
let parsed: Multiaddr;
|
|
5607
|
+
try {
|
|
5608
|
+
parsed = multiaddr(addr);
|
|
5609
|
+
} catch (error) {
|
|
5610
|
+
log('normalizeSelfAddrs: leaving unparsable multiaddr %s as-is: %o', addr, error);
|
|
5611
|
+
return addr;
|
|
5612
|
+
}
|
|
5613
|
+
return withTrailingPeerId(parsed, selfPeerId)?.toString() ?? addr;
|
|
5614
|
+
});
|
|
5615
|
+
}
|
|
5616
|
+
|
|
4486
5617
|
/**
|
|
4487
5618
|
* Enumerate the cadre's `CadrePeer` membership — the ADDRESSABLE surface (see
|
|
4488
5619
|
* {@link isMember}). Rows whose stamp is retired in `CadreControl.Revocation`
|
|
@@ -4546,13 +5677,18 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4546
5677
|
* legit member goes un-authorized on readers that only pin the new key. Full
|
|
4547
5678
|
* rotation handling (re-vouch on rotate) is the
|
|
4548
5679
|
* `flip-strand-membership-rotation-known-gap` work, not this predicate's.
|
|
5680
|
+
*
|
|
5681
|
+
* @param retry - Whether the underlying membership read may retry a transient cluster
|
|
5682
|
+
* failure. Only {@link refreshAuthorizedControlPeers} passes `false`, because it runs
|
|
5683
|
+
* as the control database's membership listener with that database's write lock held;
|
|
5684
|
+
* see its comment.
|
|
4549
5685
|
*/
|
|
4550
|
-
async listAuthorizedMembers(): Promise<Array<{ peerId: string; multiaddr: string | null }>> {
|
|
5686
|
+
async listAuthorizedMembers(retry = true): Promise<Array<{ peerId: string; multiaddr: string | null }>> {
|
|
4551
5687
|
if (!this.controlDatabase) {
|
|
4552
5688
|
throw new Error('CadreNode must be started before listing members');
|
|
4553
5689
|
}
|
|
4554
5690
|
const selfPeerId = this.peerId?.toString();
|
|
4555
|
-
const rows = await this.controlDatabase.queryCadrePeers();
|
|
5691
|
+
const rows = await this.controlDatabase.queryCadrePeers(retry);
|
|
4556
5692
|
const authorized = rows
|
|
4557
5693
|
.filter(row => row.peerId !== selfPeerId && this.hasAnchoredVoucher(row))
|
|
4558
5694
|
.map(({ peerId, multiaddr }) => ({ peerId, multiaddr }));
|
|
@@ -4728,6 +5864,26 @@ export class CadreNode implements SAppIdLookup {
|
|
|
4728
5864
|
}
|
|
4729
5865
|
}
|
|
4730
5866
|
|
|
5867
|
+
/**
|
|
5868
|
+
* Re-materialize a CLOSED strand's revoked-peer deny set now, and hang up any
|
|
5869
|
+
* connected peer it newly covers — the strand-side counterpart to
|
|
5870
|
+
* {@link refreshMembershipGate}.
|
|
5871
|
+
*
|
|
5872
|
+
* Unlike that one, NOTHING calls this automatically: strand membership is
|
|
5873
|
+
* written through the strand's own `Database` handle
|
|
5874
|
+
* (`strand-membership-writer.ts`), which raises no notification this runtime
|
|
5875
|
+
* can hook, and a revocation that arrives by REPLICATION raises none either.
|
|
5876
|
+
* The gate therefore polls (default 30 s). An app that has just called
|
|
5877
|
+
* `revokeMember` or `leaveStrand` should follow the write with this call so
|
|
5878
|
+
* the cut is immediate rather than up to one poll interval late.
|
|
5879
|
+
*
|
|
5880
|
+
* Quiet no-op for a strand that is not running, is quiesced, is open, or has
|
|
5881
|
+
* the gate disabled. Never rejects; resolves once the sweep has finished.
|
|
5882
|
+
*/
|
|
5883
|
+
async refreshRevocationEnforcement(strandId: string): Promise<void> {
|
|
5884
|
+
await this.strandManager.refreshRevocationEnforcement(strandId);
|
|
5885
|
+
}
|
|
5886
|
+
|
|
4731
5887
|
/**
|
|
4732
5888
|
* The single in-flight refresh loop behind {@link refreshMembershipGate}: read
|
|
4733
5889
|
* until the stale flag stays clear, then release the slot.
|
|
@@ -5062,7 +6218,11 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5062
6218
|
this.strandSolicitationService = new StrandSolicitationService({
|
|
5063
6219
|
...options,
|
|
5064
6220
|
partyId: this.config.controlNetwork.partyId,
|
|
5065
|
-
cadrePeerAddrs: this.getMultiaddrs()
|
|
6221
|
+
cadrePeerAddrs: this.getMultiaddrs(),
|
|
6222
|
+
// Overrides any caller-supplied hook, exactly like partyId/cadrePeerAddrs: only
|
|
6223
|
+
// this node can say which strand-network addresses it is actually listening on,
|
|
6224
|
+
// and a wrong answer here seeds a joiner's mesh with addresses that reach nobody.
|
|
6225
|
+
resolveStrandAddrs: (strandId: string) => this.getStrandMultiaddrs(strandId)
|
|
5066
6226
|
});
|
|
5067
6227
|
|
|
5068
6228
|
// Register as responder on the control node
|
|
@@ -5124,11 +6284,46 @@ export class CadreNode implements SAppIdLookup {
|
|
|
5124
6284
|
this.initializeStrandSolicitation();
|
|
5125
6285
|
}
|
|
5126
6286
|
|
|
5127
|
-
|
|
6287
|
+
const result = await this.strandSolicitationService!.formStrand(
|
|
5128
6288
|
invitation,
|
|
5129
6289
|
disclosure,
|
|
5130
6290
|
this.controlNode
|
|
5131
6291
|
);
|
|
6292
|
+
this.recordCrossPartyStrandAddrs(result.strandId, result.strandAddrs);
|
|
6293
|
+
return result;
|
|
6294
|
+
}
|
|
6295
|
+
|
|
6296
|
+
/**
|
|
6297
|
+
* Remember the responder's strand-network addresses for a strand this node just
|
|
6298
|
+
* formed, so the strand's discovery seed has something to work with when the app
|
|
6299
|
+
* launches it (see {@link crossPartyStrandAddrs}).
|
|
6300
|
+
*
|
|
6301
|
+
* Scoped strictly to `strandId`: these addresses reach ONE strand node of ONE other
|
|
6302
|
+
* party and must never seed another strand's mesh or the control peerStore. An empty
|
|
6303
|
+
* list is recorded as no entry at all rather than an empty one, so a later formation
|
|
6304
|
+
* against the same strand that DOES disclose addresses is not shadowed.
|
|
6305
|
+
*/
|
|
6306
|
+
private recordCrossPartyStrandAddrs(strandId: string, strandAddrs: readonly string[]): void {
|
|
6307
|
+
if (strandAddrs.length === 0) {
|
|
6308
|
+
log('formStrand: responder disclosed no strand addrs for %s — cross-party seed stays empty', strandId);
|
|
6309
|
+
return;
|
|
6310
|
+
}
|
|
6311
|
+
// Capped at the same `MAX_STRAND_ADDRS` one arriving list is, because this
|
|
6312
|
+
// ACCUMULATES: re-forming against the same strand (the documented recovery for a
|
|
6313
|
+
// responder whose relay reservation rotated) unions a fresh list onto the old one,
|
|
6314
|
+
// and nothing here can tell a dead entry from a live one. Without the cap, the seed
|
|
6315
|
+
// — and the peerStore rows every refresh pass re-merges from it — would grow by up
|
|
6316
|
+
// to 16 entries per redemption for the node's lifetime. Oldest entries fall off the
|
|
6317
|
+
// end, which is the right end to lose: `unionAddrs` keeps the freshest list first.
|
|
6318
|
+
const merged = unionAddrs(strandAddrs, this.crossPartyStrandAddrs.get(strandId) ?? [])
|
|
6319
|
+
.slice(0, MAX_STRAND_ADDRS);
|
|
6320
|
+
this.crossPartyStrandAddrs.set(strandId, merged);
|
|
6321
|
+
// A strand already running when this lands (a re-formation) would otherwise wait out
|
|
6322
|
+
// the rest of its ~10-minute refresh throttle before the new addresses reached its
|
|
6323
|
+
// address book. Clearing the stamp makes the next 15 s reconcile tick merge them,
|
|
6324
|
+
// which matters precisely because re-forming is the recovery path for a dead entry.
|
|
6325
|
+
this.strandPeerAddrRefreshAt.delete(strandId);
|
|
6326
|
+
log('formStrand: recorded %d cross-party strand addr(s) for %s', merged.length, strandId);
|
|
5132
6327
|
}
|
|
5133
6328
|
|
|
5134
6329
|
/**
|