@serfab/cadre-core 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/cadre-node.d.ts +70 -34
- package/dist/cadre-node.js +197 -60
- package/dist/cadre-node.js.map +1 -1
- package/dist/control-database.d.ts +24 -10
- package/dist/control-database.js +48 -16
- package/dist/control-database.js.map +1 -1
- package/dist/control-formation-recorder.d.ts +4 -4
- package/dist/control-formation-recorder.js +6 -6
- package/dist/control-formation-recorder.js.map +1 -1
- package/dist/control-read-retry.d.ts +12 -0
- package/dist/control-read-retry.js +12 -0
- package/dist/control-read-retry.js.map +1 -1
- package/dist/control-write-retry.d.ts +5 -4
- package/dist/control-write-retry.js +24 -7
- package/dist/control-write-retry.js.map +1 -1
- package/dist/index.d.ts +5 -4
- package/dist/index.js +10 -5
- package/dist/index.js.map +1 -1
- package/dist/link-budget.d.ts +182 -0
- package/dist/link-budget.js +195 -0
- package/dist/link-budget.js.map +1 -0
- package/dist/membership-connection-gater.d.ts +18 -5
- package/dist/membership-connection-gater.js +50 -10
- package/dist/membership-connection-gater.js.map +1 -1
- package/dist/peer-dial.d.ts +36 -17
- package/dist/peer-dial.js +37 -17
- package/dist/peer-dial.js.map +1 -1
- package/dist/peer-join-backfill.d.ts +58 -4
- package/dist/peer-join-backfill.js +165 -20
- package/dist/peer-join-backfill.js.map +1 -1
- package/dist/relay-reservation.d.ts +59 -11
- package/dist/relay-reservation.js +169 -43
- package/dist/relay-reservation.js.map +1 -1
- package/dist/seed-bootstrap.js +2 -1
- package/dist/seed-bootstrap.js.map +1 -1
- package/dist/storage-scope.d.ts +50 -12
- package/dist/storage-scope.js +85 -12
- package/dist/storage-scope.js.map +1 -1
- package/dist/strand-first-sync-gate.d.ts +43 -5
- package/dist/strand-first-sync-gate.js +55 -7
- package/dist/strand-first-sync-gate.js.map +1 -1
- package/dist/strand-formation-manager.js +2 -1
- package/dist/strand-formation-manager.js.map +1 -1
- package/dist/strand-id.d.ts +35 -0
- package/dist/strand-id.js +46 -0
- package/dist/strand-id.js.map +1 -0
- package/dist/strand-instance-manager.d.ts +27 -2
- package/dist/strand-instance-manager.js +75 -15
- package/dist/strand-instance-manager.js.map +1 -1
- package/dist/strand-membership-reconciler.d.ts +200 -45
- package/dist/strand-membership-reconciler.js +302 -86
- package/dist/strand-membership-reconciler.js.map +1 -1
- package/dist/strand-membership-writer.d.ts +3 -2
- package/dist/strand-membership-writer.js +67 -90
- package/dist/strand-membership-writer.js.map +1 -1
- package/dist/strand-network-config.d.ts +3 -3
- package/dist/strand-network-config.js +3 -3
- package/dist/strand-solicitation.js +2 -1
- package/dist/strand-solicitation.js.map +1 -1
- package/dist/strand-watcher.d.ts +45 -0
- package/dist/strand-watcher.js +74 -13
- package/dist/strand-watcher.js.map +1 -1
- package/dist/types.d.ts +190 -24
- package/dist/types.js +73 -4
- package/dist/types.js.map +1 -1
- package/package.json +8 -8
- package/src/cadre-node.ts +7445 -7302
- package/src/control-database.ts +48 -16
- package/src/control-formation-recorder.ts +6 -6
- package/src/control-read-retry.ts +214 -202
- package/src/control-write-retry.ts +24 -7
- package/src/index.ts +29 -2
- package/src/link-budget.ts +217 -0
- package/src/membership-connection-gater.ts +56 -11
- package/src/peer-dial.ts +39 -17
- package/src/peer-join-backfill.ts +192 -23
- package/src/relay-reservation.ts +1119 -951
- package/src/seed-bootstrap.ts +2 -1
- package/src/storage-scope.ts +163 -78
- package/src/strand-first-sync-gate.ts +55 -7
- package/src/strand-formation-manager.ts +647 -646
- package/src/strand-id.ts +48 -0
- package/src/strand-instance-manager.ts +1516 -1440
- package/src/strand-membership-reconciler.ts +334 -92
- package/src/strand-membership-writer.ts +1962 -1987
- package/src/strand-network-config.ts +250 -250
- package/src/strand-solicitation.ts +500 -499
- package/src/strand-watcher.ts +388 -325
- package/src/types.ts +192 -23
package/dist/cadre-node.js
CHANGED
|
@@ -4,11 +4,11 @@ import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
|
|
|
4
4
|
import { createLibp2pNode } from '@optimystic/db-p2p';
|
|
5
5
|
import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
|
|
6
6
|
import { multiaddr } from '@multiformats/multiaddr';
|
|
7
|
-
import { controlClusterPolicy, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
|
|
7
|
+
import { controlClusterPolicy, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS, DEFAULT_CONNECTION_MONITOR } from './types.js';
|
|
8
8
|
import { sign } from '@optimystic/quereus-plugin-crypto';
|
|
9
9
|
import { ed25519KeyPairFromLibp2p, ed25519PublicKeyFromPrivate, requireEd25519PublicKeyB64 } from './ed25519-key.js';
|
|
10
10
|
import { strandTransportKey } from './strand-transport-key.js';
|
|
11
|
-
import { controlStorageScope } from './storage-scope.js';
|
|
11
|
+
import { assertScopeKeyCharset, assertStrandScopeKey, controlStorageScope, InvalidStrandIdError, isValidStrandScopeKey } from './storage-scope.js';
|
|
12
12
|
import { generateStrandMemberKey, strandMemberKeyPair } from './strand-member-key.js';
|
|
13
13
|
import { assertNotPreSplitStrand, issueInvite, PreSplitStrandIdentityError } from './strand-membership-writer.js';
|
|
14
14
|
import { MEMBERSHIP_INVITE_TTL_MS } from './strand-formation-manager.js';
|
|
@@ -27,7 +27,8 @@ import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manag
|
|
|
27
27
|
import { PeerJoinBackfill } from './peer-join-backfill.js';
|
|
28
28
|
import { deriveCohortMembers } from './strand-cohort.js';
|
|
29
29
|
import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
|
|
30
|
-
import { dialPeerAddrs,
|
|
30
|
+
import { dialPeerAddrs, CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS } from './peer-dial.js';
|
|
31
|
+
import { peerJoinPushBudget, relayReservationBudgetMs, relayedDialBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
|
|
31
32
|
import { EnrollmentService } from './enrollment.js';
|
|
32
33
|
import { HibernationManager } from './hibernation-manager.js';
|
|
33
34
|
import { ControlDatabase, isStrandIdConflict } from './control-database.js';
|
|
@@ -394,8 +395,10 @@ export class CadreNode {
|
|
|
394
395
|
* dead entry behind; recovery today is a fresh invitation.
|
|
395
396
|
*
|
|
396
397
|
* NOTE: entries are kept for the node's lifetime, including across a
|
|
397
|
-
* {@link stopStrand} — a stopped strand may be
|
|
398
|
-
*
|
|
398
|
+
* {@link stopStrand} — a stopped strand may be claimed again with {@link addStrand}
|
|
399
|
+
* (the watcher itself will not re-offer it: the stop suppresses the id for the rest of
|
|
400
|
+
* the session) and must still get its seed. Each strand's list is capped at
|
|
401
|
+
* `MAX_STRAND_ADDRS`, but the
|
|
399
402
|
* number of KEYS is bounded only by the strands this node has ever formed as an
|
|
400
403
|
* initiator (not by time); if a node ever forms strands at scale, evict on
|
|
401
404
|
* `unpublishStrand` or cap the map.
|
|
@@ -421,8 +424,10 @@ export class CadreNode {
|
|
|
421
424
|
* (`strand-membership-reconciler.ts`, wired via `StartStrandConfig.pendingMembershipInvite`
|
|
422
425
|
* in {@link launchStrand}): it deletes the entry once the invitation is redeemed, burned
|
|
423
426
|
* against an already-seated member, or found dead (expired / cancelled / consumed
|
|
424
|
-
* elsewhere / the strand sealed)
|
|
425
|
-
*
|
|
427
|
+
* elsewhere / the strand sealed) — naming the invitation it settled, so a re-formation
|
|
428
|
+
* that replaced the entry meanwhile keeps its fresh one
|
|
429
|
+
* ({@link unstageMembershipInvite}). Until the strand is actually launched here the entry
|
|
430
|
+
* just waits; once launched, staging notifies the manager, which re-arms a finished loop.
|
|
426
431
|
*
|
|
427
432
|
* NOTE: entries live for the node's lifetime (one small pair per formed closed
|
|
428
433
|
* strand), the same unbounded-keys tripwire {@link crossPartyStrandAddrs} documents —
|
|
@@ -760,6 +765,11 @@ export class CadreNode {
|
|
|
760
765
|
// throws on a malformed `relayAddrs` entry, and that failure belongs on the same
|
|
761
766
|
// logged-and-cleaned-up path as every other one in start().
|
|
762
767
|
this.warnIfAnnounceAddrsDiscardRelay();
|
|
768
|
+
// Refuse a bad `linkRoundTripMs` HERE rather than wherever a budget is first derived
|
|
769
|
+
// from it: every consumer of it is conditional (no control storage, no relays, no dial
|
|
770
|
+
// yet), so a node with a zero or NaN declaration would otherwise boot and fail later
|
|
771
|
+
// inside a best-effort path that logs and carries on. `link-budget.ts`.
|
|
772
|
+
resolveLinkRoundTripMs(this.config.network?.linkRoundTripMs);
|
|
763
773
|
// Install the WebRTC TURN-relay tracker BEFORE any libp2p bring-up, so it
|
|
764
774
|
// wraps globalThis.RTCPeerConnection before the control node can create one.
|
|
765
775
|
// Inert on Node.js (no RTCPeerConnection); disposed in cleanup().
|
|
@@ -1184,6 +1194,12 @@ export class CadreNode {
|
|
|
1184
1194
|
* derivation from it (`db-p2p` namespaces all of the node's protocol ids as
|
|
1185
1195
|
* `/optimystic/<networkName>/...`), so the node options and the block-transfer
|
|
1186
1196
|
* protocol prefix the control backfill dials can never drift apart.
|
|
1197
|
+
*
|
|
1198
|
+
* NOTE: the party id goes in UNENCODED here, unlike in `controlStorageScope`. Safe
|
|
1199
|
+
* today because a party id is locally configured rather than replicated in, and both
|
|
1200
|
+
* ends of a connection derive this string identically — an odd party id yields an odd
|
|
1201
|
+
* but consistent protocol id, not a mismatch or an escaped name. If a party id ever
|
|
1202
|
+
* arrives from the network, encode it here as the storage scope key already does.
|
|
1187
1203
|
*/
|
|
1188
1204
|
controlNetworkName() {
|
|
1189
1205
|
return `control-${this.config.controlNetwork.partyId}`;
|
|
@@ -1259,6 +1275,11 @@ export class CadreNode {
|
|
|
1259
1275
|
// before looking anywhere else; if catch-up pushes start showing up as
|
|
1260
1276
|
// connection-churn noise, raise it.
|
|
1261
1277
|
debounceMs: 250,
|
|
1278
|
+
// Dial and response deadlines counted in link round trips, not fixed milliseconds — a
|
|
1279
|
+
// relayed dial that cannot finish inside the budget fails identically on every retry.
|
|
1280
|
+
// See `link-budget.ts`. Spread BEFORE the host's own config so an explicit
|
|
1281
|
+
// `controlBackfill.dialTimeoutMs` still wins.
|
|
1282
|
+
...peerJoinPushBudget(this.config.network?.linkRoundTripMs),
|
|
1262
1283
|
...this.config.controlBackfill
|
|
1263
1284
|
});
|
|
1264
1285
|
this.controlBackfill.start();
|
|
@@ -1293,6 +1314,11 @@ export class CadreNode {
|
|
|
1293
1314
|
// party's records, so two parties on one device must not land in one store.
|
|
1294
1315
|
// Same key as the cache label, which also makes the shared pool's `stats()` readable.
|
|
1295
1316
|
const scope = controlStorageScope(this.config.controlNetwork.partyId);
|
|
1317
|
+
// The second seam that hands a scope key to an embedder's provider; the first
|
|
1318
|
+
// (`StrandInstanceManager.startStrand`) asserts the stricter strand rule. Holds by
|
|
1319
|
+
// construction today — base64url is inside the charset — so this guards a future
|
|
1320
|
+
// edit to `controlStorageScope`, not a reachable input.
|
|
1321
|
+
assertScopeKeyCharset(scope);
|
|
1296
1322
|
const resolved = typeof provider === 'function' ? provider(scope) : provider;
|
|
1297
1323
|
this.controlStorage = wrapStorageWithCache(resolved, scope);
|
|
1298
1324
|
return this.controlStorage;
|
|
@@ -1383,11 +1409,22 @@ export class CadreNode {
|
|
|
1383
1409
|
// itself, so the unknown case is provably byte-for-byte the old behaviour.
|
|
1384
1410
|
// The yardstick moves ALONE: `assumedClusterSize` stays pinned at 2, because a
|
|
1385
1411
|
// party of phones cannot promise three quarters of its machines are awake.
|
|
1386
|
-
|
|
1412
|
+
// With neither declared this is still the frozen constant by identity, which is the
|
|
1413
|
+
// production path. A degenerate deadline is Optimystic's to refuse, in
|
|
1414
|
+
// createControlNode below.
|
|
1415
|
+
clusterPolicy: controlClusterPolicy({
|
|
1416
|
+
enrolledMachines: this.declaredEnrolledMachines,
|
|
1417
|
+
cohortQueryTimeoutMs: network?.cohortQueryTimeoutMs
|
|
1418
|
+
}),
|
|
1387
1419
|
arachnode: { enableRingZulu: profile === 'storage' },
|
|
1388
1420
|
...(identityKey && { privateKey: identityKey }),
|
|
1389
1421
|
...(network?.transports && { transports: network.transports }),
|
|
1390
1422
|
...(network?.noiseCrypto && { noiseCrypto: network.noiseCrypto }),
|
|
1423
|
+
// Always present, unlike the conditional spreads around it: an absent
|
|
1424
|
+
// `connectionMonitor` would leave libp2p's 5s ping deadline, which drops a peer
|
|
1425
|
+
// whose event loop is saturated by pure-JS Noise crypto. A configured value
|
|
1426
|
+
// replaces the default whole (see DEFAULT_CONNECTION_MONITOR).
|
|
1427
|
+
connectionMonitor: network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
|
|
1391
1428
|
// `{ wsPort }` when a listen entry names WebSocket, otherwise `{}` — and always
|
|
1392
1429
|
// `{}` when `network.transports` is set, since the embedder owns transport policy
|
|
1393
1430
|
// then. Spread NEXT to `transports` because the two answer the same question.
|
|
@@ -2735,15 +2772,22 @@ export class CadreNode {
|
|
|
2735
2772
|
* {@link SeedBootstrapService} this node builds — `applySeed`'s owner dials and
|
|
2736
2773
|
* `dialInvite`.
|
|
2737
2774
|
*
|
|
2738
|
-
* See
|
|
2739
|
-
* bounded as a whole, and
|
|
2740
|
-
*
|
|
2775
|
+
* See `peer-dial.ts`'s `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` for why a peer's dial is
|
|
2776
|
+
* bounded as a whole, and `DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS` for why each
|
|
2777
|
+
* address is bounded as well.
|
|
2778
|
+
*
|
|
2779
|
+
* Both are DERIVED from `network.linkRoundTripMs` rather than fixed, because the slowest
|
|
2780
|
+
* address either has to cover is a relayed dial and that costs a fixed number of exchanges —
|
|
2781
|
+
* `link-budget.ts` has the counts. A host's explicit `controlCohort` values still win, so a
|
|
2782
|
+
* test that drives dead addresses on purpose keeps the duration it chose.
|
|
2741
2783
|
*/
|
|
2742
2784
|
controlDialBudget() {
|
|
2743
2785
|
const cohort = this.config.network?.controlCohort;
|
|
2786
|
+
const perAddressMs = cohort?.perAddressDialTimeoutMs
|
|
2787
|
+
?? relayedDialBudgetMs(this.config.network?.linkRoundTripMs);
|
|
2744
2788
|
return {
|
|
2745
|
-
perAddressMs
|
|
2746
|
-
totalMs: cohort?.dialTimeoutMs ??
|
|
2789
|
+
perAddressMs,
|
|
2790
|
+
totalMs: cohort?.dialTimeoutMs ?? CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS * perAddressMs,
|
|
2747
2791
|
};
|
|
2748
2792
|
}
|
|
2749
2793
|
/**
|
|
@@ -3670,6 +3714,23 @@ export class CadreNode {
|
|
|
3670
3714
|
}
|
|
3671
3715
|
async handleStrandAdded(strand) {
|
|
3672
3716
|
log('Handling strand added from control network: %s', strand.Id);
|
|
3717
|
+
// A replicated row carries whatever id the founding node wrote, and the id becomes a
|
|
3718
|
+
// storage scope key and a libp2p protocol prefix. Checked FIRST — above the
|
|
3719
|
+
// discovery branch below — so a malformed id is never offered to the hosting app as
|
|
3720
|
+
// a strand it could join; that app's `addStrand` could only fail on it.
|
|
3721
|
+
//
|
|
3722
|
+
// Suppressed rather than retried: the id is a property of the row, so every later
|
|
3723
|
+
// attempt fails identically, and without this the watcher's ladder would re-emit
|
|
3724
|
+
// `strand:error` every five minutes for the life of the process. The throw still
|
|
3725
|
+
// matters — the watcher's catch runs `forgetStrand`, dropping the id from
|
|
3726
|
+
// `knownStrands`, so its removed-strand loop never detaches a strand that never ran.
|
|
3727
|
+
if (!isValidStrandScopeKey(strand.Id)) {
|
|
3728
|
+
const error = new InvalidStrandIdError(strand.Id);
|
|
3729
|
+
log('Refusing strand %s: %s', strand.Id, error.message);
|
|
3730
|
+
this.emit('strand:error', { strandId: strand.Id, error });
|
|
3731
|
+
this.strandWatcher?.suppressStrand(strand.Id);
|
|
3732
|
+
throw error;
|
|
3733
|
+
}
|
|
3673
3734
|
// Check if we have sApp config for this strand
|
|
3674
3735
|
const sAppConfig = this.sAppConfigs.get(strand.Id);
|
|
3675
3736
|
if (!sAppConfig) {
|
|
@@ -3681,10 +3742,12 @@ export class CadreNode {
|
|
|
3681
3742
|
// already present) keeps auto-starting below, unchanged.
|
|
3682
3743
|
//
|
|
3683
3744
|
// Recorded BEFORE the emit so a handler that synchronously drains
|
|
3684
|
-
// `getDiscoveredStrands()` sees this strand too. The watcher
|
|
3685
|
-
//
|
|
3686
|
-
//
|
|
3687
|
-
//
|
|
3745
|
+
// `getDiscoveredStrands()` sees this strand too. The watcher does not offer the
|
|
3746
|
+
// same strand twice — its `knownStrands` retains the id, and the one thing that
|
|
3747
|
+
// un-retains it (a failed `addStrand`, via `StrandWatcher.forgetStrand`) leaves the
|
|
3748
|
+
// sApp config registered, so the retry takes the auto-launch branch below rather
|
|
3749
|
+
// than this one. So this map — not the event — is what a late subscriber reads.
|
|
3750
|
+
// See the `strand:discovered` doc in types.ts.
|
|
3688
3751
|
this.discoveredStrands.set(strand.Id, strand);
|
|
3689
3752
|
this.emit('strand:discovered', { strandId: strand.Id, strand });
|
|
3690
3753
|
return;
|
|
@@ -4084,23 +4147,26 @@ export class CadreNode {
|
|
|
4084
4147
|
*
|
|
4085
4148
|
* A rejected call leaves nothing running but DOES leave the sApp config
|
|
4086
4149
|
* registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
|
|
4087
|
-
* automatic relaunch need it.
|
|
4088
|
-
*
|
|
4089
|
-
*
|
|
4090
|
-
*
|
|
4091
|
-
*
|
|
4092
|
-
*
|
|
4093
|
-
*
|
|
4094
|
-
*
|
|
4095
|
-
*
|
|
4096
|
-
*
|
|
4097
|
-
*
|
|
4150
|
+
* automatic relaunch need it. A failed launch here hands the strand back to that
|
|
4151
|
+
* relaunch — `StrandWatcher.forgetStrand` drops the id from the watcher's
|
|
4152
|
+
* `knownStrands` and records the backoff, so a later poll re-attempts it on the same
|
|
4153
|
+
* `pollInterval * 2^(failures-1)` ladder a watcher-driven failure gets. Because the
|
|
4154
|
+
* config stays registered, the retry takes `handleStrandAdded`'s auto-launch branch:
|
|
4155
|
+
* what the app sees is `strand:error` per failed retry and `strand:started` when one
|
|
4156
|
+
* succeeds, never a second `strand:discovered`. {@link detachStrand} (reached via
|
|
4157
|
+
* {@link stopStrand}) is what abandons a strand for good, and its stop suppresses the
|
|
4158
|
+
* strand in the watcher so this retry cannot resurrect it — a call to THIS method lifts
|
|
4159
|
+
* that suppression again, since an explicit claim is a deliberate reversal of the
|
|
4160
|
+
* deliberate stop.
|
|
4098
4161
|
*/
|
|
4099
4162
|
async addStrand(config) {
|
|
4100
4163
|
if (!this._running) {
|
|
4101
4164
|
throw new Error('CadreNode not running');
|
|
4102
4165
|
}
|
|
4103
4166
|
const { strandRow, sAppConfig, founder, partyMemberPrivateKey } = config;
|
|
4167
|
+
// Before anything is recorded: an unusable id must not leave a registered sApp
|
|
4168
|
+
// config behind, and must not lift a suppression this node set deliberately.
|
|
4169
|
+
assertStrandScopeKey(strandRow.Id);
|
|
4104
4170
|
if (partyMemberPrivateKey !== undefined && strandRow.Type !== 'c') {
|
|
4105
4171
|
// Only a closed strand seats a Member/Manager, so an open launch would drop this
|
|
4106
4172
|
// key silently. Refuse instead: the caller has confused the party identity key
|
|
@@ -4113,13 +4179,41 @@ export class CadreNode {
|
|
|
4113
4179
|
// unclaimed backlog — a later `getDiscoveredStrands()` drain must not re-offer it.
|
|
4114
4180
|
this.sAppConfigs.set(strandRow.Id, sAppConfig);
|
|
4115
4181
|
this.discoveredStrands.delete(strandRow.Id);
|
|
4182
|
+
// A claim also overrides any earlier `stopStrand` of this id: the watcher must track
|
|
4183
|
+
// the strand again, or a party-wide removal would never stop it here.
|
|
4184
|
+
this.strandWatcher?.unsuppressStrand(strandRow.Id);
|
|
4116
4185
|
log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)', strandRow.Id, sAppConfig.id, founder ?? 'derived');
|
|
4117
4186
|
// An unset `founder` is DERIVED from the row inside launchStrand (this node founds
|
|
4118
4187
|
// iff the row's FounderOwnerKey is its own owner key); an explicit flag wins — the
|
|
4119
4188
|
// formation/responder flows pass one deliberately, since their consent-seated rows
|
|
4120
4189
|
// carry a null column. See StrandConfig.founder. Same rule for the explicit
|
|
4121
4190
|
// partyMemberPrivateKey: it wins over the StrandPartyKey control-row read.
|
|
4122
|
-
|
|
4191
|
+
// Only the launch is wrapped, NOT the first-sync wait below: a wait that times out
|
|
4192
|
+
// rejects with the retryable StrandAwaitingFirstSyncError and deliberately leaves the
|
|
4193
|
+
// instance running, so there is nothing to re-offer and forgetting it would only have
|
|
4194
|
+
// the watcher re-enter a launch for a strand that is already up.
|
|
4195
|
+
let instance;
|
|
4196
|
+
try {
|
|
4197
|
+
instance = await this.launchStrand(strandRow, sAppConfig, founder, partyMemberPrivateKey);
|
|
4198
|
+
}
|
|
4199
|
+
catch (error) {
|
|
4200
|
+
// Hand the strand back to the watcher's retry ladder. Harmless on the founder path
|
|
4201
|
+
// (`foundStrand` → `publishStrand` + this call), where the watcher may never have
|
|
4202
|
+
// offered the strand: the `knownStrands` delete is a no-op and the recorded backoff
|
|
4203
|
+
// only delays the watcher's own first attempt by one poll interval, which is what
|
|
4204
|
+
// should happen right after an attempt that just failed.
|
|
4205
|
+
//
|
|
4206
|
+
// NOTE: the retry relaunches from the CONTROL row plus the registered config, not
|
|
4207
|
+
// from the row and arguments passed here — so a caller that enriched either (a
|
|
4208
|
+
// synthetic `MemberPrivateKey`, an explicit `founder` or `partyMemberPrivateKey`)
|
|
4209
|
+
// is retried with less than it asked for. Inert today: founder-ness re-derives from
|
|
4210
|
+
// `FounderOwnerKey`, the party key re-reads from `StrandPartyKey`, and the row's
|
|
4211
|
+
// shared `MemberPrivateKey` is read only by the founder bootstrap. If a joiner
|
|
4212
|
+
// launch ever starts depending on that key, gate this hand-back on the passed row
|
|
4213
|
+
// matching the control one.
|
|
4214
|
+
this.strandWatcher?.forgetStrand(strandRow.Id);
|
|
4215
|
+
throw error;
|
|
4216
|
+
}
|
|
4123
4217
|
// The first-sync write gate: a JOINING machine's database is withheld until it has
|
|
4124
4218
|
// received the strand's Header from another member, because a write before that
|
|
4125
4219
|
// forks every table it touches (see strand-first-sync-gate.ts). Waiting here is what
|
|
@@ -4208,7 +4302,8 @@ export class CadreNode {
|
|
|
4208
4302
|
* @returns The live `Strand` row — the one just inserted, or the matching one already
|
|
4209
4303
|
* there. A closed strand's caller should carry THIS row's `MemberPrivateKey` forward:
|
|
4210
4304
|
* on a repeat it is the stored key, not the argument.
|
|
4211
|
-
* @throws if the node is not started, exposes no owner signing key, the id is blank
|
|
4305
|
+
* @throws if the node is not started, exposes no owner signing key, the id is blank or
|
|
4306
|
+
* unusable as a storage scope key (`InvalidStrandIdError` — see `storage-scope.ts`), a
|
|
4212
4307
|
* row with the same id holds different content, or the control DB rejects the
|
|
4213
4308
|
* (unauthorized) insert.
|
|
4214
4309
|
*/
|
|
@@ -4217,6 +4312,11 @@ export class CadreNode {
|
|
|
4217
4312
|
// Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
|
|
4218
4313
|
// trims too, and an untrimmed row would be unreachable by the same string.
|
|
4219
4314
|
const trimmed = requireNonBlank(strandId, 'strand id');
|
|
4315
|
+
// Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
|
|
4316
|
+
// runs after this row is written, and this row replicates: an id no node can turn into
|
|
4317
|
+
// a storage scope key would otherwise reach the whole party's control database and be
|
|
4318
|
+
// declined once per member. Asserted after the trim so the id checked is the id stored.
|
|
4319
|
+
assertStrandScopeKey(trimmed);
|
|
4220
4320
|
// FounderOwnerKey records THIS machine as the row's publisher (the schema pins it to
|
|
4221
4321
|
// the signing owner), which is what later lets a relaunch derive founder-ness.
|
|
4222
4322
|
const desired = {
|
|
@@ -4457,13 +4557,20 @@ export class CadreNode {
|
|
|
4457
4557
|
// one never cleared).
|
|
4458
4558
|
await this.strandManager.clearOwnMemberPeerBinding(trimmed);
|
|
4459
4559
|
// Converge locally now rather than waiting up to a poll interval. The watcher fires
|
|
4460
|
-
// onStrandRemoved for a strand it tracked; the explicit
|
|
4461
|
-
// strandFilter never admitted this strand (the watcher never knew it, so it will
|
|
4462
|
-
// fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
4463
|
-
// cannot double-stop into an error.
|
|
4560
|
+
// onStrandRemoved for a strand it tracked; the explicit teardown below covers a node
|
|
4561
|
+
// whose strandFilter never admitted this strand (the watcher never knew it, so it will
|
|
4562
|
+
// never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
|
|
4563
|
+
// paths cannot double-stop into an error.
|
|
4564
|
+
//
|
|
4565
|
+
// detachStrand, NOT stopStrand: this is a party-wide removal, not a local abandonment
|
|
4566
|
+
// of a strand that still exists, so it must not SUPPRESS the id in the watcher. The
|
|
4567
|
+
// row is already gone — there is nothing to suppress — and an id re-published before
|
|
4568
|
+
// the next poll's suppression cleanup would then never be offered again this session.
|
|
4569
|
+
// (Same reasoning as handleStrandRemoved, which also avoids stopStrand's `_running`
|
|
4570
|
+
// guard; here the guard is already satisfied by requireOwnerSigningKey above.)
|
|
4464
4571
|
await this.strandWatcher?.forcePoll();
|
|
4465
4572
|
if (this.strandManager.getInstance(trimmed)) {
|
|
4466
|
-
await this.
|
|
4573
|
+
await this.detachStrand(trimmed);
|
|
4467
4574
|
}
|
|
4468
4575
|
log('Unpublished strand %s from control DB under owner %s', trimmed, signingKey.publicKeyB64);
|
|
4469
4576
|
}
|
|
@@ -4825,6 +4932,7 @@ export class CadreNode {
|
|
|
4825
4932
|
revocationEnforcement: this.config.strandRevocationEnforcement,
|
|
4826
4933
|
membershipReconciliation: this.config.strandMembershipReconciliation,
|
|
4827
4934
|
onSelfRevoked: (revokedStrandId) => this.emit('strand:revoked', { strandId: revokedStrandId }),
|
|
4935
|
+
onRejoinBlocked: (blockedStrandId) => this.emit('strand:rejoin-blocked', { strandId: blockedStrandId }),
|
|
4828
4936
|
// The joiner's first-sync write gate (strand-first-sync-gate.ts): a launch that
|
|
4829
4937
|
// comes up `'syncing'` announces the moment its database is published.
|
|
4830
4938
|
firstSync: this.config.strandFirstSync,
|
|
@@ -4838,7 +4946,7 @@ export class CadreNode {
|
|
|
4838
4946
|
// cleared once spent, burned, or dead — see pendingMembershipInvites.
|
|
4839
4947
|
pendingMembershipInvite: {
|
|
4840
4948
|
get: () => this.pendingMembershipInvites.get(strand.Id),
|
|
4841
|
-
clear: () =>
|
|
4949
|
+
clear: (settled) => this.unstageMembershipInvite(strand.Id, settled)
|
|
4842
4950
|
},
|
|
4843
4951
|
// The RESOLVED flag, never the raw argument — see the doc comment above.
|
|
4844
4952
|
founder: resolvedFounder,
|
|
@@ -5066,8 +5174,10 @@ export class CadreNode {
|
|
|
5066
5174
|
* caller runs the drive regardless).
|
|
5067
5175
|
*
|
|
5068
5176
|
* NOTE: against a relay that is DOWN this hook costs up to two strand-addr
|
|
5069
|
-
* timeouts (10 s each: dial by peer id, then by addr) before the
|
|
5070
|
-
* starts, so one failed re-drive
|
|
5177
|
+
* timeouts (10 s each: dial by peer id, then by addr) before the reservation drive even
|
|
5178
|
+
* starts, so one failed re-drive holds the supervisor `driving` for those 20 s plus the
|
|
5179
|
+
* drive's own deadline — 28 s at the default declared link round trip, and longer on a host
|
|
5180
|
+
* that declared a slower one (`link-budget.ts`).
|
|
5071
5181
|
* Bounded and harmless while the relay is unreachable anyway; if recovery
|
|
5072
5182
|
* latency after a relay comes back ever matters, skip the announce when the
|
|
5073
5183
|
* control node holds no connection to the relay (the drive's own dial fails
|
|
@@ -5313,16 +5423,20 @@ export class CadreNode {
|
|
|
5313
5423
|
* Stop a strand on THIS node only: untrack it from hibernation, drop its sApp config,
|
|
5314
5424
|
* stop the local instance, and emit `strand:stopped`. The shared `Strand` row is left
|
|
5315
5425
|
* intact, so on the next node RESTART the strand is rediscovered and surfaces as
|
|
5316
|
-
* `strand:discovered` again —
|
|
5317
|
-
*
|
|
5318
|
-
*
|
|
5319
|
-
*
|
|
5320
|
-
* {@link unpublishStrand}.
|
|
5426
|
+
* `strand:discovered` again — but never again in THIS session: the stop suppresses the
|
|
5427
|
+
* id in the watcher (`StrandWatcher.suppressStrand`) and drops it from
|
|
5428
|
+
* {@link getDiscoveredStrands}, so neither a later poll nor a drain can undo a
|
|
5429
|
+
* deliberate stop. Only an explicit {@link addStrand} does, which is the caller
|
|
5430
|
+
* reversing its own decision. Party-wide removal is {@link unpublishStrand}.
|
|
5321
5431
|
*/
|
|
5322
5432
|
async stopStrand(strandId) {
|
|
5323
5433
|
if (!this._running) {
|
|
5324
5434
|
throw new Error('CadreNode not running');
|
|
5325
5435
|
}
|
|
5436
|
+
// Suppressed HERE and not in detachStrand: the other caller of that method is
|
|
5437
|
+
// `handleStrandRemoved`, which arrives precisely because the control row is gone, and
|
|
5438
|
+
// a row that later reappears is a fresh strand that should be offered again.
|
|
5439
|
+
this.strandWatcher?.suppressStrand(strandId);
|
|
5326
5440
|
await this.detachStrand(strandId);
|
|
5327
5441
|
}
|
|
5328
5442
|
/**
|
|
@@ -5343,6 +5457,10 @@ export class CadreNode {
|
|
|
5343
5457
|
* readings are the intended one: `handleStrandRemoved` arrives because the control row
|
|
5344
5458
|
* is gone, and an explicit {@link stopStrand} is a deliberate abandonment — neither
|
|
5345
5459
|
* strand may be re-offered to a later `getDiscoveredStrands()` drain.
|
|
5460
|
+
*
|
|
5461
|
+
* The watcher-level suppression that makes a stop permanent is deliberately NOT here,
|
|
5462
|
+
* only in {@link stopStrand}: the two callers diverge on it. A vanished control row
|
|
5463
|
+
* that reappears is a fresh strand and must be offered again.
|
|
5346
5464
|
*/
|
|
5347
5465
|
async detachStrand(strandId) {
|
|
5348
5466
|
this.hibernationManager.untrackStrand(strandId);
|
|
@@ -5557,13 +5675,15 @@ export class CadreNode {
|
|
|
5557
5675
|
* local), and only then does it reach out. Everything above in {@link start}
|
|
5558
5676
|
* has completed by the time this runs.
|
|
5559
5677
|
*
|
|
5560
|
-
* BUDGET: the supervisor's first attempt
|
|
5561
|
-
*
|
|
5562
|
-
*
|
|
5563
|
-
*
|
|
5564
|
-
*
|
|
5565
|
-
* on links that were merely slow.
|
|
5566
|
-
*
|
|
5678
|
+
* BUDGET: the supervisor's first attempt is what `start()` waits on — deliberately the drive's
|
|
5679
|
+
* ordinary deadline rather than a boot-specific one. That deadline is now COUNTED, four link
|
|
5680
|
+
* round trips at the declared `network.linkRoundTripMs` (8 s at its default, where it was a
|
|
5681
|
+
* fixed 10 s): a healthy dial-plus-reserve is sub-second even over a WAN, so this is slack for
|
|
5682
|
+
* a slow link, while going much longer would make a dead relay indistinguishable from a hung
|
|
5683
|
+
* start and much shorter would fail nodes on links that were merely slow. A host that declares
|
|
5684
|
+
* a slower link lengthens it without touching this path — `link-budget.ts`. The retries carry
|
|
5685
|
+
* on in the background after this resolves, exactly as they do for a {@link reserveRelays}
|
|
5686
|
+
* caller.
|
|
5567
5687
|
*
|
|
5568
5688
|
* `network.requireRelay === false` softens only the outcome below: a first
|
|
5569
5689
|
* attempt that lands no `/p2p-circuit` address is logged instead of thrown, and
|
|
@@ -5625,7 +5745,13 @@ export class CadreNode {
|
|
|
5625
5745
|
this.relayReserveError = 'control node unavailable';
|
|
5626
5746
|
return this.getRelayReservationState();
|
|
5627
5747
|
}
|
|
5628
|
-
|
|
5748
|
+
// The drive's deadline is counted in link round trips (`link-budget.ts`), so a host that
|
|
5749
|
+
// declared a slower link gets a longer drive without naming a number here. A caller's own
|
|
5750
|
+
// `timeoutMs` still wins.
|
|
5751
|
+
const supervisor = superviseRelayReservation(this.controlNode, addrs, {
|
|
5752
|
+
timeoutMs: relayReservationBudgetMs(this.config.network?.linkRoundTripMs),
|
|
5753
|
+
...opts
|
|
5754
|
+
});
|
|
5629
5755
|
this.relayReserveSupervisor = supervisor;
|
|
5630
5756
|
await supervisor.firstAttempt;
|
|
5631
5757
|
// Read through `this`, not `supervisor`: an overlapping call may have replaced
|
|
@@ -6469,7 +6595,11 @@ export class CadreNode {
|
|
|
6469
6595
|
* an intact control DB) and mints + persists a fresh one otherwise. The invitation
|
|
6470
6596
|
* will admit THIS key's public half as the `Strand.Member`.
|
|
6471
6597
|
* 2. Stage the invitation in {@link pendingMembershipInvites} for the strand
|
|
6472
|
-
* bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite
|
|
6598
|
+
* bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite`,
|
|
6599
|
+
* and tell the instance manager, which re-arms an already-finished membership
|
|
6600
|
+
* reconciler so a RE-formation against a launched strand is attempted at once — the
|
|
6601
|
+
* removed-party case, where the loop finished long before the removal. A strand not
|
|
6602
|
+
* launched here yet has no loop to re-arm, and its bring-up finds the entry.
|
|
6473
6603
|
*
|
|
6474
6604
|
* Throws — failing the whole {@link formStrand} — when the identity cannot be
|
|
6475
6605
|
* persisted: a joiner "joined" without a persistable identity could never become a
|
|
@@ -6488,16 +6618,23 @@ export class CadreNode {
|
|
|
6488
6618
|
'cannot become a member without it. Fix the underlying cause, then redeem a fresh ' +
|
|
6489
6619
|
'invitation (the delivered one dies with this error).', { cause: error });
|
|
6490
6620
|
}
|
|
6491
|
-
// NOTE: staging alone is not enough on a node whose reconciler for this strand has
|
|
6492
|
-
// already finished. `StrandMembershipReconciler` latches a terminal stopped state on
|
|
6493
|
-
// its `done` path and is rebuilt only by a strand relaunch, so a RE-formation against
|
|
6494
|
-
// an already-launched strand stages an invitation no loop will ever look at. Harmless
|
|
6495
|
-
// for a party that is still a member (the invitation would only be burned), and the
|
|
6496
|
-
// live defect for one that has been REMOVED — which is exactly when an app re-forms.
|
|
6497
|
-
// Tracked as `backlog/bug-removed-party-cannot-redeem-its-way-back`; pinned by
|
|
6498
|
-
// `strand-party-removal-via-formation-e2e.integration.ts` (test 2).
|
|
6499
6621
|
this.pendingMembershipInvites.set(strandId, invite);
|
|
6500
6622
|
log('formStrand: staged membership invitation for strand %s (party key persisted)', strandId);
|
|
6623
|
+
this.strandManager.notifyMembershipInviteStaged(strandId);
|
|
6624
|
+
}
|
|
6625
|
+
/**
|
|
6626
|
+
* The reconciler's half of the {@link pendingMembershipInvites} seam: drop `settled`
|
|
6627
|
+
* (spent, burned, or dead) only while it is still the staged entry. A re-formation
|
|
6628
|
+
* replaces the entry between a pass's read and its settle, and the fresh invitation it
|
|
6629
|
+
* staged is the one the re-armed loop is about to redeem — deleting it here would lose
|
|
6630
|
+
* it silently, the very outcome the re-arm exists to prevent.
|
|
6631
|
+
*/
|
|
6632
|
+
unstageMembershipInvite(strandId, settled) {
|
|
6633
|
+
if (this.pendingMembershipInvites.get(strandId)?.inviteKey !== settled.inviteKey) {
|
|
6634
|
+
log('strand %s: a fresh membership invitation replaced the one just settled — keeping it staged', strandId);
|
|
6635
|
+
return;
|
|
6636
|
+
}
|
|
6637
|
+
this.pendingMembershipInvites.delete(strandId);
|
|
6501
6638
|
}
|
|
6502
6639
|
/**
|
|
6503
6640
|
* The pending single-use membership invitation a closed-strand formation carried back
|