@serfab/cadre-core 1.3.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +5 -3
  2. package/dist/cadre-node.d.ts +70 -34
  3. package/dist/cadre-node.js +197 -60
  4. package/dist/cadre-node.js.map +1 -1
  5. package/dist/control-database.d.ts +24 -10
  6. package/dist/control-database.js +48 -16
  7. package/dist/control-database.js.map +1 -1
  8. package/dist/control-formation-recorder.d.ts +4 -4
  9. package/dist/control-formation-recorder.js +6 -6
  10. package/dist/control-formation-recorder.js.map +1 -1
  11. package/dist/control-read-retry.d.ts +12 -0
  12. package/dist/control-read-retry.js +12 -0
  13. package/dist/control-read-retry.js.map +1 -1
  14. package/dist/control-write-retry.d.ts +5 -4
  15. package/dist/control-write-retry.js +24 -7
  16. package/dist/control-write-retry.js.map +1 -1
  17. package/dist/index.d.ts +5 -4
  18. package/dist/index.js +10 -5
  19. package/dist/index.js.map +1 -1
  20. package/dist/link-budget.d.ts +182 -0
  21. package/dist/link-budget.js +195 -0
  22. package/dist/link-budget.js.map +1 -0
  23. package/dist/membership-connection-gater.d.ts +18 -5
  24. package/dist/membership-connection-gater.js +50 -10
  25. package/dist/membership-connection-gater.js.map +1 -1
  26. package/dist/peer-dial.d.ts +36 -17
  27. package/dist/peer-dial.js +37 -17
  28. package/dist/peer-dial.js.map +1 -1
  29. package/dist/peer-join-backfill.d.ts +58 -4
  30. package/dist/peer-join-backfill.js +165 -20
  31. package/dist/peer-join-backfill.js.map +1 -1
  32. package/dist/relay-reservation.d.ts +59 -11
  33. package/dist/relay-reservation.js +169 -43
  34. package/dist/relay-reservation.js.map +1 -1
  35. package/dist/seed-bootstrap.js +2 -1
  36. package/dist/seed-bootstrap.js.map +1 -1
  37. package/dist/storage-scope.d.ts +50 -12
  38. package/dist/storage-scope.js +85 -12
  39. package/dist/storage-scope.js.map +1 -1
  40. package/dist/strand-first-sync-gate.d.ts +43 -5
  41. package/dist/strand-first-sync-gate.js +55 -7
  42. package/dist/strand-first-sync-gate.js.map +1 -1
  43. package/dist/strand-formation-manager.js +2 -1
  44. package/dist/strand-formation-manager.js.map +1 -1
  45. package/dist/strand-id.d.ts +35 -0
  46. package/dist/strand-id.js +46 -0
  47. package/dist/strand-id.js.map +1 -0
  48. package/dist/strand-instance-manager.d.ts +27 -2
  49. package/dist/strand-instance-manager.js +75 -15
  50. package/dist/strand-instance-manager.js.map +1 -1
  51. package/dist/strand-membership-reconciler.d.ts +200 -45
  52. package/dist/strand-membership-reconciler.js +302 -86
  53. package/dist/strand-membership-reconciler.js.map +1 -1
  54. package/dist/strand-membership-writer.d.ts +3 -2
  55. package/dist/strand-membership-writer.js +67 -90
  56. package/dist/strand-membership-writer.js.map +1 -1
  57. package/dist/strand-network-config.d.ts +3 -3
  58. package/dist/strand-network-config.js +3 -3
  59. package/dist/strand-solicitation.js +2 -1
  60. package/dist/strand-solicitation.js.map +1 -1
  61. package/dist/strand-watcher.d.ts +45 -0
  62. package/dist/strand-watcher.js +74 -13
  63. package/dist/strand-watcher.js.map +1 -1
  64. package/dist/types.d.ts +190 -24
  65. package/dist/types.js +73 -4
  66. package/dist/types.js.map +1 -1
  67. package/package.json +8 -8
  68. package/src/cadre-node.ts +7445 -7302
  69. package/src/control-database.ts +48 -16
  70. package/src/control-formation-recorder.ts +6 -6
  71. package/src/control-read-retry.ts +214 -202
  72. package/src/control-write-retry.ts +24 -7
  73. package/src/index.ts +29 -2
  74. package/src/link-budget.ts +217 -0
  75. package/src/membership-connection-gater.ts +56 -11
  76. package/src/peer-dial.ts +39 -17
  77. package/src/peer-join-backfill.ts +192 -23
  78. package/src/relay-reservation.ts +1119 -951
  79. package/src/seed-bootstrap.ts +2 -1
  80. package/src/storage-scope.ts +163 -78
  81. package/src/strand-first-sync-gate.ts +55 -7
  82. package/src/strand-formation-manager.ts +647 -646
  83. package/src/strand-id.ts +48 -0
  84. package/src/strand-instance-manager.ts +1516 -1440
  85. package/src/strand-membership-reconciler.ts +334 -92
  86. package/src/strand-membership-writer.ts +1962 -1987
  87. package/src/strand-network-config.ts +250 -250
  88. package/src/strand-solicitation.ts +500 -499
  89. package/src/strand-watcher.ts +388 -325
  90. package/src/types.ts +192 -23
@@ -4,11 +4,11 @@ import { peerIdFromString, peerIdFromPrivateKey } from '@libp2p/peer-id';
4
4
  import { createLibp2pNode } from '@optimystic/db-p2p';
5
5
  import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
6
6
  import { multiaddr } from '@multiformats/multiaddr';
7
- import { controlClusterPolicy, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS } from './types.js';
7
+ import { controlClusterPolicy, CONTROL_REPLICATION_BREADTH, DEFAULT_CHECKIN_WINDOW_MS, DEFAULT_CONNECTION_MONITOR } from './types.js';
8
8
  import { sign } from '@optimystic/quereus-plugin-crypto';
9
9
  import { ed25519KeyPairFromLibp2p, ed25519PublicKeyFromPrivate, requireEd25519PublicKeyB64 } from './ed25519-key.js';
10
10
  import { strandTransportKey } from './strand-transport-key.js';
11
- import { controlStorageScope } from './storage-scope.js';
11
+ import { assertScopeKeyCharset, assertStrandScopeKey, controlStorageScope, InvalidStrandIdError, isValidStrandScopeKey } from './storage-scope.js';
12
12
  import { generateStrandMemberKey, strandMemberKeyPair } from './strand-member-key.js';
13
13
  import { assertNotPreSplitStrand, issueInvite, PreSplitStrandIdentityError } from './strand-membership-writer.js';
14
14
  import { MEMBERSHIP_INVITE_TTL_MS } from './strand-formation-manager.js';
@@ -27,7 +27,8 @@ import { StrandInstanceManager, liveStrandStatus } from './strand-instance-manag
27
27
  import { PeerJoinBackfill } from './peer-join-backfill.js';
28
28
  import { deriveCohortMembers } from './strand-cohort.js';
29
29
  import { selectControlCohortDials, DEFAULT_CONTROL_COHORT_RECONCILE_MS, DEFAULT_CONTROL_COHORT_TARGET_DEGREE } from './control-cohort.js';
30
- import { dialPeerAddrs, DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS, DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS } from './peer-dial.js';
30
+ import { dialPeerAddrs, CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS } from './peer-dial.js';
31
+ import { peerJoinPushBudget, relayReservationBudgetMs, relayedDialBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
31
32
  import { EnrollmentService } from './enrollment.js';
32
33
  import { HibernationManager } from './hibernation-manager.js';
33
34
  import { ControlDatabase, isStrandIdConflict } from './control-database.js';
@@ -394,8 +395,10 @@ export class CadreNode {
394
395
  * dead entry behind; recovery today is a fresh invitation.
395
396
  *
396
397
  * NOTE: entries are kept for the node's lifetime, including across a
397
- * {@link stopStrand} — a stopped strand may be rediscovered by the watcher and must
398
- * still get its seed. Each strand's list is capped at `MAX_STRAND_ADDRS`, but the
398
+ * {@link stopStrand} — a stopped strand may be claimed again with {@link addStrand}
399
+ * (the watcher itself will not re-offer it: the stop suppresses the id for the rest of
400
+ * the session) and must still get its seed. Each strand's list is capped at
401
+ * `MAX_STRAND_ADDRS`, but the
399
402
  * number of KEYS is bounded only by the strands this node has ever formed as an
400
403
  * initiator (not by time); if a node ever forms strands at scale, evict on
401
404
  * `unpublishStrand` or cap the map.
@@ -421,8 +424,10 @@ export class CadreNode {
421
424
  * (`strand-membership-reconciler.ts`, wired via `StartStrandConfig.pendingMembershipInvite`
422
425
  * in {@link launchStrand}): it deletes the entry once the invitation is redeemed, burned
423
426
  * against an already-seated member, or found dead (expired / cancelled / consumed
424
- * elsewhere / the strand sealed). Until the strand is actually launched here the entry
425
- * just waits.
427
+ * elsewhere / the strand sealed) — naming the invitation it settled, so a re-formation
428
+ * that replaced the entry meanwhile keeps its fresh one
429
+ * ({@link unstageMembershipInvite}). Until the strand is actually launched here the entry
430
+ * just waits; once launched, staging notifies the manager, which re-arms a finished loop.
426
431
  *
427
432
  * NOTE: entries live for the node's lifetime (one small pair per formed closed
428
433
  * strand), the same unbounded-keys tripwire {@link crossPartyStrandAddrs} documents —
@@ -760,6 +765,11 @@ export class CadreNode {
760
765
  // throws on a malformed `relayAddrs` entry, and that failure belongs on the same
761
766
  // logged-and-cleaned-up path as every other one in start().
762
767
  this.warnIfAnnounceAddrsDiscardRelay();
768
+ // Refuse a bad `linkRoundTripMs` HERE rather than wherever a budget is first derived
769
+ // from it: every consumer of it is conditional (no control storage, no relays, no dial
770
+ // yet), so a node with a zero or NaN declaration would otherwise boot and fail later
771
+ // inside a best-effort path that logs and carries on. `link-budget.ts`.
772
+ resolveLinkRoundTripMs(this.config.network?.linkRoundTripMs);
763
773
  // Install the WebRTC TURN-relay tracker BEFORE any libp2p bring-up, so it
764
774
  // wraps globalThis.RTCPeerConnection before the control node can create one.
765
775
  // Inert on Node.js (no RTCPeerConnection); disposed in cleanup().
@@ -1184,6 +1194,12 @@ export class CadreNode {
1184
1194
  * derivation from it (`db-p2p` namespaces all of the node's protocol ids as
1185
1195
  * `/optimystic/<networkName>/...`), so the node options and the block-transfer
1186
1196
  * protocol prefix the control backfill dials can never drift apart.
1197
+ *
1198
+ * NOTE: the party id goes in UNENCODED here, unlike in `controlStorageScope`. Safe
1199
+ * today because a party id is locally configured rather than replicated in, and both
1200
+ * ends of a connection derive this string identically — an odd party id yields an odd
1201
+ * but consistent protocol id, not a mismatch or an escaped name. If a party id ever
1202
+ * arrives from the network, encode it here as the storage scope key already does.
1187
1203
  */
1188
1204
  controlNetworkName() {
1189
1205
  return `control-${this.config.controlNetwork.partyId}`;
@@ -1259,6 +1275,11 @@ export class CadreNode {
1259
1275
  // before looking anywhere else; if catch-up pushes start showing up as
1260
1276
  // connection-churn noise, raise it.
1261
1277
  debounceMs: 250,
1278
+ // Dial and response deadlines counted in link round trips, not fixed milliseconds — a
1279
+ // relayed dial that cannot finish inside the budget fails identically on every retry.
1280
+ // See `link-budget.ts`. Spread BEFORE the host's own config so an explicit
1281
+ // `controlBackfill.dialTimeoutMs` still wins.
1282
+ ...peerJoinPushBudget(this.config.network?.linkRoundTripMs),
1262
1283
  ...this.config.controlBackfill
1263
1284
  });
1264
1285
  this.controlBackfill.start();
@@ -1293,6 +1314,11 @@ export class CadreNode {
1293
1314
  // party's records, so two parties on one device must not land in one store.
1294
1315
  // Same key as the cache label, which also makes the shared pool's `stats()` readable.
1295
1316
  const scope = controlStorageScope(this.config.controlNetwork.partyId);
1317
+ // The second seam that hands a scope key to an embedder's provider; the first
1318
+ // (`StrandInstanceManager.startStrand`) asserts the stricter strand rule. Holds by
1319
+ // construction today — base64url is inside the charset — so this guards a future
1320
+ // edit to `controlStorageScope`, not a reachable input.
1321
+ assertScopeKeyCharset(scope);
1296
1322
  const resolved = typeof provider === 'function' ? provider(scope) : provider;
1297
1323
  this.controlStorage = wrapStorageWithCache(resolved, scope);
1298
1324
  return this.controlStorage;
@@ -1383,11 +1409,22 @@ export class CadreNode {
1383
1409
  // itself, so the unknown case is provably byte-for-byte the old behaviour.
1384
1410
  // The yardstick moves ALONE: `assumedClusterSize` stays pinned at 2, because a
1385
1411
  // party of phones cannot promise three quarters of its machines are awake.
1386
- clusterPolicy: controlClusterPolicy(this.declaredEnrolledMachines),
1412
+ // With neither declared this is still the frozen constant by identity, which is the
1413
+ // production path. A degenerate deadline is Optimystic's to refuse, in
1414
+ // createControlNode below.
1415
+ clusterPolicy: controlClusterPolicy({
1416
+ enrolledMachines: this.declaredEnrolledMachines,
1417
+ cohortQueryTimeoutMs: network?.cohortQueryTimeoutMs
1418
+ }),
1387
1419
  arachnode: { enableRingZulu: profile === 'storage' },
1388
1420
  ...(identityKey && { privateKey: identityKey }),
1389
1421
  ...(network?.transports && { transports: network.transports }),
1390
1422
  ...(network?.noiseCrypto && { noiseCrypto: network.noiseCrypto }),
1423
+ // Always present, unlike the conditional spreads around it: an absent
1424
+ // `connectionMonitor` would leave libp2p's 5s ping deadline, which drops a peer
1425
+ // whose event loop is saturated by pure-JS Noise crypto. A configured value
1426
+ // replaces the default whole (see DEFAULT_CONNECTION_MONITOR).
1427
+ connectionMonitor: network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
1391
1428
  // `{ wsPort }` when a listen entry names WebSocket, otherwise `{}` — and always
1392
1429
  // `{}` when `network.transports` is set, since the embedder owns transport policy
1393
1430
  // then. Spread NEXT to `transports` because the two answer the same question.
@@ -2735,15 +2772,22 @@ export class CadreNode {
2735
2772
  * {@link SeedBootstrapService} this node builds — `applySeed`'s owner dials and
2736
2773
  * `dialInvite`.
2737
2774
  *
2738
- * See {@link DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS} for why a peer's dial is
2739
- * bounded as a whole, and {@link DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS}
2740
- * for why each address is bounded as well.
2775
+ * See `peer-dial.ts`'s `DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS` for why a peer's dial is
2776
+ * bounded as a whole, and `DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS` for why each
2777
+ * address is bounded as well.
2778
+ *
2779
+ * Both are DERIVED from `network.linkRoundTripMs` rather than fixed, because the slowest
2780
+ * address either has to cover is a relayed dial and that costs a fixed number of exchanges —
2781
+ * `link-budget.ts` has the counts. A host's explicit `controlCohort` values still win, so a
2782
+ * test that drives dead addresses on purpose keeps the duration it chose.
2741
2783
  */
2742
2784
  controlDialBudget() {
2743
2785
  const cohort = this.config.network?.controlCohort;
2786
+ const perAddressMs = cohort?.perAddressDialTimeoutMs
2787
+ ?? relayedDialBudgetMs(this.config.network?.linkRoundTripMs);
2744
2788
  return {
2745
- perAddressMs: cohort?.perAddressDialTimeoutMs ?? DEFAULT_CONTROL_COHORT_PER_ADDRESS_DIAL_TIMEOUT_MS,
2746
- totalMs: cohort?.dialTimeoutMs ?? DEFAULT_CONTROL_COHORT_DIAL_TIMEOUT_MS,
2789
+ perAddressMs,
2790
+ totalMs: cohort?.dialTimeoutMs ?? CONTROL_COHORT_DIAL_ADDRESS_ATTEMPTS * perAddressMs,
2747
2791
  };
2748
2792
  }
2749
2793
  /**
@@ -3670,6 +3714,23 @@ export class CadreNode {
3670
3714
  }
3671
3715
  async handleStrandAdded(strand) {
3672
3716
  log('Handling strand added from control network: %s', strand.Id);
3717
+ // A replicated row carries whatever id the founding node wrote, and the id becomes a
3718
+ // storage scope key and a libp2p protocol prefix. Checked FIRST — above the
3719
+ // discovery branch below — so a malformed id is never offered to the hosting app as
3720
+ // a strand it could join; that app's `addStrand` could only fail on it.
3721
+ //
3722
+ // Suppressed rather than retried: the id is a property of the row, so every later
3723
+ // attempt fails identically, and without this the watcher's ladder would re-emit
3724
+ // `strand:error` every five minutes for the life of the process. The throw still
3725
+ // matters — the watcher's catch runs `forgetStrand`, dropping the id from
3726
+ // `knownStrands`, so its removed-strand loop never detaches a strand that never ran.
3727
+ if (!isValidStrandScopeKey(strand.Id)) {
3728
+ const error = new InvalidStrandIdError(strand.Id);
3729
+ log('Refusing strand %s: %s', strand.Id, error.message);
3730
+ this.emit('strand:error', { strandId: strand.Id, error });
3731
+ this.strandWatcher?.suppressStrand(strand.Id);
3732
+ throw error;
3733
+ }
3673
3734
  // Check if we have sApp config for this strand
3674
3735
  const sAppConfig = this.sAppConfigs.get(strand.Id);
3675
3736
  if (!sAppConfig) {
@@ -3681,10 +3742,12 @@ export class CadreNode {
3681
3742
  // already present) keeps auto-starting below, unchanged.
3682
3743
  //
3683
3744
  // Recorded BEFORE the emit so a handler that synchronously drains
3684
- // `getDiscoveredStrands()` sees this strand too. The watcher never offers
3685
- // the same strand twice (its `knownStrands` retains the id for the life of
3686
- // the process), so this map — not the event — is what a late subscriber
3687
- // reads. See the `strand:discovered` doc in types.ts.
3745
+ // `getDiscoveredStrands()` sees this strand too. The watcher does not offer the
3746
+ // same strand twice — its `knownStrands` retains the id, and the one thing that
3747
+ // un-retains it (a failed `addStrand`, via `StrandWatcher.forgetStrand`) leaves the
3748
+ // sApp config registered, so the retry takes the auto-launch branch below rather
3749
+ // than this one. So this map — not the event — is what a late subscriber reads.
3750
+ // See the `strand:discovered` doc in types.ts.
3688
3751
  this.discoveredStrands.set(strand.Id, strand);
3689
3752
  this.emit('strand:discovered', { strandId: strand.Id, strand });
3690
3753
  return;
@@ -4084,23 +4147,26 @@ export class CadreNode {
4084
4147
  *
4085
4148
  * A rejected call leaves nothing running but DOES leave the sApp config
4086
4149
  * registered, deliberately: both an explicit retry and the {@link StrandWatcher}'s
4087
- * automatic relaunch need it. That background relaunch covers only a strand the
4088
- * WATCHER launched: it forgets a row whose `onStrandAdded` threw and re-attempts it
4089
- * on a later poll (each failure re-emitting `strand:error`) until it succeeds. A
4090
- * strand claimed here after it was DISCOVERED gets no such retry — the watcher
4091
- * already recorded it in its `knownStrands` when it offered it as
4092
- * `strand:discovered`, and no later poll re-offers a strand it knows. This call has
4093
- * also already dropped it from {@link getDiscoveredStrands}, so a failed claim
4094
- * leaves the strand dormant until the caller retries or the node restarts (see
4095
- * `backlog/bug-discovered-strand-lost-when-claim-fails`).
4096
- * {@link detachStrand} (reached via {@link stopStrand}) is what abandons a strand
4097
- * for good.
4150
+ * automatic relaunch need it. A failed launch here hands the strand back to that
4151
+ * relaunch — `StrandWatcher.forgetStrand` drops the id from the watcher's
4152
+ * `knownStrands` and records the backoff, so a later poll re-attempts it on the same
4153
+ * `pollInterval * 2^(failures-1)` ladder a watcher-driven failure gets. Because the
4154
+ * config stays registered, the retry takes `handleStrandAdded`'s auto-launch branch:
4155
+ * what the app sees is `strand:error` per failed retry and `strand:started` when one
4156
+ * succeeds, never a second `strand:discovered`. {@link detachStrand} (reached via
4157
+ * {@link stopStrand}) is what abandons a strand for good, and its stop suppresses the
4158
+ * strand in the watcher so this retry cannot resurrect it — a call to THIS method lifts
4159
+ * that suppression again, since an explicit claim is a deliberate reversal of the
4160
+ * deliberate stop.
4098
4161
  */
4099
4162
  async addStrand(config) {
4100
4163
  if (!this._running) {
4101
4164
  throw new Error('CadreNode not running');
4102
4165
  }
4103
4166
  const { strandRow, sAppConfig, founder, partyMemberPrivateKey } = config;
4167
+ // Before anything is recorded: an unusable id must not leave a registered sApp
4168
+ // config behind, and must not lift a suppression this node set deliberately.
4169
+ assertStrandScopeKey(strandRow.Id);
4104
4170
  if (partyMemberPrivateKey !== undefined && strandRow.Type !== 'c') {
4105
4171
  // Only a closed strand seats a Member/Manager, so an open launch would drop this
4106
4172
  // key silently. Refuse instead: the caller has confused the party identity key
@@ -4113,13 +4179,41 @@ export class CadreNode {
4113
4179
  // unclaimed backlog — a later `getDiscoveredStrands()` drain must not re-offer it.
4114
4180
  this.sAppConfigs.set(strandRow.Id, sAppConfig);
4115
4181
  this.discoveredStrands.delete(strandRow.Id);
4182
+ // A claim also overrides any earlier `stopStrand` of this id: the watcher must track
4183
+ // the strand again, or a party-wide removal would never stop it here.
4184
+ this.strandWatcher?.unsuppressStrand(strandRow.Id);
4116
4185
  log('Registered sAppConfig for strand %s (sApp: %s, founder: %s)', strandRow.Id, sAppConfig.id, founder ?? 'derived');
4117
4186
  // An unset `founder` is DERIVED from the row inside launchStrand (this node founds
4118
4187
  // iff the row's FounderOwnerKey is its own owner key); an explicit flag wins — the
4119
4188
  // formation/responder flows pass one deliberately, since their consent-seated rows
4120
4189
  // carry a null column. See StrandConfig.founder. Same rule for the explicit
4121
4190
  // partyMemberPrivateKey: it wins over the StrandPartyKey control-row read.
4122
- const instance = await this.launchStrand(strandRow, sAppConfig, founder, partyMemberPrivateKey);
4191
+ // Only the launch is wrapped, NOT the first-sync wait below: a wait that times out
4192
+ // rejects with the retryable StrandAwaitingFirstSyncError and deliberately leaves the
4193
+ // instance running, so there is nothing to re-offer and forgetting it would only have
4194
+ // the watcher re-enter a launch for a strand that is already up.
4195
+ let instance;
4196
+ try {
4197
+ instance = await this.launchStrand(strandRow, sAppConfig, founder, partyMemberPrivateKey);
4198
+ }
4199
+ catch (error) {
4200
+ // Hand the strand back to the watcher's retry ladder. Harmless on the founder path
4201
+ // (`foundStrand` → `publishStrand` + this call), where the watcher may never have
4202
+ // offered the strand: the `knownStrands` delete is a no-op and the recorded backoff
4203
+ // only delays the watcher's own first attempt by one poll interval, which is what
4204
+ // should happen right after an attempt that just failed.
4205
+ //
4206
+ // NOTE: the retry relaunches from the CONTROL row plus the registered config, not
4207
+ // from the row and arguments passed here — so a caller that enriched either (a
4208
+ // synthetic `MemberPrivateKey`, an explicit `founder` or `partyMemberPrivateKey`)
4209
+ // is retried with less than it asked for. Inert today: founder-ness re-derives from
4210
+ // `FounderOwnerKey`, the party key re-reads from `StrandPartyKey`, and the row's
4211
+ // shared `MemberPrivateKey` is read only by the founder bootstrap. If a joiner
4212
+ // launch ever starts depending on that key, gate this hand-back on the passed row
4213
+ // matching the control one.
4214
+ this.strandWatcher?.forgetStrand(strandRow.Id);
4215
+ throw error;
4216
+ }
4123
4217
  // The first-sync write gate: a JOINING machine's database is withheld until it has
4124
4218
  // received the strand's Header from another member, because a write before that
4125
4219
  // forks every table it touches (see strand-first-sync-gate.ts). Waiting here is what
@@ -4208,7 +4302,8 @@ export class CadreNode {
4208
4302
  * @returns The live `Strand` row — the one just inserted, or the matching one already
4209
4303
  * there. A closed strand's caller should carry THIS row's `MemberPrivateKey` forward:
4210
4304
  * on a repeat it is the stored key, not the argument.
4211
- * @throws if the node is not started, exposes no owner signing key, the id is blank, a
4305
+ * @throws if the node is not started, exposes no owner signing key, the id is blank or
4306
+ * unusable as a storage scope key (`InvalidStrandIdError` — see `storage-scope.ts`), a
4212
4307
  * row with the same id holds different content, or the control DB rejects the
4213
4308
  * (unauthorized) insert.
4214
4309
  */
@@ -4217,6 +4312,11 @@ export class CadreNode {
4217
4312
  // Trim/reject here so the id that lands matches the one unpublishStrand looks up: it
4218
4313
  // trims too, and an untrimmed row would be unreachable by the same string.
4219
4314
  const trimmed = requireNonBlank(strandId, 'strand id');
4315
+ // Refuse an unusable id HERE, not only at launch. `addStrand` already asserts, but it
4316
+ // runs after this row is written, and this row replicates: an id no node can turn into
4317
+ // a storage scope key would otherwise reach the whole party's control database and be
4318
+ // declined once per member. Asserted after the trim so the id checked is the id stored.
4319
+ assertStrandScopeKey(trimmed);
4220
4320
  // FounderOwnerKey records THIS machine as the row's publisher (the schema pins it to
4221
4321
  // the signing owner), which is what later lets a relaunch derive founder-ness.
4222
4322
  const desired = {
@@ -4457,13 +4557,20 @@ export class CadreNode {
4457
4557
  // one never cleared).
4458
4558
  await this.strandManager.clearOwnMemberPeerBinding(trimmed);
4459
4559
  // Converge locally now rather than waiting up to a poll interval. The watcher fires
4460
- // onStrandRemoved for a strand it tracked; the explicit stop below covers a node whose
4461
- // strandFilter never admitted this strand (the watcher never knew it, so it will never
4462
- // fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two paths
4463
- // cannot double-stop into an error.
4560
+ // onStrandRemoved for a strand it tracked; the explicit teardown below covers a node
4561
+ // whose strandFilter never admitted this strand (the watcher never knew it, so it will
4562
+ // never fire). StrandInstanceManager.stopStrand no-ops on an unknown id, so the two
4563
+ // paths cannot double-stop into an error.
4564
+ //
4565
+ // detachStrand, NOT stopStrand: this is a party-wide removal, not a local abandonment
4566
+ // of a strand that still exists, so it must not SUPPRESS the id in the watcher. The
4567
+ // row is already gone — there is nothing to suppress — and an id re-published before
4568
+ // the next poll's suppression cleanup would then never be offered again this session.
4569
+ // (Same reasoning as handleStrandRemoved, which also avoids stopStrand's `_running`
4570
+ // guard; here the guard is already satisfied by requireOwnerSigningKey above.)
4464
4571
  await this.strandWatcher?.forcePoll();
4465
4572
  if (this.strandManager.getInstance(trimmed)) {
4466
- await this.stopStrand(trimmed);
4573
+ await this.detachStrand(trimmed);
4467
4574
  }
4468
4575
  log('Unpublished strand %s from control DB under owner %s', trimmed, signingKey.publicKeyB64);
4469
4576
  }
@@ -4825,6 +4932,7 @@ export class CadreNode {
4825
4932
  revocationEnforcement: this.config.strandRevocationEnforcement,
4826
4933
  membershipReconciliation: this.config.strandMembershipReconciliation,
4827
4934
  onSelfRevoked: (revokedStrandId) => this.emit('strand:revoked', { strandId: revokedStrandId }),
4935
+ onRejoinBlocked: (blockedStrandId) => this.emit('strand:rejoin-blocked', { strandId: blockedStrandId }),
4828
4936
  // The joiner's first-sync write gate (strand-first-sync-gate.ts): a launch that
4829
4937
  // comes up `'syncing'` announces the moment its database is published.
4830
4938
  firstSync: this.config.strandFirstSync,
@@ -4838,7 +4946,7 @@ export class CadreNode {
4838
4946
  // cleared once spent, burned, or dead — see pendingMembershipInvites.
4839
4947
  pendingMembershipInvite: {
4840
4948
  get: () => this.pendingMembershipInvites.get(strand.Id),
4841
- clear: () => { this.pendingMembershipInvites.delete(strand.Id); }
4949
+ clear: (settled) => this.unstageMembershipInvite(strand.Id, settled)
4842
4950
  },
4843
4951
  // The RESOLVED flag, never the raw argument — see the doc comment above.
4844
4952
  founder: resolvedFounder,
@@ -5066,8 +5174,10 @@ export class CadreNode {
5066
5174
  * caller runs the drive regardless).
5067
5175
  *
5068
5176
  * NOTE: against a relay that is DOWN this hook costs up to two strand-addr
5069
- * timeouts (10 s each: dial by peer id, then by addr) before the 10 s drive even
5070
- * starts, so one failed re-drive can hold the supervisor `driving` for ~30 s.
5177
+ * timeouts (10 s each: dial by peer id, then by addr) before the reservation drive even
5178
+ * starts, so one failed re-drive holds the supervisor `driving` for those 20 s plus the
5179
+ * drive's own deadline — 28 s at the default declared link round trip, and longer on a host
5180
+ * that declared a slower one (`link-budget.ts`).
5071
5181
  * Bounded and harmless while the relay is unreachable anyway; if recovery
5072
5182
  * latency after a relay comes back ever matters, skip the announce when the
5073
5183
  * control node holds no connection to the relay (the drive's own dial fails
@@ -5313,16 +5423,20 @@ export class CadreNode {
5313
5423
  * Stop a strand on THIS node only: untrack it from hibernation, drop its sApp config,
5314
5424
  * stop the local instance, and emit `strand:stopped`. The shared `Strand` row is left
5315
5425
  * intact, so on the next node RESTART the strand is rediscovered and surfaces as
5316
- * `strand:discovered` again — not on the next watcher poll, which never re-offers a
5317
- * strand it has already seen (`StrandWatcher.knownStrands` retains the id for the life
5318
- * of the process). The stop also drops the strand from {@link getDiscoveredStrands},
5319
- * so a drain cannot undo a deliberate stop. Party-wide removal is
5320
- * {@link unpublishStrand}.
5426
+ * `strand:discovered` again — but never again in THIS session: the stop suppresses the
5427
+ * id in the watcher (`StrandWatcher.suppressStrand`) and drops it from
5428
+ * {@link getDiscoveredStrands}, so neither a later poll nor a drain can undo a
5429
+ * deliberate stop. Only an explicit {@link addStrand} does, which is the caller
5430
+ * reversing its own decision. Party-wide removal is {@link unpublishStrand}.
5321
5431
  */
5322
5432
  async stopStrand(strandId) {
5323
5433
  if (!this._running) {
5324
5434
  throw new Error('CadreNode not running');
5325
5435
  }
5436
+ // Suppressed HERE and not in detachStrand: the other caller of that method is
5437
+ // `handleStrandRemoved`, which arrives precisely because the control row is gone, and
5438
+ // a row that later reappears is a fresh strand that should be offered again.
5439
+ this.strandWatcher?.suppressStrand(strandId);
5326
5440
  await this.detachStrand(strandId);
5327
5441
  }
5328
5442
  /**
@@ -5343,6 +5457,10 @@ export class CadreNode {
5343
5457
  * readings are the intended one: `handleStrandRemoved` arrives because the control row
5344
5458
  * is gone, and an explicit {@link stopStrand} is a deliberate abandonment — neither
5345
5459
  * strand may be re-offered to a later `getDiscoveredStrands()` drain.
5460
+ *
5461
+ * The watcher-level suppression that makes a stop permanent is deliberately NOT here,
5462
+ * only in {@link stopStrand}: the two callers diverge on it. A vanished control row
5463
+ * that reappears is a fresh strand and must be offered again.
5346
5464
  */
5347
5465
  async detachStrand(strandId) {
5348
5466
  this.hibernationManager.untrackStrand(strandId);
@@ -5557,13 +5675,15 @@ export class CadreNode {
5557
5675
  * local), and only then does it reach out. Everything above in {@link start}
5558
5676
  * has completed by the time this runs.
5559
5677
  *
5560
- * BUDGET: the supervisor's first attempt — `relay-reservation.ts`'s
5561
- * `DEFAULT_RELAY_RESERVE_TIMEOUT_MS`, 10 s — is what `start()` waits on — deliberately the module default rather
5562
- * than a boot-specific one. A healthy dial-plus-reserve is sub-second even over
5563
- * a WAN, so 10 s is slack for a slow link; going much longer would make a dead
5564
- * relay indistinguishable from a hung start, and much shorter would fail nodes
5565
- * on links that were merely slow. The retries carry on in the background after
5566
- * this resolves, exactly as they do for a {@link reserveRelays} caller.
5678
+ * BUDGET: the supervisor's first attempt is what `start()` waits on — deliberately the drive's
5679
+ * ordinary deadline rather than a boot-specific one. That deadline is now COUNTED, four link
5680
+ * round trips at the declared `network.linkRoundTripMs` (8 s at its default, where it was a
5681
+ * fixed 10 s): a healthy dial-plus-reserve is sub-second even over a WAN, so this is slack for
5682
+ * a slow link, while going much longer would make a dead relay indistinguishable from a hung
5683
+ * start and much shorter would fail nodes on links that were merely slow. A host that declares
5684
+ * a slower link lengthens it without touching this path — `link-budget.ts`. The retries carry
5685
+ * on in the background after this resolves, exactly as they do for a {@link reserveRelays}
5686
+ * caller.
5567
5687
  *
5568
5688
  * `network.requireRelay === false` softens only the outcome below: a first
5569
5689
  * attempt that lands no `/p2p-circuit` address is logged instead of thrown, and
@@ -5625,7 +5745,13 @@ export class CadreNode {
5625
5745
  this.relayReserveError = 'control node unavailable';
5626
5746
  return this.getRelayReservationState();
5627
5747
  }
5628
- const supervisor = superviseRelayReservation(this.controlNode, addrs, opts);
5748
+ // The drive's deadline is counted in link round trips (`link-budget.ts`), so a host that
5749
+ // declared a slower link gets a longer drive without naming a number here. A caller's own
5750
+ // `timeoutMs` still wins.
5751
+ const supervisor = superviseRelayReservation(this.controlNode, addrs, {
5752
+ timeoutMs: relayReservationBudgetMs(this.config.network?.linkRoundTripMs),
5753
+ ...opts
5754
+ });
5629
5755
  this.relayReserveSupervisor = supervisor;
5630
5756
  await supervisor.firstAttempt;
5631
5757
  // Read through `this`, not `supervisor`: an overlapping call may have replaced
@@ -6469,7 +6595,11 @@ export class CadreNode {
6469
6595
  * an intact control DB) and mints + persists a fresh one otherwise. The invitation
6470
6596
  * will admit THIS key's public half as the `Strand.Member`.
6471
6597
  * 2. Stage the invitation in {@link pendingMembershipInvites} for the strand
6472
- * bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite`.
6598
+ * bring-up flow (`strand-node-binds-member-peer`) to redeem via `consumeInvite`,
6599
+ * and tell the instance manager, which re-arms an already-finished membership
6600
+ * reconciler so a RE-formation against a launched strand is attempted at once — the
6601
+ * removed-party case, where the loop finished long before the removal. A strand not
6602
+ * launched here yet has no loop to re-arm, and its bring-up finds the entry.
6473
6603
  *
6474
6604
  * Throws — failing the whole {@link formStrand} — when the identity cannot be
6475
6605
  * persisted: a joiner "joined" without a persistable identity could never become a
@@ -6488,16 +6618,23 @@ export class CadreNode {
6488
6618
  'cannot become a member without it. Fix the underlying cause, then redeem a fresh ' +
6489
6619
  'invitation (the delivered one dies with this error).', { cause: error });
6490
6620
  }
6491
- // NOTE: staging alone is not enough on a node whose reconciler for this strand has
6492
- // already finished. `StrandMembershipReconciler` latches a terminal stopped state on
6493
- // its `done` path and is rebuilt only by a strand relaunch, so a RE-formation against
6494
- // an already-launched strand stages an invitation no loop will ever look at. Harmless
6495
- // for a party that is still a member (the invitation would only be burned), and the
6496
- // live defect for one that has been REMOVED — which is exactly when an app re-forms.
6497
- // Tracked as `backlog/bug-removed-party-cannot-redeem-its-way-back`; pinned by
6498
- // `strand-party-removal-via-formation-e2e.integration.ts` (test 2).
6499
6621
  this.pendingMembershipInvites.set(strandId, invite);
6500
6622
  log('formStrand: staged membership invitation for strand %s (party key persisted)', strandId);
6623
+ this.strandManager.notifyMembershipInviteStaged(strandId);
6624
+ }
6625
+ /**
6626
+ * The reconciler's half of the {@link pendingMembershipInvites} seam: drop `settled`
6627
+ * (spent, burned, or dead) only while it is still the staged entry. A re-formation
6628
+ * replaces the entry between a pass's read and its settle, and the fresh invitation it
6629
+ * staged is the one the re-armed loop is about to redeem — deleting it here would lose
6630
+ * it silently, the very outcome the re-arm exists to prevent.
6631
+ */
6632
+ unstageMembershipInvite(strandId, settled) {
6633
+ if (this.pendingMembershipInvites.get(strandId)?.inviteKey !== settled.inviteKey) {
6634
+ log('strand %s: a fresh membership invitation replaced the one just settled — keeping it staged', strandId);
6635
+ return;
6636
+ }
6637
+ this.pendingMembershipInvites.delete(strandId);
6501
6638
  }
6502
6639
  /**
6503
6640
  * The pending single-use membership invitation a closed-strand formation carried back