@serfab/cadre-core 1.7.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/README.md +8 -6
  2. package/dist/cadre-node.d.ts +527 -206
  3. package/dist/cadre-node.js +1084 -528
  4. package/dist/cadre-node.js.map +1 -1
  5. package/dist/control-authorization.d.ts +2 -2
  6. package/dist/control-authorization.js +1 -0
  7. package/dist/control-authorization.js.map +1 -1
  8. package/dist/control-cohort.d.ts +6 -0
  9. package/dist/control-cohort.js +6 -0
  10. package/dist/control-cohort.js.map +1 -1
  11. package/dist/control-database.d.ts +46 -8
  12. package/dist/control-database.js +97 -18
  13. package/dist/control-database.js.map +1 -1
  14. package/dist/control-formation-recorder.d.ts +37 -14
  15. package/dist/control-formation-recorder.js +60 -10
  16. package/dist/control-formation-recorder.js.map +1 -1
  17. package/dist/control-read-retry.d.ts +13 -12
  18. package/dist/control-read-retry.js +14 -12
  19. package/dist/control-read-retry.js.map +1 -1
  20. package/dist/control-schema.d.ts +1 -1
  21. package/dist/control-schema.js +67 -5
  22. package/dist/control-schema.js.map +1 -1
  23. package/dist/control-stream.d.ts +9 -1
  24. package/dist/control-stream.js +24 -1
  25. package/dist/control-stream.js.map +1 -1
  26. package/dist/control-write-retry.d.ts +12 -2
  27. package/dist/control-write-retry.js +13 -2
  28. package/dist/control-write-retry.js.map +1 -1
  29. package/dist/delegate-admission.d.ts +11 -8
  30. package/dist/delegate-admission.js +19 -11
  31. package/dist/delegate-admission.js.map +1 -1
  32. package/dist/diagnostics/connection-path.d.ts +0 -7
  33. package/dist/diagnostics/connection-path.js +0 -7
  34. package/dist/diagnostics/connection-path.js.map +1 -1
  35. package/dist/formation-approval.d.ts +7 -0
  36. package/dist/formation-approval.js +9 -3
  37. package/dist/formation-approval.js.map +1 -1
  38. package/dist/fs-atomic.d.ts +11 -2
  39. package/dist/fs-atomic.js +13 -3
  40. package/dist/fs-atomic.js.map +1 -1
  41. package/dist/hibernation-manager.d.ts +111 -8
  42. package/dist/hibernation-manager.js +234 -49
  43. package/dist/hibernation-manager.js.map +1 -1
  44. package/dist/index.d.ts +10 -10
  45. package/dist/index.js +25 -20
  46. package/dist/index.js.map +1 -1
  47. package/dist/joined-strand-store.d.ts +90 -25
  48. package/dist/joined-strand-store.js +223 -55
  49. package/dist/joined-strand-store.js.map +1 -1
  50. package/dist/key-store-file.js +14 -5
  51. package/dist/key-store-file.js.map +1 -1
  52. package/dist/link-budget.d.ts +367 -69
  53. package/dist/link-budget.js +375 -72
  54. package/dist/link-budget.js.map +1 -1
  55. package/dist/membership-connection-gater.d.ts +30 -43
  56. package/dist/membership-connection-gater.js +31 -42
  57. package/dist/membership-connection-gater.js.map +1 -1
  58. package/dist/node-local-snapshot.d.ts +6 -4
  59. package/dist/node-local-snapshot.js +7 -3
  60. package/dist/node-local-snapshot.js.map +1 -1
  61. package/dist/peer-dial.d.ts +24 -4
  62. package/dist/peer-dial.js +74 -5
  63. package/dist/peer-dial.js.map +1 -1
  64. package/dist/peer-join-backfill.d.ts +2 -10
  65. package/dist/peer-join-backfill.js +8 -8
  66. package/dist/peer-join-backfill.js.map +1 -1
  67. package/dist/push-fanout.d.ts +6 -0
  68. package/dist/push-fanout.js +6 -0
  69. package/dist/push-fanout.js.map +1 -1
  70. package/dist/relay-reservation.d.ts +3 -1
  71. package/dist/relay-reservation.js +3 -1
  72. package/dist/relay-reservation.js.map +1 -1
  73. package/dist/relay-server.d.ts +79 -0
  74. package/dist/relay-server.js +74 -0
  75. package/dist/relay-server.js.map +1 -0
  76. package/dist/seed-bootstrap.d.ts +57 -4
  77. package/dist/seed-bootstrap.js +107 -83
  78. package/dist/seed-bootstrap.js.map +1 -1
  79. package/dist/storage-scope.d.ts +33 -17
  80. package/dist/storage-scope.js +44 -23
  81. package/dist/storage-scope.js.map +1 -1
  82. package/dist/strand-addr-protocol.d.ts +58 -21
  83. package/dist/strand-addr-protocol.js +133 -85
  84. package/dist/strand-addr-protocol.js.map +1 -1
  85. package/dist/strand-database.d.ts +20 -2
  86. package/dist/strand-database.js +42 -5
  87. package/dist/strand-database.js.map +1 -1
  88. package/dist/strand-first-sync-gate.d.ts +59 -21
  89. package/dist/strand-first-sync-gate.js +58 -19
  90. package/dist/strand-first-sync-gate.js.map +1 -1
  91. package/dist/strand-formation-deadlines.d.ts +151 -0
  92. package/dist/strand-formation-deadlines.js +171 -0
  93. package/dist/strand-formation-deadlines.js.map +1 -0
  94. package/dist/strand-formation-manager.d.ts +63 -30
  95. package/dist/strand-formation-manager.js +67 -41
  96. package/dist/strand-formation-manager.js.map +1 -1
  97. package/dist/strand-formation-protocol.d.ts +45 -46
  98. package/dist/strand-formation-protocol.js +62 -126
  99. package/dist/strand-formation-protocol.js.map +1 -1
  100. package/dist/strand-fret-addrs.d.ts +54 -0
  101. package/dist/strand-fret-addrs.js +108 -0
  102. package/dist/strand-fret-addrs.js.map +1 -0
  103. package/dist/strand-instance-manager.d.ts +122 -44
  104. package/dist/strand-instance-manager.js +270 -114
  105. package/dist/strand-instance-manager.js.map +1 -1
  106. package/dist/strand-membership-writer.d.ts +6 -0
  107. package/dist/strand-membership-writer.js +11 -0
  108. package/dist/strand-membership-writer.js.map +1 -1
  109. package/dist/strand-network-state-file.d.ts +12 -0
  110. package/dist/strand-network-state-file.js +38 -0
  111. package/dist/strand-network-state-file.js.map +1 -0
  112. package/dist/strand-network-state.d.ts +111 -0
  113. package/dist/strand-network-state.js +191 -0
  114. package/dist/strand-network-state.js.map +1 -0
  115. package/dist/strand-revocation-enforcer.js +2 -1
  116. package/dist/strand-revocation-enforcer.js.map +1 -1
  117. package/dist/strand-solicitation.d.ts +36 -20
  118. package/dist/strand-solicitation.js +4 -4
  119. package/dist/strand-solicitation.js.map +1 -1
  120. package/dist/strand-wake-protocol.d.ts +80 -23
  121. package/dist/strand-wake-protocol.js +89 -64
  122. package/dist/strand-wake-protocol.js.map +1 -1
  123. package/dist/types.d.ts +148 -61
  124. package/dist/types.js +4 -3
  125. package/dist/types.js.map +1 -1
  126. package/package.json +11 -10
  127. package/src/cadre-node.ts +8400 -7776
  128. package/src/control-authorization.ts +2 -1
  129. package/src/control-cohort.ts +6 -0
  130. package/src/control-database.ts +122 -18
  131. package/src/control-formation-recorder.ts +354 -305
  132. package/src/control-read-retry.ts +14 -12
  133. package/src/control-schema.ts +67 -5
  134. package/src/control-stream.ts +25 -1
  135. package/src/control-write-retry.ts +13 -2
  136. package/src/delegate-admission.ts +23 -13
  137. package/src/diagnostics/connection-path.ts +0 -7
  138. package/src/formation-approval.ts +610 -604
  139. package/src/fs-atomic.ts +14 -4
  140. package/src/hibernation-manager.ts +638 -437
  141. package/src/index.ts +43 -65
  142. package/src/joined-strand-store.ts +247 -57
  143. package/src/key-store-file.ts +14 -5
  144. package/src/link-budget.ts +393 -74
  145. package/src/membership-connection-gater.ts +31 -45
  146. package/src/node-local-snapshot.ts +12 -7
  147. package/src/peer-dial.ts +81 -7
  148. package/src/peer-join-backfill.ts +10 -9
  149. package/src/push-fanout.ts +6 -0
  150. package/src/relay-reservation.ts +3 -1
  151. package/src/relay-server.ts +98 -0
  152. package/src/seed-bootstrap.ts +133 -91
  153. package/src/storage-scope.ts +44 -23
  154. package/src/strand-addr-protocol.ts +547 -458
  155. package/src/strand-database.ts +287 -242
  156. package/src/strand-first-sync-gate.ts +58 -19
  157. package/src/strand-formation-deadlines.ts +227 -0
  158. package/src/strand-formation-manager.ts +700 -647
  159. package/src/strand-formation-protocol.ts +106 -154
  160. package/src/strand-fret-addrs.ts +145 -0
  161. package/src/strand-instance-manager.ts +307 -140
  162. package/src/strand-membership-writer.ts +12 -0
  163. package/src/strand-network-state-file.ts +40 -0
  164. package/src/strand-network-state.ts +243 -0
  165. package/src/strand-revocation-enforcer.ts +2 -1
  166. package/src/strand-solicitation.ts +519 -500
  167. package/src/strand-wake-protocol.ts +401 -355
  168. package/src/types.ts +150 -61
  169. package/dist/strand-peer-book-file.d.ts +0 -12
  170. package/dist/strand-peer-book-file.js +0 -37
  171. package/dist/strand-peer-book-file.js.map +0 -1
  172. package/dist/strand-peer-book-protocol.d.ts +0 -159
  173. package/dist/strand-peer-book-protocol.js +0 -360
  174. package/dist/strand-peer-book-protocol.js.map +0 -1
  175. package/dist/strand-peer-book-swap.d.ts +0 -104
  176. package/dist/strand-peer-book-swap.js +0 -284
  177. package/dist/strand-peer-book-swap.js.map +0 -1
  178. package/dist/strand-peer-book.d.ts +0 -159
  179. package/dist/strand-peer-book.js +0 -354
  180. package/dist/strand-peer-book.js.map +0 -1
  181. package/dist/strand-peer-observer.d.ts +0 -88
  182. package/dist/strand-peer-observer.js +0 -180
  183. package/dist/strand-peer-observer.js.map +0 -1
  184. package/src/strand-peer-book-file.ts +0 -39
  185. package/src/strand-peer-book-protocol.ts +0 -463
  186. package/src/strand-peer-book-swap.ts +0 -338
  187. package/src/strand-peer-book.ts +0 -468
  188. package/src/strand-peer-observer.ts +0 -235
@@ -3,8 +3,7 @@ import { createLibp2pNode } from '@optimystic/db-p2p';
3
3
  import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
4
4
  import { StrandDatabase } from './strand-database.js';
5
5
  import { PeerJoinBackfill } from './peer-join-backfill.js';
6
- import { StrandPeerObserver } from './strand-peer-observer.js';
7
- import { StrandPeerBookSwap } from './strand-peer-book-swap.js';
6
+ import { strandNetworkStatePersistence } from './strand-network-state.js';
8
7
  import { StrandRevocationEnforcer, createRevocationConnectionGater, readStrandRevocationRows } from './strand-revocation-enforcer.js';
9
8
  import { StrandMembershipReconciler } from './strand-membership-reconciler.js';
10
9
  import { removeMemberPeer } from './strand-membership-writer.js';
@@ -14,8 +13,9 @@ import { assertStrandScopeKey } from './storage-scope.js';
14
13
  import { StrandFirstSyncGate, StrandAwaitingFirstSyncError, strandFirstSyncComplete, DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS } from './strand-first-sync-gate.js';
15
14
  import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
16
15
  import { strandNodeAddrs } from './strand-network-config.js';
16
+ import { resolveRelayServer } from './relay-server.js';
17
17
  import { superviseRelayReservation } from './relay-reservation.js';
18
- import { connectionManagerTimeouts, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
18
+ import { declaredCohortReadDeadlineMs, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
19
19
  const log = debug('sereus:cadre:strand-manager');
20
20
  const timing = debug('sereus:cadre:timing');
21
21
  /**
@@ -99,6 +99,24 @@ function resolveStrandStorage(provider, strandId) {
99
99
  // lifetime — so the wrap survives every runtime rebuild the instance goes through.
100
100
  return wrapStorageWithCache(storage, strandId);
101
101
  }
102
+ /**
103
+ * Verify an sApp's schema signature (fail-closed unless `requireSignedSchemas` is
104
+ * false) and project the config onto the instance's `sAppInfo`.
105
+ */
106
+ function verifiedSAppInfo(strandId, sAppConfig, requireSignedSchemas) {
107
+ assertSchemaSignature(sAppConfig, { requireSignature: requireSignedSchemas ?? true });
108
+ log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
109
+ return {
110
+ id: sAppConfig.id,
111
+ version: sAppConfig.version,
112
+ schema: sAppConfig.schema,
113
+ signature: sAppConfig.signature
114
+ };
115
+ }
116
+ /** Log wording for what a launch runs: the sApp and its version, or a storage replica. */
117
+ function describeSApp(sAppConfig) {
118
+ return sAppConfig ? `sApp: ${sAppConfig.id} v${sAppConfig.version}` : 'storage replica';
119
+ }
102
120
  /**
103
121
  * Manages individual strand instances - creates and destroys isolated libp2p nodes
104
122
  * for each strand the cadre participates in.
@@ -121,23 +139,6 @@ export class StrandInstanceManager {
121
139
  * missed writes).
122
140
  */
123
141
  this.backfills = new Map();
124
- /**
125
- * The per-strand peer-book observation writer (`strand-peer-observer.ts`), keyed
126
- * by strand id. Same lifecycle rationale as {@link backfills}: started in
127
- * `buildStrandRuntime` right after the libp2p node exists, stopped and dropped in
128
- * `releaseRuntime`, so quiesce → resume re-arms it on the rebuilt node. Only
129
- * present when the launch config supplied `onStrandPeerIdentified`.
130
- */
131
- this.peerObservers = new Map();
132
- /**
133
- * Per-strand strand peer book SWAP driver ({@link StrandPeerBookSwap}) — the signed
134
- * exchange of book entries between strand peers. Same lifetime as the peer-book
135
- * observer above: armed in `buildStrandRuntime` right after the libp2p node exists
136
- * (its protocol handler must be registered before an early peer dials it), stopped
137
- * and dropped in `releaseRuntime`. Only present when the launch config supplied
138
- * `strandPeerBook`.
139
- */
140
- this.peerBookSwaps = new Map();
141
142
  /**
142
143
  * The per-strand revoked-peer enforcer (closed strands only), keyed by strand
143
144
  * id. Same lifecycle rationale as {@link backfills}: created in
@@ -204,6 +205,23 @@ export class StrandInstanceManager {
204
205
  * wake becomes the better trade.
205
206
  */
206
207
  this.strandStorages = new Map();
208
+ /**
209
+ * The runtime build in flight per strand id (`buildStrandRuntime`, from `startStrand` or
210
+ * `resumeStrand`), present until it settles. The instance is tracked for the whole build,
211
+ * with its database not yet constructed or not yet initialized, so a caller that must act
212
+ * on the FINISHED runtime ({@link attachSApp}, {@link whenRuntimeBuilt}) waits on this instead. A failed build's
213
+ * rejection belongs to the call that started it.
214
+ */
215
+ this.runtimeBuilds = new Map();
216
+ /**
217
+ * The latest resume or quiesce requested per strand id, present until it settles. Each one
218
+ * starts only once the one requested before it has settled, so a wake that lands during a
219
+ * quiesce rebuilds after it, and a quiesce requested during a rebuild releases what that
220
+ * rebuild finished. The WHOLE operation, not its entry in {@link runtimeBuilds}: that settles
221
+ * before a resume's own catch records `'error'`, so a joiner awaiting it could read a stale
222
+ * status.
223
+ */
224
+ this.transitions = new Map();
207
225
  this.stopping = false;
208
226
  log('StrandInstanceManager created');
209
227
  }
@@ -259,7 +277,7 @@ export class StrandInstanceManager {
259
277
  log('Strand %s already running', strandId);
260
278
  return this.instances.get(strandId);
261
279
  }
262
- log('Starting strand instance: %s (sApp: %s v%s)', strandId, sAppConfig.id, sAppConfig.version);
280
+ log('Starting strand instance: %s (%s)', strandId, describeSApp(sAppConfig));
263
281
  const tTotal = performance.now();
264
282
  // The id becomes two names below: the storage scope key the embedder's provider
265
283
  // turns into a directory or database name, and `networkName` in buildStrandRuntime,
@@ -269,17 +287,7 @@ export class StrandInstanceManager {
269
287
  // function returns early when no provider is configured — a node with no storage
270
288
  // still reaches buildStrandRuntime and still mints the protocol prefix.
271
289
  assertStrandScopeKey(strandId);
272
- // Verify schema signature before proceeding (fail-closed by default)
273
- const requireSignature = config.requireSignedSchemas ?? true;
274
- assertSchemaSignature(sAppConfig, { requireSignature });
275
- log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
276
- // Convert SAppConfig to SAppInfo for the instance
277
- const sAppInfo = {
278
- id: sAppConfig.id,
279
- version: sAppConfig.version,
280
- schema: sAppConfig.schema,
281
- signature: sAppConfig.signature
282
- };
290
+ const sAppInfo = sAppConfig ? verifiedSAppInfo(strandId, sAppConfig, config.requireSignedSchemas) : undefined;
283
291
  // Resolve this strand's storage ONCE, before anything is recorded. If a factory
284
292
  // function is provided, it is called with the strandId to create strand-specific
285
293
  // storage (e.g. strand-isolated directories). A provider that throws therefore
@@ -289,7 +297,7 @@ export class StrandInstanceManager {
289
297
  log('Strand %s using provided storage provider', strandId);
290
298
  }
291
299
  // Determine latency hint: sApp config > default
292
- const latencyHint = sAppConfig.latencyHint ?? config.defaultLatencyHint;
300
+ const latencyHint = sAppConfig?.latencyHint ?? config.defaultLatencyHint;
293
301
  const instance = {
294
302
  strandId,
295
303
  status: 'starting',
@@ -305,9 +313,9 @@ export class StrandInstanceManager {
305
313
  this.strandStorages.set(strandId, strandStorage);
306
314
  }
307
315
  try {
308
- await this.buildStrandRuntime(instance, config);
316
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, config));
309
317
  timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
310
- log('Strand %s started successfully with sApp %s', strandId, sAppConfig.id);
318
+ log('Strand %s started successfully (%s)', strandId, describeSApp(sAppConfig));
311
319
  return instance;
312
320
  }
313
321
  catch (error) {
@@ -326,6 +334,34 @@ export class StrandInstanceManager {
326
334
  throw error;
327
335
  }
328
336
  }
337
+ /** Hold `build` in {@link runtimeBuilds} until it settles, and settle as it does. */
338
+ async trackRuntimeBuild(strandId, build) {
339
+ this.runtimeBuilds.set(strandId, build);
340
+ try {
341
+ await build;
342
+ }
343
+ finally {
344
+ if (this.runtimeBuilds.get(strandId) === build) {
345
+ this.runtimeBuilds.delete(strandId);
346
+ }
347
+ }
348
+ }
349
+ /**
350
+ * Wait until no runtime build of `strandId` is in flight. Never rejects: a failed build
351
+ * is reported by the call that started it, and the caller re-reads the instance after.
352
+ *
353
+ * NOTE: after a failed launch the re-read sees the instance gone only because
354
+ * `startStrand`'s catch deletes it synchronously, and its continuation is queued before
355
+ * this waiter's. If that cleanup ever awaits before `instances.delete`, a waiter can act
356
+ * on the dying record; move the cleanup inside the tracked promise then.
357
+ */
358
+ async settleRuntimeBuilds(strandId) {
359
+ for (let build = this.runtimeBuilds.get(strandId); build; build = this.runtimeBuilds.get(strandId)) {
360
+ await build.catch((error) => {
361
+ log('Strand %s: the runtime build waited on failed (reported by its launcher): %o', strandId, error);
362
+ });
363
+ }
364
+ }
329
365
  /**
330
366
  * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
331
367
  * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
@@ -353,9 +389,9 @@ export class StrandInstanceManager {
353
389
  // with — hence one binding for both, not two literals that can drift.
354
390
  const networkName = `strand-${strandId}`;
355
391
  const protocolPrefix = `/optimystic/${networkName}`;
356
- // Determine relay mode: if explicitly set in config, use that;
357
- // otherwise default to true for storage profile nodes.
358
- const enableRelay = config.network?.enableRelay ?? (config.profile === 'storage');
392
+ // Whether this node runs the circuit-relay server, and its init — the same
393
+ // resolution the control node takes from the same config (`relay-server.ts`).
394
+ const relayServer = resolveRelayServer(config.network, config.profile);
359
395
  // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
360
396
  // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
361
397
  // bind one port), the announce config is dropped (it names the control node's
@@ -423,7 +459,12 @@ export class StrandInstanceManager {
423
459
  networkName,
424
460
  storage: strandStorage,
425
461
  fretProfile: config.profile === 'storage' ? 'core' : 'edge',
426
- relay: enableRelay,
462
+ relay: relayServer.enabled,
463
+ // NOTE: unlike the control node, a strand node has no unauthorized-reservation budget,
464
+ // so with the per-connection cap off any peer that reaches this node can hold one of
465
+ // `maxReservations` slots and forward uncapped through it. If strand relays are ever
466
+ // abused for bandwidth, give strand nodes a budget or a capped init of their own.
467
+ ...(relayServer.enabled && { relayServerInit: relayServer.init }),
427
468
  clusterSize: strandClusterSize,
428
469
  // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
429
470
  // breadth reasoning, and the shape match with the control policy is a coincidence.
@@ -435,16 +476,29 @@ export class StrandInstanceManager {
435
476
  // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
436
477
  // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
437
478
  // serving set that changed while the strand slept.
438
- // The read deadline is the same field the control node reads, because the two
439
- // networks ride one link; absent, the base policy's COHORT_READ_DEADLINE_MS stands.
479
+ // The read deadline is settled by the same helper the control node uses, because the
480
+ // two networks ride one link: the host's own, else derived from its declared
481
+ // `linkRoundTripMs`, else nothing, and the base policy's COHORT_READ_DEADLINE_MS stands.
440
482
  clusterPolicy: strandClusterPolicy(strandClusterSize, {
441
483
  servingMachines: config.servingMachines,
442
- cohortQueryTimeoutMs: config.network?.cohortQueryTimeoutMs
484
+ cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(config.network)
443
485
  }),
444
486
  arachnode: {
445
487
  enableRingZulu: config.profile === 'storage'
446
488
  },
447
489
  ...(config.privateKey && { privateKey: config.privateKey }),
490
+ // Built per runtime, not per launch: the adapter stops saving once the strand's
491
+ // state is forgotten, and a rebuilt runtime is what starts it saving again.
492
+ //
493
+ // NOTE: the saved state also restores which peers db-p2p saw serving the strand,
494
+ // and it counts them in the cohort until FRET finds them unreachable. A node built
495
+ // while every other member is offline therefore fails its first writes ("Failed to
496
+ // get super-majority") — for 4-6 s on loopback, the window a running node already
497
+ // sees when a member drops. If apps report failed sends after relaunching alone on
498
+ // a slow relayed link, retry the write in the app or raise it upstream (db-p2p's
499
+ // remembered-serving fallback); dropping `persistence` brings back the split
500
+ // restart.
501
+ ...(config.networkState && { persistence: strandNetworkStatePersistence(config.networkState, strandId) }),
448
502
  ...(config.network?.transports && { transports: config.network.transports }),
449
503
  ...(config.network?.noiseCrypto && { noiseCrypto: config.network.noiseCrypto }),
450
504
  // Unconditional, and the same default the control node takes: every node of
@@ -452,10 +506,10 @@ export class StrandInstanceManager {
452
506
  // widened ping deadline has to reach the strand nodes too (see
453
507
  // DEFAULT_CONNECTION_MONITOR).
454
508
  connectionMonitor: config.network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
455
- // The same declared-link limits the control node takes, and for the same reason: a
456
- // strand node is the listener for every other member's strand node (`link-budget.ts`,
457
- // `connectionManagerTimeouts`).
458
- connectionManager: connectionManagerTimeouts(config.network?.linkRoundTripMs),
509
+ // The same declared link the control node states, and for the same reason: a strand
510
+ // node is also the listener for every other member's strand node, and Optimystic
511
+ // derives that listener's limit from this (`link-budget.ts`).
512
+ linkRoundTripMs: resolveLinkRoundTripMs(config.network?.linkRoundTripMs),
459
513
  // Listen entries plus the WebSocket transport switch they imply — a strand node
460
514
  // announces nothing the operator configured (`strand-network-config.ts`), and
461
515
  // spreads AFTER `transports` above because the switch is a no-op whenever the
@@ -473,46 +527,6 @@ export class StrandInstanceManager {
473
527
  });
474
528
  timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
475
529
  instance.libp2pNode = node;
476
- // The peer-book observation writer, armed BEFORE the database bring-up below
477
- // (unlike the backfill) because the bootstrap dials that follow `libp2p.start()`
478
- // land during that bring-up, and they are exactly the peers worth remembering:
479
- // identify for an early peer may already have fired by the time anything after
480
- // the database initializes runs. Registered in the map immediately so the
481
- // failure rollback (releaseRuntime) stops it like every other runtime component.
482
- const onStrandPeerIdentified = config.onStrandPeerIdentified;
483
- if (onStrandPeerIdentified) {
484
- const observer = new StrandPeerObserver({
485
- label: strandId,
486
- libp2p: node,
487
- protocolPrefix,
488
- onObserved: (observation) => onStrandPeerIdentified(strandId, observation)
489
- });
490
- this.peerObservers.set(strandId, observer);
491
- observer.start();
492
- }
493
- // The strand peer book SWAP, armed at the same moment and for the same reason as
494
- // the observer: a peer that dials this node's swap handler before it is registered
495
- // fails its exchange and sits out a ten-minute throttle. The own entry is signed
496
- // now and re-signed on every address change, which covers the relay reservation
497
- // landing below (`self:peer:update` fires when the circuit addr is added), so no
498
- // separate post-relay step is needed. Registered in the map immediately so the
499
- // failure rollback (releaseRuntime) stops it like every other runtime component.
500
- //
501
- // NOTE: this is a third `peer:identify` listener per running strand, beside the
502
- // observer's and the backfill's — see the backfill's NOTE below for when to fold
503
- // them into one dispatcher.
504
- if (config.strandPeerBook) {
505
- const swap = new StrandPeerBookSwap({
506
- strandId,
507
- libp2p: node,
508
- protocolPrefix,
509
- store: config.strandPeerBook,
510
- privateKey: config.privateKey,
511
- linkRoundTripMs: config.network?.linkRoundTripMs
512
- });
513
- this.peerBookSwaps.set(strandId, swap);
514
- swap.start();
515
- }
516
530
  // One reservation supervisor PER RELAY, started now so the first attempts
517
531
  // overlap the database bring-up below (the relay is never in a strand's
518
532
  // Optimystic cohort — its protocol ids are namespaced `/optimystic/strand-<id>/…`
@@ -663,7 +677,7 @@ export class StrandInstanceManager {
663
677
  // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
664
678
  //
665
679
  // NOTE: a relay that is down costs this launch one full drive — the reservation budget
666
- // counted from the declared link round trip, 14 s at its default (`link-budget.ts`) — and
680
+ // counted from the declared link round trip, 18 s at its default (`link-budget.ts`) — and
667
681
  // `StrandWatcher` launches strands one at a time, so N strands cost N of those in
668
682
  // bring-up during a relay outage, and MORE on a host that declared a slower link. If that
669
683
  // ever matters, stop awaiting here (the circuit addr then lands after `active`) rather
@@ -671,8 +685,9 @@ export class StrandInstanceManager {
671
685
  t0 = performance.now();
672
686
  await this.awaitFirstRelayAttempts(strandId);
673
687
  timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
688
+ // No `lastActivity` stamp: bringing a runtime up is not activity. A check-in marks
689
+ // activity before its resume and must see only what landed while the strand was up.
674
690
  instance.status = liveStrandStatus(instance);
675
- instance.lastActivity = new Date();
676
691
  if (instance.status === 'syncing') {
677
692
  log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
678
693
  'a member of the strand is reached', strandId);
@@ -720,20 +735,6 @@ export class StrandInstanceManager {
720
735
  backfill.stop();
721
736
  this.backfills.delete(instance.strandId);
722
737
  }
723
- // The peer-book observer goes with the node it listened on; a resume re-arms
724
- // one on the rebuilt node.
725
- const peerObserver = this.peerObservers.get(instance.strandId);
726
- if (peerObserver) {
727
- peerObserver.stop();
728
- this.peerObservers.delete(instance.strandId);
729
- }
730
- // The book swap likewise: unsubscribed and its handler unregistered while the node is
731
- // still up; a resume re-arms one, which re-signs the own entry over the new addresses.
732
- const peerBookSwap = this.peerBookSwaps.get(instance.strandId);
733
- if (peerBookSwap) {
734
- this.peerBookSwaps.delete(instance.strandId);
735
- await peerBookSwap.stop();
736
- }
737
738
  // The revoked-peer enforcer goes with the runtime it gated: a resume
738
739
  // rebuilds it with a fresh (initially empty, fail-open) snapshot.
739
740
  const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
@@ -934,23 +935,53 @@ export class StrandInstanceManager {
934
935
  * Quiesce a strand: release its strand-network resources (stop the libp2p node,
935
936
  * close the StrandDatabase) while RETAINING the instance record — identity,
936
937
  * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
937
- * resumed later. Mechanically this is `stopStrand` minus the instance/config
938
- * deletion. The caller sets the post-quiesce status (e.g. `hibernating`).
939
- * No-ops when the strand is missing or already quiesced.
938
+ * resumed later, then mark it `hibernating`. Mechanically this is `stopStrand` minus
939
+ * the instance/config deletion. A tracked strand with no runtime left (a failed
940
+ * rebuild already released it) is only marked; an untracked one is left alone.
941
+ *
942
+ * Ordered with {@link resumeStrand} (see {@link transitions}): it starts once the resume
943
+ * or launch build before it has settled, so it releases a finished runtime, and the
944
+ * status write is its last step, so a resume requested meanwhile writes after it. A call
945
+ * made while another quiesce is the latest request joins it. A failed release leaves the
946
+ * status as it was and rejects.
940
947
  */
941
948
  async quiesceStrand(strandId) {
949
+ const previous = this.transitions.get(strandId);
950
+ if (previous?.kind === 'quiesce') {
951
+ log('quiesceStrand: strand %s — joining the quiesce already requested', strandId);
952
+ return previous.operation;
953
+ }
954
+ const operation = this.runQuiesce(strandId, previous);
955
+ this.holdTransition(strandId, { kind: 'quiesce', operation });
956
+ return operation;
957
+ }
958
+ /** Body of {@link quiesceStrand}. */
959
+ async runQuiesce(strandId, previous) {
960
+ await this.settleTransition(strandId, previous);
961
+ await this.settleRuntimeBuilds(strandId);
942
962
  const instance = this.instances.get(strandId);
943
963
  if (!instance) {
944
964
  log('quiesceStrand: strand %s not found', strandId);
945
965
  return;
946
966
  }
947
- if (!instance.libp2pNode && !instance.database) {
967
+ if (instance.libp2pNode || instance.database) {
968
+ log('Quiescing strand instance: %s', strandId);
969
+ await this.releaseRuntime(instance);
970
+ log('Strand %s quiesced (resources released, instance retained)', strandId);
971
+ }
972
+ else {
948
973
  log('quiesceStrand: strand %s already quiesced', strandId);
949
- return;
950
974
  }
951
- log('Quiescing strand instance: %s', strandId);
952
- await this.releaseRuntime(instance);
953
- log('Strand %s quiesced (resources released, instance retained)', strandId);
975
+ instance.status = 'hibernating';
976
+ }
977
+ /**
978
+ * Whether the latest runtime transition requested for `strandId` is a quiesce that has
979
+ * not finished. Its handles may still be attached and its status still live, but it is on
980
+ * its way to `hibernating`, so a caller that needs it up must {@link resumeStrand} it — the
981
+ * resume runs after the quiesce — rather than treat it as live.
982
+ */
983
+ isQuiescing(strandId) {
984
+ return this.transitions.get(strandId)?.kind === 'quiesce';
954
985
  }
955
986
  /**
956
987
  * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
@@ -959,11 +990,58 @@ export class StrandInstanceManager {
959
990
  * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
960
991
  * count) and updates the retained config so a later resume reuses the latest
961
992
  * values. Returns the live instance unchanged if it is already running.
993
+ *
994
+ * Overlapping calls share one rebuild: a call made while a resume is the latest request
995
+ * for the strand — a wake landing during a check-in — joins it and settles exactly as it
996
+ * does, `'error'` status included. The joiner's own `overrides` are ignored; the first
997
+ * resume's seed wins, and both callers resolved the cohort seed moments apart. A call made
998
+ * while a quiesce is the latest request runs after that quiesce and rebuilds (see
999
+ * {@link transitions}), and one made while `startStrand` is still building waits for that
1000
+ * build rather than starting a second.
962
1001
  */
963
1002
  async resumeStrand(strandId, overrides) {
964
1003
  if (this.stopping) {
965
1004
  throw new Error('StrandInstanceManager is stopping');
966
1005
  }
1006
+ // No `await` between this read and `holdTransition`, or a second caller slips through
1007
+ // and builds.
1008
+ const previous = this.transitions.get(strandId);
1009
+ if (previous?.kind === 'resume') {
1010
+ log('resumeStrand: strand %s — joining the resume already in flight', strandId);
1011
+ return previous.operation;
1012
+ }
1013
+ const operation = this.runResume(strandId, previous, overrides);
1014
+ this.holdTransition(strandId, { kind: 'resume', operation });
1015
+ return operation;
1016
+ }
1017
+ /** Record `transition` as the latest for `strandId` until it settles. */
1018
+ holdTransition(strandId, transition) {
1019
+ this.transitions.set(strandId, transition);
1020
+ const release = () => {
1021
+ if (this.transitions.get(strandId) === transition) {
1022
+ this.transitions.delete(strandId);
1023
+ }
1024
+ };
1025
+ // The rejection itself reaches the caller through the operation returned to it.
1026
+ void transition.operation.then(release, release);
1027
+ }
1028
+ /** Wait for the transition requested before this one. Never rejects: its failure belongs to its caller. */
1029
+ async settleTransition(strandId, previous) {
1030
+ if (!previous) {
1031
+ return;
1032
+ }
1033
+ await previous.operation.catch((error) => {
1034
+ log('Strand %s: the %s queued ahead failed (reported by its caller): %o', strandId, previous.kind, error);
1035
+ });
1036
+ }
1037
+ /** Body of {@link resumeStrand}; at most one runs per strand at a time. */
1038
+ async runResume(strandId, previous, overrides) {
1039
+ // Before any "already live" test: a quiesce ahead still has the handles attached, and
1040
+ // mid-build (a launch still building) the libp2p node is attached while the database is
1041
+ // not. A failed launch has already dropped the instance, and the check below reports it
1042
+ // untracked.
1043
+ await this.settleTransition(strandId, previous);
1044
+ await this.settleRuntimeBuilds(strandId);
967
1045
  const instance = this.instances.get(strandId);
968
1046
  if (!instance) {
969
1047
  throw new Error(`Cannot resume strand ${strandId}: not tracked`);
@@ -997,7 +1075,7 @@ export class StrandInstanceManager {
997
1075
  this.launchConfigs.set(strandId, resumeConfig);
998
1076
  instance.status = 'starting';
999
1077
  try {
1000
- await this.buildStrandRuntime(instance, resumeConfig);
1078
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, resumeConfig));
1001
1079
  timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
1002
1080
  log('Strand %s resumed successfully', strandId);
1003
1081
  return instance;
@@ -1025,7 +1103,8 @@ export class StrandInstanceManager {
1025
1103
  * - `'already-founder'` — the retained config already founds; nothing to do.
1026
1104
  * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
1027
1105
  * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
1028
- * database), so the bootstrap could not run here: the CALLER must wake the
1106
+ * database) or being quiesced (its database is closing), so the bootstrap could
1107
+ * not run here: the CALLER must wake the
1029
1108
  * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
1030
1109
  * manager does not) so the rebuild — which now founds — runs it.
1031
1110
  * @param resolvePartyKey - Asked for the party's own membership key when (and only
@@ -1054,7 +1133,7 @@ export class StrandInstanceManager {
1054
1133
  // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1055
1134
  // config object, which is not ours to rewrite.
1056
1135
  this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
1057
- if (!instance.database && !this.firstSyncGates.has(strandId)) {
1136
+ if (this.isQuiescing(strandId) || (!instance.database && !this.firstSyncGates.has(strandId))) {
1058
1137
  return 'needs-resume';
1059
1138
  }
1060
1139
  try {
@@ -1066,6 +1145,70 @@ export class StrandInstanceManager {
1066
1145
  }
1067
1146
  return 'bootstrapped';
1068
1147
  }
1148
+ /**
1149
+ * Give a tracked storage replica ({@link StartStrandConfig.sAppConfig} absent) the app's
1150
+ * schema in place — the seam for an app on this machine claiming a strand this node was
1151
+ * already hosting. No runtime rebuild: the libp2p node, its peer id and connections, the
1152
+ * store and its warm cache all stay, and the new `App` tables read the blocks the replica
1153
+ * already holds. Callers attach BEFORE {@link foundExistingStrand}: the founder bootstrap
1154
+ * writes the sApp into `Strand.Header` and refuses a database that has none.
1155
+ *
1156
+ * A runtime build in flight (the replica's launch, or a hibernation wake) is waited out
1157
+ * first, so the attach acts on the database that build produced. Then, in order: the
1158
+ * schema signature is checked before anything changes; the retained launch config takes
1159
+ * the sApp, so every later rebuild applies it through `composeStrand`; the live database —
1160
+ * published, or still held by the first-sync gate — gets the schema; and only then does
1161
+ * the instance record `sAppInfo` and the sApp's latency hint. A quiesced instance gets the
1162
+ * config alone, and its next resume applies the schema.
1163
+ *
1164
+ * @returns `'already-attached'` when the instance already runs an sApp — a DIFFERENT one
1165
+ * is logged and left as it is, as a second claim of a claimed strand always has been —
1166
+ * else `'attached'`.
1167
+ * @throws when the strand is not tracked (a caller bug, or a launch that failed while
1168
+ * this waited on it); when the schema signature is refused, having changed nothing; and
1169
+ * whatever the live apply throws (e.g. a quiesce closed the database mid-apply) — the
1170
+ * retained config then still carries the sApp and `sAppInfo` stays unset, so the next
1171
+ * claim retries the apply.
1172
+ *
1173
+ * NOTE: two claims of one replica racing each other both pass the `sAppInfo` check and
1174
+ * apply concurrently; the loser of the declarative diff can reject, and its `addStrand`
1175
+ * retry then resolves `'already-attached'`. Needs an app calling `addStrand` twice at once
1176
+ * (or beside a watcher retry); if it is ever seen, chain attaches per strand id.
1177
+ */
1178
+ async attachSApp(strandId, sAppConfig, options = {}) {
1179
+ await this.settleRuntimeBuilds(strandId);
1180
+ const instance = this.instances.get(strandId);
1181
+ const config = this.launchConfigs.get(strandId);
1182
+ if (!instance || !config) {
1183
+ throw new Error(`Cannot attach an sApp to strand ${strandId}: not tracked`);
1184
+ }
1185
+ if (instance.sAppInfo) {
1186
+ if (instance.sAppInfo.id !== sAppConfig.id) {
1187
+ log('attachSApp: strand %s already runs sApp %s — the claim for sApp %s is ignored', strandId, instance.sAppInfo.id, sAppConfig.id);
1188
+ }
1189
+ return 'already-attached';
1190
+ }
1191
+ const sAppInfo = verifiedSAppInfo(strandId, sAppConfig, options.requireSignedSchemas);
1192
+ // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1193
+ // config object, which is not ours to rewrite.
1194
+ this.launchConfigs.set(strandId, { ...config, sAppConfig });
1195
+ const database = instance.database ?? this.firstSyncGates.get(strandId)?.database;
1196
+ if (database) {
1197
+ await database.attachAppSchema(sAppConfig);
1198
+ }
1199
+ else {
1200
+ log('attachSApp: strand %s is quiesced — its next resume applies the sApp schema', strandId);
1201
+ }
1202
+ instance.sAppInfo = sAppInfo;
1203
+ if (sAppConfig.latencyHint) {
1204
+ // HibernationManager reads the hint whenever it arms a timer, so the app's hint governs
1205
+ // from the next one: a timer already armed runs once at the old duration, and a replica
1206
+ // launched under a realtime default was never tracked, so it stays up until relaunched.
1207
+ instance.latencyHint = sAppConfig.latencyHint;
1208
+ }
1209
+ log('Strand %s upgraded from storage replica to %s', strandId, describeSApp(sAppConfig));
1210
+ return 'attached';
1211
+ }
1069
1212
  /**
1070
1213
  * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
1071
1214
  * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
@@ -1117,6 +1260,17 @@ export class StrandInstanceManager {
1117
1260
  waiters.add(waiter);
1118
1261
  });
1119
1262
  }
1263
+ /**
1264
+ * The tracked instance once no runtime build of it is in flight, or `undefined` when none
1265
+ * is tracked by then — never launched, or the launch waited on failed and dropped its
1266
+ * record. For a caller about to hand a tracked instance to an app: until its build settles
1267
+ * the instance is `'starting'` with no database, and a failed launch leaves no instance.
1268
+ * Never rejects; a failed build is reported by the call that started it.
1269
+ */
1270
+ async whenRuntimeBuilt(strandId) {
1271
+ await this.settleRuntimeBuilds(strandId);
1272
+ return this.instances.get(strandId);
1273
+ }
1120
1274
  /**
1121
1275
  * Hand a launch's database to the app: set `instance.database`, retire the gate that
1122
1276
  * held it, release every {@link whenWritable} waiter, and — when the launch had already
@@ -1146,9 +1300,11 @@ export class StrandInstanceManager {
1146
1300
  this.firstSyncGates.delete(strandId);
1147
1301
  }
1148
1302
  instance.database = database;
1149
- instance.lastActivity = new Date();
1150
1303
  const wasGated = instance.status !== 'starting';
1151
1304
  if (wasGated) {
1305
+ // A Header delivered by a peer is strand activity; a publish during bring-up is not
1306
+ // (see the build's end in `buildStrandRuntime`).
1307
+ instance.lastActivity = new Date();
1152
1308
  instance.status = 'active';
1153
1309
  log('Strand %s is now writable: Strand.Header held', strandId);
1154
1310
  }