@serfab/cadre-core 1.6.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/README.md +8 -6
  2. package/dist/cadre-node.d.ts +548 -137
  3. package/dist/cadre-node.js +1208 -383
  4. package/dist/cadre-node.js.map +1 -1
  5. package/dist/control-authorization.d.ts +2 -2
  6. package/dist/control-authorization.js +1 -0
  7. package/dist/control-authorization.js.map +1 -1
  8. package/dist/control-cohort.d.ts +6 -0
  9. package/dist/control-cohort.js +6 -0
  10. package/dist/control-cohort.js.map +1 -1
  11. package/dist/control-database.d.ts +46 -8
  12. package/dist/control-database.js +97 -18
  13. package/dist/control-database.js.map +1 -1
  14. package/dist/control-formation-recorder.d.ts +37 -14
  15. package/dist/control-formation-recorder.js +60 -10
  16. package/dist/control-formation-recorder.js.map +1 -1
  17. package/dist/control-read-retry.d.ts +13 -12
  18. package/dist/control-read-retry.js +14 -12
  19. package/dist/control-read-retry.js.map +1 -1
  20. package/dist/control-schema.d.ts +1 -1
  21. package/dist/control-schema.js +67 -5
  22. package/dist/control-schema.js.map +1 -1
  23. package/dist/control-stream.d.ts +9 -1
  24. package/dist/control-stream.js +24 -1
  25. package/dist/control-stream.js.map +1 -1
  26. package/dist/control-write-retry.d.ts +12 -2
  27. package/dist/control-write-retry.js +13 -2
  28. package/dist/control-write-retry.js.map +1 -1
  29. package/dist/delegate-admission.d.ts +11 -8
  30. package/dist/delegate-admission.js +19 -11
  31. package/dist/delegate-admission.js.map +1 -1
  32. package/dist/diagnostics/connection-path.d.ts +0 -7
  33. package/dist/diagnostics/connection-path.js +0 -7
  34. package/dist/diagnostics/connection-path.js.map +1 -1
  35. package/dist/formation-approval.d.ts +7 -0
  36. package/dist/formation-approval.js +9 -3
  37. package/dist/formation-approval.js.map +1 -1
  38. package/dist/fs-atomic.d.ts +11 -2
  39. package/dist/fs-atomic.js +13 -3
  40. package/dist/fs-atomic.js.map +1 -1
  41. package/dist/hibernation-manager.d.ts +111 -8
  42. package/dist/hibernation-manager.js +234 -49
  43. package/dist/hibernation-manager.js.map +1 -1
  44. package/dist/index.d.ts +13 -6
  45. package/dist/index.js +27 -5
  46. package/dist/index.js.map +1 -1
  47. package/dist/joined-strand-store.d.ts +177 -0
  48. package/dist/joined-strand-store.js +431 -0
  49. package/dist/joined-strand-store.js.map +1 -0
  50. package/dist/key-store-file.js +14 -5
  51. package/dist/key-store-file.js.map +1 -1
  52. package/dist/key-store.d.ts +6 -0
  53. package/dist/key-store.js.map +1 -1
  54. package/dist/link-budget.d.ts +367 -69
  55. package/dist/link-budget.js +375 -72
  56. package/dist/link-budget.js.map +1 -1
  57. package/dist/membership-connection-gater.d.ts +30 -43
  58. package/dist/membership-connection-gater.js +31 -42
  59. package/dist/membership-connection-gater.js.map +1 -1
  60. package/dist/node-local-snapshot.d.ts +5 -4
  61. package/dist/node-local-snapshot.js +6 -3
  62. package/dist/node-local-snapshot.js.map +1 -1
  63. package/dist/peer-dial.d.ts +24 -4
  64. package/dist/peer-dial.js +74 -5
  65. package/dist/peer-dial.js.map +1 -1
  66. package/dist/peer-join-backfill.d.ts +35 -15
  67. package/dist/peer-join-backfill.js +115 -55
  68. package/dist/peer-join-backfill.js.map +1 -1
  69. package/dist/push-fanout.d.ts +6 -0
  70. package/dist/push-fanout.js +6 -0
  71. package/dist/push-fanout.js.map +1 -1
  72. package/dist/relay-reservation.d.ts +3 -1
  73. package/dist/relay-reservation.js +3 -1
  74. package/dist/relay-reservation.js.map +1 -1
  75. package/dist/relay-server.d.ts +79 -0
  76. package/dist/relay-server.js +74 -0
  77. package/dist/relay-server.js.map +1 -0
  78. package/dist/seed-bootstrap.d.ts +57 -4
  79. package/dist/seed-bootstrap.js +107 -83
  80. package/dist/seed-bootstrap.js.map +1 -1
  81. package/dist/storage-scope.d.ts +33 -17
  82. package/dist/storage-scope.js +44 -23
  83. package/dist/storage-scope.js.map +1 -1
  84. package/dist/strand-addr-protocol.d.ts +58 -21
  85. package/dist/strand-addr-protocol.js +133 -85
  86. package/dist/strand-addr-protocol.js.map +1 -1
  87. package/dist/strand-database.d.ts +20 -2
  88. package/dist/strand-database.js +42 -5
  89. package/dist/strand-database.js.map +1 -1
  90. package/dist/strand-first-sync-gate.d.ts +59 -21
  91. package/dist/strand-first-sync-gate.js +58 -19
  92. package/dist/strand-first-sync-gate.js.map +1 -1
  93. package/dist/strand-formation-deadlines.d.ts +151 -0
  94. package/dist/strand-formation-deadlines.js +171 -0
  95. package/dist/strand-formation-deadlines.js.map +1 -0
  96. package/dist/strand-formation-manager.d.ts +63 -30
  97. package/dist/strand-formation-manager.js +67 -41
  98. package/dist/strand-formation-manager.js.map +1 -1
  99. package/dist/strand-formation-protocol.d.ts +45 -35
  100. package/dist/strand-formation-protocol.js +59 -122
  101. package/dist/strand-formation-protocol.js.map +1 -1
  102. package/dist/strand-instance-manager.d.ts +152 -6
  103. package/dist/strand-instance-manager.js +331 -42
  104. package/dist/strand-instance-manager.js.map +1 -1
  105. package/dist/strand-membership-writer.d.ts +6 -0
  106. package/dist/strand-membership-writer.js +11 -0
  107. package/dist/strand-membership-writer.js.map +1 -1
  108. package/dist/strand-peer-book-file.d.ts +12 -0
  109. package/dist/strand-peer-book-file.js +37 -0
  110. package/dist/strand-peer-book-file.js.map +1 -0
  111. package/dist/strand-peer-book-protocol.d.ts +168 -0
  112. package/dist/strand-peer-book-protocol.js +356 -0
  113. package/dist/strand-peer-book-protocol.js.map +1 -0
  114. package/dist/strand-peer-book-swap.d.ts +104 -0
  115. package/dist/strand-peer-book-swap.js +284 -0
  116. package/dist/strand-peer-book-swap.js.map +1 -0
  117. package/dist/strand-peer-book.d.ts +159 -0
  118. package/dist/strand-peer-book.js +361 -0
  119. package/dist/strand-peer-book.js.map +1 -0
  120. package/dist/strand-peer-observer.d.ts +88 -0
  121. package/dist/strand-peer-observer.js +180 -0
  122. package/dist/strand-peer-observer.js.map +1 -0
  123. package/dist/strand-revocation-enforcer.js +2 -1
  124. package/dist/strand-revocation-enforcer.js.map +1 -1
  125. package/dist/strand-solicitation.d.ts +36 -20
  126. package/dist/strand-solicitation.js +4 -4
  127. package/dist/strand-solicitation.js.map +1 -1
  128. package/dist/strand-wake-protocol.d.ts +80 -23
  129. package/dist/strand-wake-protocol.js +89 -64
  130. package/dist/strand-wake-protocol.js.map +1 -1
  131. package/dist/types.d.ts +181 -33
  132. package/dist/types.js +4 -3
  133. package/dist/types.js.map +1 -1
  134. package/package.json +11 -7
  135. package/src/cadre-node.ts +1316 -398
  136. package/src/control-authorization.ts +2 -1
  137. package/src/control-cohort.ts +6 -0
  138. package/src/control-database.ts +122 -18
  139. package/src/control-formation-recorder.ts +354 -305
  140. package/src/control-read-retry.ts +14 -12
  141. package/src/control-schema.ts +67 -5
  142. package/src/control-stream.ts +25 -1
  143. package/src/control-write-retry.ts +13 -2
  144. package/src/delegate-admission.ts +23 -13
  145. package/src/diagnostics/connection-path.ts +0 -7
  146. package/src/formation-approval.ts +610 -604
  147. package/src/fs-atomic.ts +14 -4
  148. package/src/hibernation-manager.ts +638 -437
  149. package/src/index.ts +91 -6
  150. package/src/joined-strand-store.ts +498 -0
  151. package/src/key-store-file.ts +14 -5
  152. package/src/key-store.ts +6 -0
  153. package/src/link-budget.ts +393 -74
  154. package/src/membership-connection-gater.ts +31 -45
  155. package/src/node-local-snapshot.ts +293 -289
  156. package/src/peer-dial.ts +81 -7
  157. package/src/peer-join-backfill.ts +120 -57
  158. package/src/push-fanout.ts +6 -0
  159. package/src/relay-reservation.ts +3 -1
  160. package/src/relay-server.ts +98 -0
  161. package/src/seed-bootstrap.ts +133 -91
  162. package/src/storage-scope.ts +44 -23
  163. package/src/strand-addr-protocol.ts +547 -458
  164. package/src/strand-database.ts +287 -242
  165. package/src/strand-first-sync-gate.ts +58 -19
  166. package/src/strand-formation-deadlines.ts +227 -0
  167. package/src/strand-formation-manager.ts +700 -647
  168. package/src/strand-formation-protocol.ts +103 -150
  169. package/src/strand-instance-manager.ts +387 -45
  170. package/src/strand-membership-writer.ts +12 -0
  171. package/src/strand-peer-book-file.ts +39 -0
  172. package/src/strand-peer-book-protocol.ts +466 -0
  173. package/src/strand-peer-book-swap.ts +338 -0
  174. package/src/strand-peer-book.ts +475 -0
  175. package/src/strand-peer-observer.ts +235 -0
  176. package/src/strand-revocation-enforcer.ts +2 -1
  177. package/src/strand-solicitation.ts +519 -500
  178. package/src/strand-wake-protocol.ts +401 -355
  179. package/src/types.ts +185 -33
@@ -3,6 +3,8 @@ import { createLibp2pNode } from '@optimystic/db-p2p';
3
3
  import { wrapStorageWithCache, disposeStorageCache } from '@serfab/quereus-plugin-sereus';
4
4
  import { StrandDatabase } from './strand-database.js';
5
5
  import { PeerJoinBackfill } from './peer-join-backfill.js';
6
+ import { StrandPeerObserver } from './strand-peer-observer.js';
7
+ import { StrandPeerBookSwap } from './strand-peer-book-swap.js';
6
8
  import { StrandRevocationEnforcer, createRevocationConnectionGater, readStrandRevocationRows } from './strand-revocation-enforcer.js';
7
9
  import { StrandMembershipReconciler } from './strand-membership-reconciler.js';
8
10
  import { removeMemberPeer } from './strand-membership-writer.js';
@@ -12,8 +14,9 @@ import { assertStrandScopeKey } from './storage-scope.js';
12
14
  import { StrandFirstSyncGate, StrandAwaitingFirstSyncError, strandFirstSyncComplete, DEFAULT_STRAND_FIRST_SYNC_TIMEOUT_MS } from './strand-first-sync-gate.js';
13
15
  import { DEFAULT_CONNECTION_MONITOR, resolveStrandClusterSize, strandClusterPolicy } from './types.js';
14
16
  import { strandNodeAddrs } from './strand-network-config.js';
17
+ import { resolveRelayServer } from './relay-server.js';
15
18
  import { superviseRelayReservation } from './relay-reservation.js';
16
- import { connectionManagerTimeouts, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
19
+ import { declaredCohortReadDeadlineMs, peerJoinPushBudget, relayReservationBudgetMs, resolveLinkRoundTripMs } from './link-budget.js';
17
20
  const log = debug('sereus:cadre:strand-manager');
18
21
  const timing = debug('sereus:cadre:timing');
19
22
  /**
@@ -97,6 +100,24 @@ function resolveStrandStorage(provider, strandId) {
97
100
  // lifetime — so the wrap survives every runtime rebuild the instance goes through.
98
101
  return wrapStorageWithCache(storage, strandId);
99
102
  }
103
+ /**
104
+ * Verify an sApp's schema signature (fail-closed unless `requireSignedSchemas` is
105
+ * false) and project the config onto the instance's `sAppInfo`.
106
+ */
107
+ function verifiedSAppInfo(strandId, sAppConfig, requireSignedSchemas) {
108
+ assertSchemaSignature(sAppConfig, { requireSignature: requireSignedSchemas ?? true });
109
+ log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
110
+ return {
111
+ id: sAppConfig.id,
112
+ version: sAppConfig.version,
113
+ schema: sAppConfig.schema,
114
+ signature: sAppConfig.signature
115
+ };
116
+ }
117
+ /** Log wording for what a launch runs: the sApp and its version, or a storage replica. */
118
+ function describeSApp(sAppConfig) {
119
+ return sAppConfig ? `sApp: ${sAppConfig.id} v${sAppConfig.version}` : 'storage replica';
120
+ }
100
121
  /**
101
122
  * Manages individual strand instances - creates and destroys isolated libp2p nodes
102
123
  * for each strand the cadre participates in.
@@ -119,6 +140,23 @@ export class StrandInstanceManager {
119
140
  * missed writes).
120
141
  */
121
142
  this.backfills = new Map();
143
+ /**
144
+ * The per-strand peer-book observation writer (`strand-peer-observer.ts`), keyed
145
+ * by strand id. Same lifecycle rationale as {@link backfills}: started in
146
+ * `buildStrandRuntime` right after the libp2p node exists, stopped and dropped in
147
+ * `releaseRuntime`, so quiesce → resume re-arms it on the rebuilt node. Only
148
+ * present when the launch config supplied `onStrandPeerIdentified`.
149
+ */
150
+ this.peerObservers = new Map();
151
+ /**
152
+ * Per-strand strand peer book SWAP driver ({@link StrandPeerBookSwap}) — the signed
153
+ * exchange of book entries between strand peers. Same lifetime as the peer-book
154
+ * observer above: armed in `buildStrandRuntime` right after the libp2p node exists
155
+ * (its protocol handler must be registered before an early peer dials it), stopped
156
+ * and dropped in `releaseRuntime`. Only present when the launch config supplied
157
+ * `strandPeerBook`.
158
+ */
159
+ this.peerBookSwaps = new Map();
122
160
  /**
123
161
  * The per-strand revoked-peer enforcer (closed strands only), keyed by strand
124
162
  * id. Same lifecycle rationale as {@link backfills}: created in
@@ -185,6 +223,23 @@ export class StrandInstanceManager {
185
223
  * wake becomes the better trade.
186
224
  */
187
225
  this.strandStorages = new Map();
226
+ /**
227
+ * The runtime build in flight per strand id (`buildStrandRuntime`, from `startStrand` or
228
+ * `resumeStrand`), present until it settles. The instance is tracked for the whole build,
229
+ * with its database not yet constructed or not yet initialized, so a caller that must act
230
+ * on the FINISHED runtime ({@link attachSApp}, {@link whenRuntimeBuilt}) waits on this instead. A failed build's
231
+ * rejection belongs to the call that started it.
232
+ */
233
+ this.runtimeBuilds = new Map();
234
+ /**
235
+ * The latest resume or quiesce requested per strand id, present until it settles. Each one
236
+ * starts only once the one requested before it has settled, so a wake that lands during a
237
+ * quiesce rebuilds after it, and a quiesce requested during a rebuild releases what that
238
+ * rebuild finished. The WHOLE operation, not its entry in {@link runtimeBuilds}: that settles
239
+ * before a resume's own catch records `'error'`, so a joiner awaiting it could read a stale
240
+ * status.
241
+ */
242
+ this.transitions = new Map();
188
243
  this.stopping = false;
189
244
  log('StrandInstanceManager created');
190
245
  }
@@ -240,7 +295,7 @@ export class StrandInstanceManager {
240
295
  log('Strand %s already running', strandId);
241
296
  return this.instances.get(strandId);
242
297
  }
243
- log('Starting strand instance: %s (sApp: %s v%s)', strandId, sAppConfig.id, sAppConfig.version);
298
+ log('Starting strand instance: %s (%s)', strandId, describeSApp(sAppConfig));
244
299
  const tTotal = performance.now();
245
300
  // The id becomes two names below: the storage scope key the embedder's provider
246
301
  // turns into a directory or database name, and `networkName` in buildStrandRuntime,
@@ -250,17 +305,7 @@ export class StrandInstanceManager {
250
305
  // function returns early when no provider is configured — a node with no storage
251
306
  // still reaches buildStrandRuntime and still mints the protocol prefix.
252
307
  assertStrandScopeKey(strandId);
253
- // Verify schema signature before proceeding (fail-closed by default)
254
- const requireSignature = config.requireSignedSchemas ?? true;
255
- assertSchemaSignature(sAppConfig, { requireSignature });
256
- log('Strand %s sApp schema signature verified (author: %s)', strandId, sAppConfig.id);
257
- // Convert SAppConfig to SAppInfo for the instance
258
- const sAppInfo = {
259
- id: sAppConfig.id,
260
- version: sAppConfig.version,
261
- schema: sAppConfig.schema,
262
- signature: sAppConfig.signature
263
- };
308
+ const sAppInfo = sAppConfig ? verifiedSAppInfo(strandId, sAppConfig, config.requireSignedSchemas) : undefined;
264
309
  // Resolve this strand's storage ONCE, before anything is recorded. If a factory
265
310
  // function is provided, it is called with the strandId to create strand-specific
266
311
  // storage (e.g. strand-isolated directories). A provider that throws therefore
@@ -270,7 +315,7 @@ export class StrandInstanceManager {
270
315
  log('Strand %s using provided storage provider', strandId);
271
316
  }
272
317
  // Determine latency hint: sApp config > default
273
- const latencyHint = sAppConfig.latencyHint ?? config.defaultLatencyHint;
318
+ const latencyHint = sAppConfig?.latencyHint ?? config.defaultLatencyHint;
274
319
  const instance = {
275
320
  strandId,
276
321
  status: 'starting',
@@ -286,9 +331,9 @@ export class StrandInstanceManager {
286
331
  this.strandStorages.set(strandId, strandStorage);
287
332
  }
288
333
  try {
289
- await this.buildStrandRuntime(instance, config);
334
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, config));
290
335
  timing('[startStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
291
- log('Strand %s started successfully with sApp %s', strandId, sAppConfig.id);
336
+ log('Strand %s started successfully (%s)', strandId, describeSApp(sAppConfig));
292
337
  return instance;
293
338
  }
294
339
  catch (error) {
@@ -307,6 +352,34 @@ export class StrandInstanceManager {
307
352
  throw error;
308
353
  }
309
354
  }
355
+ /** Hold `build` in {@link runtimeBuilds} until it settles, and settle as it does. */
356
+ async trackRuntimeBuild(strandId, build) {
357
+ this.runtimeBuilds.set(strandId, build);
358
+ try {
359
+ await build;
360
+ }
361
+ finally {
362
+ if (this.runtimeBuilds.get(strandId) === build) {
363
+ this.runtimeBuilds.delete(strandId);
364
+ }
365
+ }
366
+ }
367
+ /**
368
+ * Wait until no runtime build of `strandId` is in flight. Never rejects: a failed build
369
+ * is reported by the call that started it, and the caller re-reads the instance after.
370
+ *
371
+ * NOTE: after a failed launch the re-read sees the instance gone only because
372
+ * `startStrand`'s catch deletes it synchronously, and its continuation is queued before
373
+ * this waiter's. If that cleanup ever awaits before `instances.delete`, a waiter can act
374
+ * on the dying record; move the cleanup inside the tracked promise then.
375
+ */
376
+ async settleRuntimeBuilds(strandId) {
377
+ for (let build = this.runtimeBuilds.get(strandId); build; build = this.runtimeBuilds.get(strandId)) {
378
+ await build.catch((error) => {
379
+ log('Strand %s: the runtime build waited on failed (reported by its launcher): %o', strandId, error);
380
+ });
381
+ }
382
+ }
310
383
  /**
311
384
  * Build (or rebuild) the libp2p node + StrandDatabase for an instance and
312
385
  * attach them, transitioning it to `active`. Shared by `startStrand` (fresh
@@ -334,9 +407,9 @@ export class StrandInstanceManager {
334
407
  // with — hence one binding for both, not two literals that can drift.
335
408
  const networkName = `strand-${strandId}`;
336
409
  const protocolPrefix = `/optimystic/${networkName}`;
337
- // Determine relay mode: if explicitly set in config, use that;
338
- // otherwise default to true for storage profile nodes.
339
- const enableRelay = config.network?.enableRelay ?? (config.profile === 'storage');
410
+ // Whether this node runs the circuit-relay server, and its init — the same
411
+ // resolution the control node takes from the same config (`relay-server.ts`).
412
+ const relayServer = resolveRelayServer(config.network, config.profile);
340
413
  // The strand-node VIEW of the machine's one `NetworkConfig`, not the control
341
414
  // node's resolution: fixed direct listen ports become ephemeral (two nodes cannot
342
415
  // bind one port), the announce config is dropped (it names the control node's
@@ -404,7 +477,12 @@ export class StrandInstanceManager {
404
477
  networkName,
405
478
  storage: strandStorage,
406
479
  fretProfile: config.profile === 'storage' ? 'core' : 'edge',
407
- relay: enableRelay,
480
+ relay: relayServer.enabled,
481
+ // NOTE: unlike the control node, a strand node has no unauthorized-reservation budget,
482
+ // so with the per-connection cap off any peer that reaches this node can hold one of
483
+ // `maxReservations` slots and forward uncapped through it. If strand relays are ever
484
+ // abused for bandwidth, give strand nodes a budget or a capped init of their own.
485
+ ...(relayServer.enabled && { relayServerInit: relayServer.init }),
408
486
  clusterSize: strandClusterSize,
409
487
  // Deliberately NOT CONTROL_CLUSTER_POLICY: a strand is application data with its own
410
488
  // breadth reasoning, and the shape match with the control policy is a coincidence.
@@ -416,11 +494,12 @@ export class StrandInstanceManager {
416
494
  // returns the frozen STRAND_CLUSTER_POLICY itself, declaring nothing. Resolved
417
495
  // HERE rather than at `startStrand`, so a wake from hibernation would pick up a
418
496
  // serving set that changed while the strand slept.
419
- // The read deadline is the same field the control node reads, because the two
420
- // networks ride one link; absent, the base policy's COHORT_READ_DEADLINE_MS stands.
497
+ // The read deadline is settled by the same helper the control node uses, because the
498
+ // two networks ride one link: the host's own, else derived from its declared
499
+ // `linkRoundTripMs`, else nothing, and the base policy's COHORT_READ_DEADLINE_MS stands.
421
500
  clusterPolicy: strandClusterPolicy(strandClusterSize, {
422
501
  servingMachines: config.servingMachines,
423
- cohortQueryTimeoutMs: config.network?.cohortQueryTimeoutMs
502
+ cohortQueryTimeoutMs: declaredCohortReadDeadlineMs(config.network)
424
503
  }),
425
504
  arachnode: {
426
505
  enableRingZulu: config.profile === 'storage'
@@ -433,10 +512,10 @@ export class StrandInstanceManager {
433
512
  // widened ping deadline has to reach the strand nodes too (see
434
513
  // DEFAULT_CONNECTION_MONITOR).
435
514
  connectionMonitor: config.network?.connectionMonitor ?? DEFAULT_CONNECTION_MONITOR,
436
- // The same declared-link limits the control node takes, and for the same reason: a
437
- // strand node is the listener for every other member's strand node (`link-budget.ts`,
438
- // `connectionManagerTimeouts`).
439
- connectionManager: connectionManagerTimeouts(config.network?.linkRoundTripMs),
515
+ // The same declared link the control node states, and for the same reason: a strand
516
+ // node is also the listener for every other member's strand node, and Optimystic
517
+ // derives that listener's limit from this (`link-budget.ts`).
518
+ linkRoundTripMs: resolveLinkRoundTripMs(config.network?.linkRoundTripMs),
440
519
  // Listen entries plus the WebSocket transport switch they imply — a strand node
441
520
  // announces nothing the operator configured (`strand-network-config.ts`), and
442
521
  // spreads AFTER `transports` above because the switch is a no-op whenever the
@@ -454,6 +533,46 @@ export class StrandInstanceManager {
454
533
  });
455
534
  timing('[buildStrandRuntime:%s] createLibp2pNode: %dms', strandId, Math.round(performance.now() - t0));
456
535
  instance.libp2pNode = node;
536
+ // The peer-book observation writer, armed BEFORE the database bring-up below
537
+ // (unlike the backfill) because the bootstrap dials that follow `libp2p.start()`
538
+ // land during that bring-up, and they are exactly the peers worth remembering:
539
+ // identify for an early peer may already have fired by the time anything after
540
+ // the database initializes runs. Registered in the map immediately so the
541
+ // failure rollback (releaseRuntime) stops it like every other runtime component.
542
+ const onStrandPeerIdentified = config.onStrandPeerIdentified;
543
+ if (onStrandPeerIdentified) {
544
+ const observer = new StrandPeerObserver({
545
+ label: strandId,
546
+ libp2p: node,
547
+ protocolPrefix,
548
+ onObserved: (observation) => onStrandPeerIdentified(strandId, observation)
549
+ });
550
+ this.peerObservers.set(strandId, observer);
551
+ observer.start();
552
+ }
553
+ // The strand peer book SWAP, armed at the same moment and for the same reason as
554
+ // the observer: a peer that dials this node's swap handler before it is registered
555
+ // fails its exchange and sits out a ten-minute throttle. The own entry is signed
556
+ // now and re-signed on every address change, which covers the relay reservation
557
+ // landing below (`self:peer:update` fires when the circuit addr is added), so no
558
+ // separate post-relay step is needed. Registered in the map immediately so the
559
+ // failure rollback (releaseRuntime) stops it like every other runtime component.
560
+ //
561
+ // NOTE: this is a third `peer:identify` listener per running strand, beside the
562
+ // observer's and the backfill's — see the backfill's NOTE below for when to fold
563
+ // them into one dispatcher.
564
+ if (config.strandPeerBook) {
565
+ const swap = new StrandPeerBookSwap({
566
+ strandId,
567
+ libp2p: node,
568
+ protocolPrefix,
569
+ store: config.strandPeerBook,
570
+ privateKey: config.privateKey,
571
+ linkRoundTripMs: config.network?.linkRoundTripMs
572
+ });
573
+ this.peerBookSwaps.set(strandId, swap);
574
+ swap.start();
575
+ }
457
576
  // One reservation supervisor PER RELAY, started now so the first attempts
458
577
  // overlap the database bring-up below (the relay is never in a strand's
459
578
  // Optimystic cohort — its protocol ids are namespaced `/optimystic/strand-<id>/…`
@@ -566,7 +685,7 @@ export class StrandInstanceManager {
566
685
  // peer-join-backfill.ts for the strand-side argument (and why the control
567
686
  // network, which DOES gate, is different).
568
687
  //
569
- // NOTE: cost is one PeerJoinBackfill object + one `connection:open` listener
688
+ // NOTE: cost is one PeerJoinBackfill object + one `peer:identify` listener
570
689
  // per running strand — linear in strand count, negligible at the handful a
571
690
  // device or host runs today. If a node ever hosts strands by the hundred,
572
691
  // move to one shared listener that dispatches by strand id.
@@ -604,7 +723,7 @@ export class StrandInstanceManager {
604
723
  // failing here would only trade that for `StrandWatcher`'s full-rebuild retry.
605
724
  //
606
725
  // NOTE: a relay that is down costs this launch one full drive — the reservation budget
607
- // counted from the declared link round trip, 14 s at its default (`link-budget.ts`) — and
726
+ // counted from the declared link round trip, 18 s at its default (`link-budget.ts`) — and
608
727
  // `StrandWatcher` launches strands one at a time, so N strands cost N of those in
609
728
  // bring-up during a relay outage, and MORE on a host that declared a slower link. If that
610
729
  // ever matters, stop awaiting here (the circuit addr then lands after `active`) rather
@@ -612,8 +731,9 @@ export class StrandInstanceManager {
612
731
  t0 = performance.now();
613
732
  await this.awaitFirstRelayAttempts(strandId);
614
733
  timing('[buildStrandRuntime:%s] relay first attempts: %dms', strandId, Math.round(performance.now() - t0));
734
+ // No `lastActivity` stamp: bringing a runtime up is not activity. A check-in marks
735
+ // activity before its resume and must see only what landed while the strand was up.
615
736
  instance.status = liveStrandStatus(instance);
616
- instance.lastActivity = new Date();
617
737
  if (instance.status === 'syncing') {
618
738
  log('Strand %s launched as a joiner with no Strand.Header held yet — writes are withheld until ' +
619
739
  'a member of the strand is reached', strandId);
@@ -661,6 +781,20 @@ export class StrandInstanceManager {
661
781
  backfill.stop();
662
782
  this.backfills.delete(instance.strandId);
663
783
  }
784
+ // The peer-book observer goes with the node it listened on; a resume re-arms
785
+ // one on the rebuilt node.
786
+ const peerObserver = this.peerObservers.get(instance.strandId);
787
+ if (peerObserver) {
788
+ peerObserver.stop();
789
+ this.peerObservers.delete(instance.strandId);
790
+ }
791
+ // The book swap likewise: unsubscribed and its handler unregistered while the node is
792
+ // still up; a resume re-arms one, which re-signs the own entry over the new addresses.
793
+ const peerBookSwap = this.peerBookSwaps.get(instance.strandId);
794
+ if (peerBookSwap) {
795
+ this.peerBookSwaps.delete(instance.strandId);
796
+ await peerBookSwap.stop();
797
+ }
664
798
  // The revoked-peer enforcer goes with the runtime it gated: a resume
665
799
  // rebuilds it with a fresh (initially empty, fail-open) snapshot.
666
800
  const revocationEnforcer = this.revocationEnforcers.get(instance.strandId);
@@ -861,23 +995,53 @@ export class StrandInstanceManager {
861
995
  * Quiesce a strand: release its strand-network resources (stop the libp2p node,
862
996
  * close the StrandDatabase) while RETAINING the instance record — identity,
863
997
  * sAppInfo, keys, latency hint, metadata — and its launch config so it can be
864
- * resumed later. Mechanically this is `stopStrand` minus the instance/config
865
- * deletion. The caller sets the post-quiesce status (e.g. `hibernating`).
866
- * No-ops when the strand is missing or already quiesced.
998
+ * resumed later, then mark it `hibernating`. Mechanically this is `stopStrand` minus
999
+ * the instance/config deletion. A tracked strand with no runtime left (a failed
1000
+ * rebuild already released it) is only marked; an untracked one is left alone.
1001
+ *
1002
+ * Ordered with {@link resumeStrand} (see {@link transitions}): it starts once the resume
1003
+ * or launch build before it has settled, so it releases a finished runtime, and the
1004
+ * status write is its last step, so a resume requested meanwhile writes after it. A call
1005
+ * made while another quiesce is the latest request joins it. A failed release leaves the
1006
+ * status as it was and rejects.
867
1007
  */
868
1008
  async quiesceStrand(strandId) {
1009
+ const previous = this.transitions.get(strandId);
1010
+ if (previous?.kind === 'quiesce') {
1011
+ log('quiesceStrand: strand %s — joining the quiesce already requested', strandId);
1012
+ return previous.operation;
1013
+ }
1014
+ const operation = this.runQuiesce(strandId, previous);
1015
+ this.holdTransition(strandId, { kind: 'quiesce', operation });
1016
+ return operation;
1017
+ }
1018
+ /** Body of {@link quiesceStrand}. */
1019
+ async runQuiesce(strandId, previous) {
1020
+ await this.settleTransition(strandId, previous);
1021
+ await this.settleRuntimeBuilds(strandId);
869
1022
  const instance = this.instances.get(strandId);
870
1023
  if (!instance) {
871
1024
  log('quiesceStrand: strand %s not found', strandId);
872
1025
  return;
873
1026
  }
874
- if (!instance.libp2pNode && !instance.database) {
1027
+ if (instance.libp2pNode || instance.database) {
1028
+ log('Quiescing strand instance: %s', strandId);
1029
+ await this.releaseRuntime(instance);
1030
+ log('Strand %s quiesced (resources released, instance retained)', strandId);
1031
+ }
1032
+ else {
875
1033
  log('quiesceStrand: strand %s already quiesced', strandId);
876
- return;
877
1034
  }
878
- log('Quiescing strand instance: %s', strandId);
879
- await this.releaseRuntime(instance);
880
- log('Strand %s quiesced (resources released, instance retained)', strandId);
1035
+ instance.status = 'hibernating';
1036
+ }
1037
+ /**
1038
+ * Whether the latest runtime transition requested for `strandId` is a quiesce that has
1039
+ * not finished. Its handles may still be attached and its status still live, but it is on
1040
+ * its way to `hibernating`, so a caller that needs it up must {@link resumeStrand} it — the
1041
+ * resume runs after the quiesce — rather than treat it as live.
1042
+ */
1043
+ isQuiescing(strandId) {
1044
+ return this.transitions.get(strandId)?.kind === 'quiesce';
881
1045
  }
882
1046
  /**
883
1047
  * Resume a previously-quiesced strand: rebuild its libp2p node + StrandDatabase
@@ -886,11 +1050,58 @@ export class StrandInstanceManager {
886
1050
  * launch (the cohort `bootstrapNodes` seed and the strand's `servingMachines`
887
1051
  * count) and updates the retained config so a later resume reuses the latest
888
1052
  * values. Returns the live instance unchanged if it is already running.
1053
+ *
1054
+ * Overlapping calls share one rebuild: a call made while a resume is the latest request
1055
+ * for the strand — a wake landing during a check-in — joins it and settles exactly as it
1056
+ * does, `'error'` status included. The joiner's own `overrides` are ignored; the first
1057
+ * resume's seed wins, and both callers resolved the cohort seed moments apart. A call made
1058
+ * while a quiesce is the latest request runs after that quiesce and rebuilds (see
1059
+ * {@link transitions}), and one made while `startStrand` is still building waits for that
1060
+ * build rather than starting a second.
889
1061
  */
890
1062
  async resumeStrand(strandId, overrides) {
891
1063
  if (this.stopping) {
892
1064
  throw new Error('StrandInstanceManager is stopping');
893
1065
  }
1066
+ // No `await` between this read and `holdTransition`, or a second caller slips through
1067
+ // and builds.
1068
+ const previous = this.transitions.get(strandId);
1069
+ if (previous?.kind === 'resume') {
1070
+ log('resumeStrand: strand %s — joining the resume already in flight', strandId);
1071
+ return previous.operation;
1072
+ }
1073
+ const operation = this.runResume(strandId, previous, overrides);
1074
+ this.holdTransition(strandId, { kind: 'resume', operation });
1075
+ return operation;
1076
+ }
1077
+ /** Record `transition` as the latest for `strandId` until it settles. */
1078
+ holdTransition(strandId, transition) {
1079
+ this.transitions.set(strandId, transition);
1080
+ const release = () => {
1081
+ if (this.transitions.get(strandId) === transition) {
1082
+ this.transitions.delete(strandId);
1083
+ }
1084
+ };
1085
+ // The rejection itself reaches the caller through the operation returned to it.
1086
+ void transition.operation.then(release, release);
1087
+ }
1088
+ /** Wait for the transition requested before this one. Never rejects: its failure belongs to its caller. */
1089
+ async settleTransition(strandId, previous) {
1090
+ if (!previous) {
1091
+ return;
1092
+ }
1093
+ await previous.operation.catch((error) => {
1094
+ log('Strand %s: the %s queued ahead failed (reported by its caller): %o', strandId, previous.kind, error);
1095
+ });
1096
+ }
1097
+ /** Body of {@link resumeStrand}; at most one runs per strand at a time. */
1098
+ async runResume(strandId, previous, overrides) {
1099
+ // Before any "already live" test: a quiesce ahead still has the handles attached, and
1100
+ // mid-build (a launch still building) the libp2p node is attached while the database is
1101
+ // not. A failed launch has already dropped the instance, and the check below reports it
1102
+ // untracked.
1103
+ await this.settleTransition(strandId, previous);
1104
+ await this.settleRuntimeBuilds(strandId);
894
1105
  const instance = this.instances.get(strandId);
895
1106
  if (!instance) {
896
1107
  throw new Error(`Cannot resume strand ${strandId}: not tracked`);
@@ -924,7 +1135,7 @@ export class StrandInstanceManager {
924
1135
  this.launchConfigs.set(strandId, resumeConfig);
925
1136
  instance.status = 'starting';
926
1137
  try {
927
- await this.buildStrandRuntime(instance, resumeConfig);
1138
+ await this.trackRuntimeBuild(strandId, this.buildStrandRuntime(instance, resumeConfig));
928
1139
  timing('[resumeStrand:%s] total: %dms', strandId, Math.round(performance.now() - tTotal));
929
1140
  log('Strand %s resumed successfully', strandId);
930
1141
  return instance;
@@ -952,7 +1163,8 @@ export class StrandInstanceManager {
952
1163
  * - `'already-founder'` — the retained config already founds; nothing to do.
953
1164
  * - `'bootstrapped'` — config flipped and the live database ran the bootstrap.
954
1165
  * - `'needs-resume'` — config flipped, but the instance is quiesced (no live
955
- * database), so the bootstrap could not run here: the CALLER must wake the
1166
+ * database) or being quiesced (its database is closing), so the bootstrap could
1167
+ * not run here: the CALLER must wake the
956
1168
  * strand (`CadreNode.wakeStrand`, which owns the hibernation bookkeeping this
957
1169
  * manager does not) so the rebuild — which now founds — runs it.
958
1170
  * @param resolvePartyKey - Asked for the party's own membership key when (and only
@@ -981,7 +1193,7 @@ export class StrandInstanceManager {
981
1193
  // A fresh object rather than mutating in place: startStrand retains the CALLER'S
982
1194
  // config object, which is not ours to rewrite.
983
1195
  this.launchConfigs.set(strandId, { ...config, founder: true, partyMemberPrivateKey });
984
- if (!instance.database && !this.firstSyncGates.has(strandId)) {
1196
+ if (this.isQuiescing(strandId) || (!instance.database && !this.firstSyncGates.has(strandId))) {
985
1197
  return 'needs-resume';
986
1198
  }
987
1199
  try {
@@ -993,6 +1205,70 @@ export class StrandInstanceManager {
993
1205
  }
994
1206
  return 'bootstrapped';
995
1207
  }
1208
+ /**
1209
+ * Give a tracked storage replica ({@link StartStrandConfig.sAppConfig} absent) the app's
1210
+ * schema in place — the seam for an app on this machine claiming a strand this node was
1211
+ * already hosting. No runtime rebuild: the libp2p node, its peer id and connections, the
1212
+ * store and its warm cache all stay, and the new `App` tables read the blocks the replica
1213
+ * already holds. Callers attach BEFORE {@link foundExistingStrand}: the founder bootstrap
1214
+ * writes the sApp into `Strand.Header` and refuses a database that has none.
1215
+ *
1216
+ * A runtime build in flight (the replica's launch, or a hibernation wake) is waited out
1217
+ * first, so the attach acts on the database that build produced. Then, in order: the
1218
+ * schema signature is checked before anything changes; the retained launch config takes
1219
+ * the sApp, so every later rebuild applies it through `composeStrand`; the live database —
1220
+ * published, or still held by the first-sync gate — gets the schema; and only then does
1221
+ * the instance record `sAppInfo` and the sApp's latency hint. A quiesced instance gets the
1222
+ * config alone, and its next resume applies the schema.
1223
+ *
1224
+ * @returns `'already-attached'` when the instance already runs an sApp — a DIFFERENT one
1225
+ * is logged and left as it is, as a second claim of a claimed strand always has been —
1226
+ * else `'attached'`.
1227
+ * @throws when the strand is not tracked (a caller bug, or a launch that failed while
1228
+ * this waited on it); when the schema signature is refused, having changed nothing; and
1229
+ * whatever the live apply throws (e.g. a quiesce closed the database mid-apply) — the
1230
+ * retained config then still carries the sApp and `sAppInfo` stays unset, so the next
1231
+ * claim retries the apply.
1232
+ *
1233
+ * NOTE: two claims of one replica racing each other both pass the `sAppInfo` check and
1234
+ * apply concurrently; the loser of the declarative diff can reject, and its `addStrand`
1235
+ * retry then resolves `'already-attached'`. Needs an app calling `addStrand` twice at once
1236
+ * (or beside a watcher retry); if it is ever seen, chain attaches per strand id.
1237
+ */
1238
+ async attachSApp(strandId, sAppConfig, options = {}) {
1239
+ await this.settleRuntimeBuilds(strandId);
1240
+ const instance = this.instances.get(strandId);
1241
+ const config = this.launchConfigs.get(strandId);
1242
+ if (!instance || !config) {
1243
+ throw new Error(`Cannot attach an sApp to strand ${strandId}: not tracked`);
1244
+ }
1245
+ if (instance.sAppInfo) {
1246
+ if (instance.sAppInfo.id !== sAppConfig.id) {
1247
+ log('attachSApp: strand %s already runs sApp %s — the claim for sApp %s is ignored', strandId, instance.sAppInfo.id, sAppConfig.id);
1248
+ }
1249
+ return 'already-attached';
1250
+ }
1251
+ const sAppInfo = verifiedSAppInfo(strandId, sAppConfig, options.requireSignedSchemas);
1252
+ // A fresh object rather than mutating in place: startStrand retains the CALLER'S
1253
+ // config object, which is not ours to rewrite.
1254
+ this.launchConfigs.set(strandId, { ...config, sAppConfig });
1255
+ const database = instance.database ?? this.firstSyncGates.get(strandId)?.database;
1256
+ if (database) {
1257
+ await database.attachAppSchema(sAppConfig);
1258
+ }
1259
+ else {
1260
+ log('attachSApp: strand %s is quiesced — its next resume applies the sApp schema', strandId);
1261
+ }
1262
+ instance.sAppInfo = sAppInfo;
1263
+ if (sAppConfig.latencyHint) {
1264
+ // HibernationManager reads the hint whenever it arms a timer, so the app's hint governs
1265
+ // from the next one: a timer already armed runs once at the old duration, and a replica
1266
+ // launched under a realtime default was never tracked, so it stays up until relaunched.
1267
+ instance.latencyHint = sAppConfig.latencyHint;
1268
+ }
1269
+ log('Strand %s upgraded from storage replica to %s', strandId, describeSApp(sAppConfig));
1270
+ return 'attached';
1271
+ }
996
1272
  /**
997
1273
  * Whether `strandId` is launched but still waiting for its first sync — a joiner whose
998
1274
  * runtime is up while its database is withheld (status `'syncing'`, or `'idle'` after
@@ -1044,6 +1320,17 @@ export class StrandInstanceManager {
1044
1320
  waiters.add(waiter);
1045
1321
  });
1046
1322
  }
1323
+ /**
1324
+ * The tracked instance once no runtime build of it is in flight, or `undefined` when none
1325
+ * is tracked by then — never launched, or the launch waited on failed and dropped its
1326
+ * record. For a caller about to hand a tracked instance to an app: until its build settles
1327
+ * the instance is `'starting'` with no database, and a failed launch leaves no instance.
1328
+ * Never rejects; a failed build is reported by the call that started it.
1329
+ */
1330
+ async whenRuntimeBuilt(strandId) {
1331
+ await this.settleRuntimeBuilds(strandId);
1332
+ return this.instances.get(strandId);
1333
+ }
1047
1334
  /**
1048
1335
  * Hand a launch's database to the app: set `instance.database`, retire the gate that
1049
1336
  * held it, release every {@link whenWritable} waiter, and — when the launch had already
@@ -1073,9 +1360,11 @@ export class StrandInstanceManager {
1073
1360
  this.firstSyncGates.delete(strandId);
1074
1361
  }
1075
1362
  instance.database = database;
1076
- instance.lastActivity = new Date();
1077
1363
  const wasGated = instance.status !== 'starting';
1078
1364
  if (wasGated) {
1365
+ // A Header delivered by a peer is strand activity; a publish during bring-up is not
1366
+ // (see the build's end in `buildStrandRuntime`).
1367
+ instance.lastActivity = new Date();
1079
1368
  instance.status = 'active';
1080
1369
  log('Strand %s is now writable: Strand.Header held', strandId);
1081
1370
  }